Line data Source code
1 : /* Medium-level subroutines: convert bit-field store and extract
2 : and shifts, multiplies and divides to rtl instructions.
3 : Copyright (C) 1987-2026 Free Software Foundation, Inc.
4 :
5 : This file is part of GCC.
6 :
7 : GCC is free software; you can redistribute it and/or modify it under
8 : the terms of the GNU General Public License as published by the Free
9 : Software Foundation; either version 3, or (at your option) any later
10 : version.
11 :
12 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 : for more details.
16 :
17 : You should have received a copy of the GNU General Public License
18 : along with GCC; see the file COPYING3. If not see
19 : <http://www.gnu.org/licenses/>. */
20 :
21 : /* Work around tree-optimization/91825. */
22 : #pragma GCC diagnostic warning "-Wmaybe-uninitialized"
23 :
24 : #include "config.h"
25 : #include "system.h"
26 : #include "coretypes.h"
27 : #include "backend.h"
28 : #include "target.h"
29 : #include "rtl.h"
30 : #include "tree.h"
31 : #include "predict.h"
32 : #include "memmodel.h"
33 : #include "tm_p.h"
34 : #include "optabs.h"
35 : #include "expmed.h"
36 : #include "regs.h"
37 : #include "emit-rtl.h"
38 : #include "diagnostic-core.h"
39 : #include "fold-const.h"
40 : #include "stor-layout.h"
41 : #include "dojump.h"
42 : #include "explow.h"
43 : #include "expr.h"
44 : #include "langhooks.h"
45 : #include "tree-vector-builder.h"
46 : #include "recog.h"
47 :
48 : struct target_expmed default_target_expmed;
49 : #if SWITCHABLE_TARGET
50 : struct target_expmed *this_target_expmed = &default_target_expmed;
51 : #endif
52 :
53 : static bool store_integral_bit_field (rtx, opt_scalar_int_mode,
54 : unsigned HOST_WIDE_INT,
55 : unsigned HOST_WIDE_INT,
56 : poly_uint64, poly_uint64,
57 : machine_mode, rtx, bool, bool);
58 : static void store_fixed_bit_field (rtx, opt_scalar_int_mode,
59 : unsigned HOST_WIDE_INT,
60 : unsigned HOST_WIDE_INT,
61 : poly_uint64, poly_uint64,
62 : rtx, scalar_int_mode, bool);
63 : static void store_fixed_bit_field_1 (rtx, scalar_int_mode,
64 : unsigned HOST_WIDE_INT,
65 : unsigned HOST_WIDE_INT,
66 : rtx, scalar_int_mode, bool);
67 : static void store_split_bit_field (rtx, opt_scalar_int_mode,
68 : unsigned HOST_WIDE_INT,
69 : unsigned HOST_WIDE_INT,
70 : poly_uint64, poly_uint64,
71 : rtx, scalar_int_mode, bool);
72 : static rtx extract_integral_bit_field (rtx, opt_scalar_int_mode,
73 : unsigned HOST_WIDE_INT,
74 : unsigned HOST_WIDE_INT, int, rtx,
75 : machine_mode, machine_mode, bool, bool);
76 : static rtx extract_fixed_bit_field (machine_mode, rtx, opt_scalar_int_mode,
77 : unsigned HOST_WIDE_INT,
78 : unsigned HOST_WIDE_INT, rtx, int, bool);
79 : static rtx extract_fixed_bit_field_1 (machine_mode, rtx, scalar_int_mode,
80 : unsigned HOST_WIDE_INT,
81 : unsigned HOST_WIDE_INT, rtx, int, bool);
82 : static rtx lshift_value (machine_mode, unsigned HOST_WIDE_INT, int);
83 : static rtx extract_split_bit_field (rtx, opt_scalar_int_mode,
84 : unsigned HOST_WIDE_INT,
85 : unsigned HOST_WIDE_INT, int, bool);
86 : static void do_cmp_and_jump (rtx, rtx, enum rtx_code, machine_mode, rtx_code_label *);
87 : static rtx expand_smod_pow2 (scalar_int_mode, rtx, HOST_WIDE_INT);
88 : static rtx expand_sdiv_pow2 (scalar_int_mode, rtx, HOST_WIDE_INT);
89 :
90 : /* Return a constant integer mask value of mode MODE with BITSIZE ones
91 : followed by BITPOS zeros, or the complement of that if COMPLEMENT.
92 : The mask is truncated if necessary to the width of mode MODE. The
93 : mask is zero-extended if BITSIZE+BITPOS is too small for MODE. */
94 :
95 : static inline rtx
96 229124 : mask_rtx (scalar_int_mode mode, int bitpos, int bitsize, bool complement)
97 : {
98 229124 : return immed_wide_int_const
99 229124 : (wi::shifted_mask (bitpos, bitsize, complement,
100 229124 : GET_MODE_PRECISION (mode)), mode);
101 : }
102 :
103 : /* Test whether a value is zero of a power of two. */
104 : #define EXACT_POWER_OF_2_OR_ZERO_P(x) \
105 : (((x) & ((x) - HOST_WIDE_INT_1U)) == 0)
106 :
107 : struct init_expmed_rtl
108 : {
109 : rtx reg;
110 : rtx plus;
111 : rtx neg;
112 : rtx mult;
113 : rtx sdiv;
114 : rtx udiv;
115 : rtx sdiv_32;
116 : rtx smod_32;
117 : rtx wide_mult;
118 : rtx wide_lshr;
119 : rtx wide_trunc;
120 : rtx shift;
121 : rtx shift_mult;
122 : rtx shift_add;
123 : rtx shift_sub0;
124 : rtx shift_sub1;
125 : rtx zext;
126 : rtx trunc;
127 :
128 : rtx pow2[MAX_BITS_PER_WORD];
129 : rtx cint[MAX_BITS_PER_WORD];
130 : };
131 :
132 : static void
133 30681700 : init_expmed_one_conv (struct init_expmed_rtl *all, scalar_int_mode to_mode,
134 : scalar_int_mode from_mode, bool speed)
135 : {
136 30681700 : int to_size, from_size;
137 30681700 : rtx which;
138 :
139 30681700 : to_size = GET_MODE_PRECISION (to_mode);
140 30681700 : from_size = GET_MODE_PRECISION (from_mode);
141 :
142 : /* Most partial integers have a precision less than the "full"
143 : integer it requires for storage. In case one doesn't, for
144 : comparison purposes here, reduce the bit size by one in that
145 : case. */
146 30681700 : if (GET_MODE_CLASS (to_mode) == MODE_PARTIAL_INT
147 30681700 : && pow2p_hwi (to_size))
148 6136340 : to_size --;
149 30681700 : if (GET_MODE_CLASS (from_mode) == MODE_PARTIAL_INT
150 30681700 : && pow2p_hwi (from_size))
151 0 : from_size --;
152 :
153 : /* Assume cost of zero-extend and sign-extend is the same. */
154 30681700 : which = (to_size < from_size ? all->trunc : all->zext);
155 :
156 30681700 : PUT_MODE (all->reg, from_mode);
157 30681700 : set_convert_cost (to_mode, from_mode, speed,
158 : set_src_cost (which, to_mode, speed));
159 : /* Restore all->reg's mode. */
160 30681700 : PUT_MODE (all->reg, to_mode);
161 30681700 : }
162 :
163 : static void
164 17970710 : init_expmed_one_mode (struct init_expmed_rtl *all,
165 : machine_mode mode, int speed)
166 : {
167 17970710 : int m, n, mode_bitsize;
168 17970710 : machine_mode mode_from;
169 :
170 17970710 : mode_bitsize = GET_MODE_UNIT_BITSIZE (mode);
171 :
172 17970710 : PUT_MODE (all->reg, mode);
173 17970710 : PUT_MODE (all->plus, mode);
174 17970710 : PUT_MODE (all->neg, mode);
175 17970710 : PUT_MODE (all->mult, mode);
176 17970710 : PUT_MODE (all->sdiv, mode);
177 17970710 : PUT_MODE (all->udiv, mode);
178 17970710 : PUT_MODE (all->sdiv_32, mode);
179 17970710 : PUT_MODE (all->smod_32, mode);
180 17970710 : PUT_MODE (all->wide_trunc, mode);
181 17970710 : PUT_MODE (all->shift, mode);
182 17970710 : PUT_MODE (all->shift_mult, mode);
183 17970710 : PUT_MODE (all->shift_add, mode);
184 17970710 : PUT_MODE (all->shift_sub0, mode);
185 17970710 : PUT_MODE (all->shift_sub1, mode);
186 17970710 : PUT_MODE (all->zext, mode);
187 17970710 : PUT_MODE (all->trunc, mode);
188 :
189 17970710 : set_add_cost (speed, mode, set_src_cost (all->plus, mode, speed));
190 17970710 : set_neg_cost (speed, mode, set_src_cost (all->neg, mode, speed));
191 17970710 : set_mul_cost (speed, mode, set_src_cost (all->mult, mode, speed));
192 17970710 : set_sdiv_cost (speed, mode, set_src_cost (all->sdiv, mode, speed));
193 17970710 : set_udiv_cost (speed, mode, set_src_cost (all->udiv, mode, speed));
194 :
195 17970710 : set_sdiv_pow2_cheap (speed, mode, (set_src_cost (all->sdiv_32, mode, speed)
196 17970710 : <= 2 * add_cost (speed, mode)));
197 17970710 : set_smod_pow2_cheap (speed, mode, (set_src_cost (all->smod_32, mode, speed)
198 17970710 : <= 4 * add_cost (speed, mode)));
199 :
200 17970710 : set_shift_cost (speed, mode, 0, 0);
201 17970710 : {
202 17970710 : int cost = add_cost (speed, mode);
203 17970710 : set_shiftadd_cost (speed, mode, 0, cost);
204 17970710 : set_shiftsub0_cost (speed, mode, 0, cost);
205 17970710 : set_shiftsub1_cost (speed, mode, 0, cost);
206 : }
207 :
208 17970710 : n = MIN (MAX_BITS_PER_WORD, mode_bitsize);
209 603114560 : for (m = 1; m < n; m++)
210 : {
211 585143850 : XEXP (all->shift, 1) = all->cint[m];
212 585143850 : XEXP (all->shift_mult, 1) = all->pow2[m];
213 :
214 585143850 : set_shift_cost (speed, mode, m, set_src_cost (all->shift, mode, speed));
215 585143850 : set_shiftadd_cost (speed, mode, m, set_src_cost (all->shift_add, mode,
216 : speed));
217 585143850 : set_shiftsub0_cost (speed, mode, m, set_src_cost (all->shift_sub0, mode,
218 : speed));
219 585143850 : set_shiftsub1_cost (speed, mode, m, set_src_cost (all->shift_sub1, mode,
220 : speed));
221 : }
222 :
223 17970710 : scalar_int_mode int_mode_to;
224 17970710 : if (is_a <scalar_int_mode> (mode, &int_mode_to))
225 : {
226 35064800 : for (mode_from = MIN_MODE_INT; mode_from <= MAX_MODE_INT;
227 30681700 : mode_from = (machine_mode)(mode_from + 1))
228 30681700 : init_expmed_one_conv (all, int_mode_to,
229 : as_a <scalar_int_mode> (mode_from), speed);
230 :
231 4383100 : scalar_int_mode wider_mode;
232 4383100 : if (GET_MODE_CLASS (int_mode_to) == MODE_INT
233 4383100 : && GET_MODE_WIDER_MODE (int_mode_to).exists (&wider_mode))
234 : {
235 2629860 : PUT_MODE (all->reg, mode);
236 2629860 : PUT_MODE (all->zext, wider_mode);
237 2629860 : PUT_MODE (all->wide_mult, wider_mode);
238 2629860 : PUT_MODE (all->wide_lshr, wider_mode);
239 2629860 : XEXP (all->wide_lshr, 1)
240 2629860 : = gen_int_shift_amount (wider_mode, mode_bitsize);
241 :
242 2629860 : set_mul_widen_cost (speed, wider_mode,
243 : set_src_cost (all->wide_mult, wider_mode, speed));
244 2629860 : set_mul_highpart_cost (speed, int_mode_to,
245 : set_src_cost (all->wide_trunc,
246 : int_mode_to, speed));
247 : }
248 : }
249 17970710 : }
250 :
251 : void
252 219155 : init_expmed (void)
253 : {
254 219155 : struct init_expmed_rtl all;
255 219155 : machine_mode mode = QImode;
256 219155 : int m, speed;
257 :
258 219155 : memset (&all, 0, sizeof all);
259 14025920 : for (m = 1; m < MAX_BITS_PER_WORD; m++)
260 : {
261 13806765 : all.pow2[m] = GEN_INT (HOST_WIDE_INT_1 << m);
262 13806765 : all.cint[m] = GEN_INT (m);
263 : }
264 :
265 : /* Avoid using hard regs in ways which may be unsupported. */
266 219155 : all.reg = gen_raw_REG (mode, LAST_VIRTUAL_REGISTER + 1);
267 219155 : all.plus = gen_rtx_PLUS (mode, all.reg, all.reg);
268 219155 : all.neg = gen_rtx_NEG (mode, all.reg);
269 219155 : all.mult = gen_rtx_MULT (mode, all.reg, all.reg);
270 219155 : all.sdiv = gen_rtx_DIV (mode, all.reg, all.reg);
271 219155 : all.udiv = gen_rtx_UDIV (mode, all.reg, all.reg);
272 219155 : all.sdiv_32 = gen_rtx_DIV (mode, all.reg, all.pow2[5]);
273 219155 : all.smod_32 = gen_rtx_MOD (mode, all.reg, all.pow2[5]);
274 219155 : all.zext = gen_rtx_ZERO_EXTEND (mode, all.reg);
275 219155 : all.wide_mult = gen_rtx_MULT (mode, all.zext, all.zext);
276 219155 : all.wide_lshr = gen_rtx_LSHIFTRT (mode, all.wide_mult, all.reg);
277 219155 : all.wide_trunc = gen_rtx_TRUNCATE (mode, all.wide_lshr);
278 219155 : all.shift = gen_rtx_ASHIFT (mode, all.reg, all.reg);
279 219155 : all.shift_mult = gen_rtx_MULT (mode, all.reg, all.reg);
280 219155 : all.shift_add = gen_rtx_PLUS (mode, all.shift_mult, all.reg);
281 219155 : all.shift_sub0 = gen_rtx_MINUS (mode, all.shift_mult, all.reg);
282 219155 : all.shift_sub1 = gen_rtx_MINUS (mode, all.reg, all.shift_mult);
283 219155 : all.trunc = gen_rtx_TRUNCATE (mode, all.reg);
284 :
285 657465 : for (speed = 0; speed < 2; speed++)
286 : {
287 438310 : crtl->maybe_hot_insn_p = speed;
288 438310 : set_zero_cost (speed, set_src_cost (const0_rtx, QImode, speed));
289 :
290 3506480 : for (mode = MIN_MODE_INT; mode <= MAX_MODE_INT;
291 3068170 : mode = (machine_mode)(mode + 1))
292 3068170 : init_expmed_one_mode (&all, mode, speed);
293 :
294 : if (MIN_MODE_PARTIAL_INT != VOIDmode)
295 1753240 : for (mode = MIN_MODE_PARTIAL_INT; mode <= MAX_MODE_PARTIAL_INT;
296 1314930 : mode = (machine_mode)(mode + 1))
297 1314930 : init_expmed_one_mode (&all, mode, speed);
298 :
299 : if (MIN_MODE_VECTOR_INT != VOIDmode)
300 14025920 : for (mode = MIN_MODE_VECTOR_INT; mode <= MAX_MODE_VECTOR_INT;
301 13587610 : mode = (machine_mode)(mode + 1))
302 13587610 : init_expmed_one_mode (&all, mode, speed);
303 : }
304 :
305 219155 : if (alg_hash_used_p ())
306 : {
307 1057 : struct alg_hash_entry *p = alg_hash_entry_ptr (0);
308 1057 : memset (p, 0, sizeof (*p) * NUM_ALG_HASH_ENTRIES);
309 : }
310 : else
311 218098 : set_alg_hash_used_p (true);
312 219155 : default_rtl_profile ();
313 :
314 219155 : ggc_free (all.trunc);
315 219155 : ggc_free (all.shift_sub1);
316 219155 : ggc_free (all.shift_sub0);
317 219155 : ggc_free (all.shift_add);
318 219155 : ggc_free (all.shift_mult);
319 219155 : ggc_free (all.shift);
320 219155 : ggc_free (all.wide_trunc);
321 219155 : ggc_free (all.wide_lshr);
322 219155 : ggc_free (all.wide_mult);
323 219155 : ggc_free (all.zext);
324 219155 : ggc_free (all.smod_32);
325 219155 : ggc_free (all.sdiv_32);
326 219155 : ggc_free (all.udiv);
327 219155 : ggc_free (all.sdiv);
328 219155 : ggc_free (all.mult);
329 219155 : ggc_free (all.neg);
330 219155 : ggc_free (all.plus);
331 219155 : ggc_free (all.reg);
332 219155 : }
333 :
334 : /* Return an rtx representing minus the value of X.
335 : MODE is the intended mode of the result,
336 : useful if X is a CONST_INT. */
337 :
338 : rtx
339 1038865 : negate_rtx (machine_mode mode, rtx x)
340 : {
341 1038865 : rtx result = simplify_unary_operation (NEG, mode, x, mode);
342 :
343 1038865 : if (result == 0)
344 1998 : result = expand_unop (mode, neg_optab, x, NULL_RTX, 0);
345 :
346 1038865 : return result;
347 : }
348 :
349 : /* Whether reverse storage order is supported on the target. */
350 : static int reverse_storage_order_supported = -1;
351 :
352 : /* Check whether reverse storage order is supported on the target. */
353 :
354 : static void
355 286 : check_reverse_storage_order_support (void)
356 : {
357 286 : if (BYTES_BIG_ENDIAN != WORDS_BIG_ENDIAN)
358 : {
359 : reverse_storage_order_supported = 0;
360 : sorry ("reverse scalar storage order");
361 : }
362 : else
363 286 : reverse_storage_order_supported = 1;
364 286 : }
365 :
366 : /* Whether reverse FP storage order is supported on the target. */
367 : static int reverse_float_storage_order_supported = -1;
368 :
369 : /* Check whether reverse FP storage order is supported on the target. */
370 :
371 : static void
372 55 : check_reverse_float_storage_order_support (void)
373 : {
374 55 : if (FLOAT_WORDS_BIG_ENDIAN != WORDS_BIG_ENDIAN)
375 : {
376 : reverse_float_storage_order_supported = 0;
377 : sorry ("reverse floating-point scalar storage order");
378 : }
379 : else
380 55 : reverse_float_storage_order_supported = 1;
381 55 : }
382 :
383 : /* Return an rtx representing value of X with reverse storage order.
384 : MODE is the intended mode of the result,
385 : useful if X is a CONST_INT. */
386 :
387 : rtx
388 3266 : flip_storage_order (machine_mode mode, rtx x)
389 : {
390 3266 : scalar_int_mode int_mode;
391 3266 : rtx result;
392 :
393 3266 : if (mode == QImode)
394 : return x;
395 :
396 2415 : if (COMPLEX_MODE_P (mode))
397 : {
398 44 : rtx real = read_complex_part (x, false);
399 44 : rtx imag = read_complex_part (x, true);
400 :
401 88 : real = flip_storage_order (GET_MODE_INNER (mode), real);
402 88 : imag = flip_storage_order (GET_MODE_INNER (mode), imag);
403 :
404 44 : return gen_rtx_CONCAT (mode, real, imag);
405 : }
406 :
407 2371 : if (UNLIKELY (reverse_storage_order_supported < 0))
408 286 : check_reverse_storage_order_support ();
409 :
410 2371 : if (!is_a <scalar_int_mode> (mode, &int_mode))
411 : {
412 243 : if (FLOAT_MODE_P (mode)
413 243 : && UNLIKELY (reverse_float_storage_order_supported < 0))
414 55 : check_reverse_float_storage_order_support ();
415 :
416 243 : if (!int_mode_for_size (GET_MODE_PRECISION (mode), 0).exists (&int_mode)
417 243 : || !targetm.scalar_mode_supported_p (int_mode))
418 : {
419 0 : sorry ("reverse storage order for %smode", GET_MODE_NAME (mode));
420 0 : return x;
421 : }
422 243 : x = gen_lowpart (int_mode, x);
423 : }
424 :
425 2371 : result = simplify_unary_operation (BSWAP, int_mode, x, int_mode);
426 2371 : if (result == 0)
427 1051 : result = expand_unop (int_mode, bswap_optab, x, NULL_RTX, 1);
428 :
429 2371 : if (int_mode != mode)
430 243 : result = gen_lowpart (mode, result);
431 :
432 : return result;
433 : }
434 :
435 : /* If MODE is set, adjust bitfield memory MEM so that it points to the
436 : first unit of mode MODE that contains a bitfield of size BITSIZE at
437 : bit position BITNUM. If MODE is not set, return a BLKmode reference
438 : to every byte in the bitfield. Set *NEW_BITNUM to the bit position
439 : of the field within the new memory. */
440 :
441 : static rtx
442 422983 : narrow_bit_field_mem (rtx mem, opt_scalar_int_mode mode,
443 : unsigned HOST_WIDE_INT bitsize,
444 : unsigned HOST_WIDE_INT bitnum,
445 : unsigned HOST_WIDE_INT *new_bitnum)
446 : {
447 422983 : scalar_int_mode imode;
448 422983 : if (mode.exists (&imode))
449 : {
450 422983 : unsigned int unit = GET_MODE_BITSIZE (imode);
451 422983 : *new_bitnum = bitnum % unit;
452 422983 : HOST_WIDE_INT offset = (bitnum - *new_bitnum) / BITS_PER_UNIT;
453 422983 : return adjust_bitfield_address (mem, imode, offset);
454 : }
455 : else
456 : {
457 0 : *new_bitnum = bitnum % BITS_PER_UNIT;
458 0 : HOST_WIDE_INT offset = bitnum / BITS_PER_UNIT;
459 0 : HOST_WIDE_INT size = ((*new_bitnum + bitsize + BITS_PER_UNIT - 1)
460 0 : / BITS_PER_UNIT);
461 0 : return adjust_bitfield_address_size (mem, BLKmode, offset, size);
462 : }
463 : }
464 :
465 : /* The caller wants to perform insertion or extraction PATTERN on a
466 : bitfield of size BITSIZE at BITNUM bits into memory operand OP0.
467 : BITREGION_START and BITREGION_END are as for store_bit_field
468 : and FIELDMODE is the natural mode of the field.
469 :
470 : Search for a mode that is compatible with the memory access
471 : restrictions and (where applicable) with a register insertion or
472 : extraction. Return the new memory on success, storing the adjusted
473 : bit position in *NEW_BITNUM. Return null otherwise. */
474 :
475 : static rtx
476 204330 : adjust_bit_field_mem_for_reg (enum extraction_pattern pattern,
477 : rtx op0, HOST_WIDE_INT bitsize,
478 : HOST_WIDE_INT bitnum,
479 : poly_uint64 bitregion_start,
480 : poly_uint64 bitregion_end,
481 : machine_mode fieldmode,
482 : unsigned HOST_WIDE_INT *new_bitnum)
483 : {
484 408660 : bit_field_mode_iterator iter (bitsize, bitnum, bitregion_start,
485 204330 : bitregion_end, MEM_ALIGN (op0),
486 204334 : MEM_VOLATILE_P (op0));
487 204330 : scalar_int_mode best_mode;
488 204330 : if (iter.next_mode (&best_mode))
489 : {
490 : /* We can use a memory in BEST_MODE. See whether this is true for
491 : any wider modes. All other things being equal, we prefer to
492 : use the widest mode possible because it tends to expose more
493 : CSE opportunities. */
494 197827 : if (!iter.prefer_smaller_modes ())
495 : {
496 : /* Limit the search to the mode required by the corresponding
497 : register insertion or extraction instruction, if any. */
498 317 : scalar_int_mode limit_mode = word_mode;
499 317 : extraction_insn insn;
500 634 : if (get_best_reg_extraction_insn (&insn, pattern,
501 317 : GET_MODE_BITSIZE (best_mode),
502 : fieldmode))
503 317 : limit_mode = insn.field_mode;
504 :
505 317 : scalar_int_mode wider_mode;
506 317 : while (iter.next_mode (&wider_mode)
507 1052 : && GET_MODE_SIZE (wider_mode) <= GET_MODE_SIZE (limit_mode))
508 111 : best_mode = wider_mode;
509 : }
510 197827 : return narrow_bit_field_mem (op0, best_mode, bitsize, bitnum,
511 : new_bitnum);
512 : }
513 : return NULL_RTX;
514 : }
515 :
516 : /* Return true if a bitfield of size BITSIZE at bit number BITNUM within
517 : a structure of mode STRUCT_MODE represents a lowpart subreg. The subreg
518 : offset is then BITNUM / BITS_PER_UNIT. */
519 :
520 : static bool
521 828345 : lowpart_bit_field_p (poly_uint64 bitnum, poly_uint64 bitsize,
522 : machine_mode struct_mode)
523 : {
524 828345 : poly_uint64 regsize = REGMODE_NATURAL_SIZE (struct_mode);
525 828345 : if (BYTES_BIG_ENDIAN)
526 : return (multiple_p (bitnum, BITS_PER_UNIT)
527 : && (known_eq (bitnum + bitsize, GET_MODE_BITSIZE (struct_mode))
528 : || multiple_p (bitnum + bitsize,
529 : regsize * BITS_PER_UNIT)));
530 : else
531 828345 : return multiple_p (bitnum, regsize * BITS_PER_UNIT);
532 : }
533 :
534 : /* Return true if -fstrict-volatile-bitfields applies to an access of OP0
535 : containing BITSIZE bits starting at BITNUM, with field mode FIELDMODE.
536 : Return false if the access would touch memory outside the range
537 : BITREGION_START to BITREGION_END for conformance to the C++ memory
538 : model. */
539 :
540 : static bool
541 1590523 : strict_volatile_bitfield_p (rtx op0, unsigned HOST_WIDE_INT bitsize,
542 : unsigned HOST_WIDE_INT bitnum,
543 : scalar_int_mode fieldmode,
544 : poly_uint64 bitregion_start,
545 : poly_uint64 bitregion_end)
546 : {
547 1590523 : unsigned HOST_WIDE_INT modesize = GET_MODE_BITSIZE (fieldmode);
548 :
549 : /* -fstrict-volatile-bitfields must be enabled and we must have a
550 : volatile MEM. */
551 1590523 : if (!MEM_P (op0)
552 163196 : || !MEM_VOLATILE_P (op0)
553 1590735 : || flag_strict_volatile_bitfields <= 0)
554 : return false;
555 :
556 : /* The bit size must not be larger than the field mode, and
557 : the field mode must not be larger than a word. */
558 14 : if (bitsize > modesize || modesize > BITS_PER_WORD)
559 : return false;
560 :
561 : /* Check for cases of unaligned fields that must be split. */
562 14 : if (bitnum % modesize + bitsize > modesize)
563 : return false;
564 :
565 : /* The memory must be sufficiently aligned for a MODESIZE access.
566 : This condition guarantees, that the memory access will not
567 : touch anything after the end of the structure. */
568 11 : if (MEM_ALIGN (op0) < modesize)
569 : return false;
570 :
571 : /* Check for cases where the C++ memory model applies. */
572 11 : if (maybe_ne (bitregion_end, 0U)
573 11 : && (maybe_lt (bitnum - bitnum % modesize, bitregion_start)
574 4 : || maybe_gt (bitnum - bitnum % modesize + modesize - 1,
575 : bitregion_end)))
576 0 : return false;
577 :
578 : return true;
579 : }
580 :
581 : /* Return true if OP is a memory and if a bitfield of size BITSIZE at
582 : bit number BITNUM can be treated as a simple value of mode MODE.
583 : Store the byte offset in *BYTENUM if so. */
584 :
585 : static bool
586 554767 : simple_mem_bitfield_p (rtx op0, poly_uint64 bitsize, poly_uint64 bitnum,
587 : machine_mode mode, poly_uint64 *bytenum)
588 : {
589 554767 : return (MEM_P (op0)
590 257299 : && multiple_p (bitnum, BITS_PER_UNIT, bytenum)
591 205542 : && known_eq (bitsize, GET_MODE_BITSIZE (mode))
592 604868 : && (!targetm.slow_unaligned_access (mode, MEM_ALIGN (op0))
593 0 : || (multiple_p (bitnum, GET_MODE_ALIGNMENT (mode))
594 0 : && MEM_ALIGN (op0) >= GET_MODE_ALIGNMENT (mode))));
595 : }
596 :
597 : /* Try to use instruction INSV to store VALUE into a field of OP0.
598 : If OP0_MODE is defined, it is the mode of OP0, otherwise OP0 is a
599 : BLKmode MEM. VALUE_MODE is the mode of VALUE. BITSIZE and BITNUM
600 : are as for store_bit_field. */
601 :
602 : static bool
603 110462 : store_bit_field_using_insv (const extraction_insn *insv, rtx op0,
604 : opt_scalar_int_mode op0_mode,
605 : unsigned HOST_WIDE_INT bitsize,
606 : unsigned HOST_WIDE_INT bitnum,
607 : rtx value, scalar_int_mode value_mode)
608 : {
609 110462 : class expand_operand ops[4];
610 110462 : rtx value1;
611 110462 : rtx xop0 = op0;
612 110462 : rtx_insn *last = get_last_insn ();
613 110462 : bool copy_back = false;
614 :
615 110462 : scalar_int_mode op_mode = insv->field_mode;
616 110462 : unsigned int unit = GET_MODE_BITSIZE (op_mode);
617 110462 : if (bitsize == 0 || bitsize > unit)
618 : return false;
619 :
620 110453 : if (MEM_P (xop0))
621 : /* Get a reference to the first byte of the field. */
622 0 : xop0 = narrow_bit_field_mem (xop0, insv->struct_mode, bitsize, bitnum,
623 : &bitnum);
624 : else
625 : {
626 : /* Convert from counting within OP0 to counting in OP_MODE. */
627 110453 : if (BYTES_BIG_ENDIAN)
628 : bitnum += unit - GET_MODE_BITSIZE (op0_mode.require ());
629 :
630 : /* If xop0 is a register, we need it in OP_MODE
631 : to make it acceptable to the format of insv. */
632 110453 : if (GET_CODE (xop0) == SUBREG)
633 : {
634 : /* If such a SUBREG can't be created, give up. */
635 35172 : if (!validate_subreg (op_mode, GET_MODE (SUBREG_REG (xop0)),
636 35172 : SUBREG_REG (xop0), SUBREG_BYTE (xop0)))
637 : return false;
638 : /* We can't just change the mode, because this might clobber op0,
639 : and we will need the original value of op0 if insv fails. */
640 35172 : xop0 = gen_rtx_SUBREG (op_mode, SUBREG_REG (xop0),
641 35172 : SUBREG_BYTE (xop0));
642 : }
643 110453 : if (REG_P (xop0) && GET_MODE (xop0) != op_mode)
644 26776 : xop0 = gen_lowpart_SUBREG (op_mode, xop0);
645 : }
646 :
647 : /* If the destination is a paradoxical subreg such that we need a
648 : truncate to the inner mode, perform the insertion on a temporary and
649 : truncate the result to the original destination. Note that we can't
650 : just truncate the paradoxical subreg as (truncate:N (subreg:W (reg:N
651 : X) 0)) is (reg:N X). */
652 110453 : if (GET_CODE (xop0) == SUBREG
653 61948 : && REG_P (SUBREG_REG (xop0))
654 172401 : && !TRULY_NOOP_TRUNCATION_MODES_P (GET_MODE (SUBREG_REG (xop0)),
655 : op_mode))
656 : {
657 0 : rtx tem = gen_reg_rtx (op_mode);
658 0 : emit_move_insn (tem, xop0);
659 0 : xop0 = tem;
660 0 : copy_back = true;
661 : }
662 :
663 : /* There are similar overflow check at the start of store_bit_field_1,
664 : but that only check the situation where the field lies completely
665 : outside the register, while there do have situation where the field
666 : lies partially in the register, we need to adjust bitsize for this
667 : partial overflow situation. Without this fix, pr48335-2.c on big-endian
668 : will broken on those arch support bit insert instruction, like arm, aarch64
669 : etc. */
670 110453 : if (bitsize + bitnum > unit && bitnum < unit)
671 : {
672 2 : warning (OPT_Wextra, "write of %wu-bit data outside the bound of "
673 : "destination object, data truncated into %wu-bit",
674 : bitsize, unit - bitnum);
675 2 : bitsize = unit - bitnum;
676 : }
677 :
678 : /* If BITS_BIG_ENDIAN is zero on a BYTES_BIG_ENDIAN machine, we count
679 : "backwards" from the size of the unit we are inserting into.
680 : Otherwise, we count bits from the most significant on a
681 : BYTES/BITS_BIG_ENDIAN machine. */
682 :
683 110453 : if (BITS_BIG_ENDIAN != BYTES_BIG_ENDIAN)
684 : bitnum = unit - bitsize - bitnum;
685 :
686 : /* Convert VALUE to op_mode (which insv insn wants) in VALUE1. */
687 110453 : value1 = value;
688 110453 : if (value_mode != op_mode)
689 : {
690 142942 : if (GET_MODE_BITSIZE (value_mode) >= bitsize)
691 : {
692 71471 : rtx tmp;
693 : /* Optimization: Don't bother really extending VALUE
694 : if it has all the bits we will actually use. However,
695 : if we must narrow it, be sure we do it correctly. */
696 :
697 214413 : if (GET_MODE_SIZE (value_mode) < GET_MODE_SIZE (op_mode))
698 : {
699 32711 : tmp = simplify_subreg (op_mode, value1, value_mode, 0);
700 32711 : if (! tmp)
701 32122 : tmp = simplify_gen_subreg (op_mode,
702 : force_reg (value_mode, value1),
703 : value_mode, 0);
704 : }
705 : else
706 : {
707 38760 : if (targetm.mode_rep_extended (op_mode, value_mode) != UNKNOWN)
708 0 : tmp = simplify_gen_unary (TRUNCATE, op_mode,
709 : value1, value_mode);
710 : else
711 : {
712 38760 : tmp = gen_lowpart_if_possible (op_mode, value1);
713 38760 : if (! tmp)
714 0 : tmp = gen_lowpart (op_mode, force_reg (value_mode, value1));
715 : }
716 : }
717 : value1 = tmp;
718 : }
719 0 : else if (CONST_INT_P (value))
720 0 : value1 = gen_int_mode (INTVAL (value), op_mode);
721 : else
722 : /* Parse phase is supposed to make VALUE's data type
723 : match that of the component reference, which is a type
724 : at least as wide as the field; so VALUE should have
725 : a mode that corresponds to that type. */
726 0 : gcc_assert (CONSTANT_P (value));
727 : }
728 :
729 110453 : create_fixed_operand (&ops[0], xop0);
730 110453 : create_integer_operand (&ops[1], bitsize);
731 110453 : create_integer_operand (&ops[2], bitnum);
732 110453 : create_input_operand (&ops[3], value1, op_mode);
733 110453 : if (maybe_expand_insn (insv->icode, 4, ops))
734 : {
735 2076 : if (copy_back)
736 0 : convert_move (op0, xop0, true);
737 2076 : return true;
738 : }
739 108377 : delete_insns_since (last);
740 108377 : return false;
741 : }
742 :
743 : /* A subroutine of store_bit_field, with the same arguments. Return true
744 : if the operation could be implemented.
745 :
746 : If FALLBACK_P is true, fall back to store_fixed_bit_field if we have
747 : no other way of implementing the operation. If FALLBACK_P is false,
748 : return false instead.
749 :
750 : if UNDEFINED_P is true then STR_RTX is undefined and may be set using
751 : a subreg instead. */
752 :
753 : static bool
754 899486 : store_bit_field_1 (rtx str_rtx, poly_uint64 bitsize, poly_uint64 bitnum,
755 : poly_uint64 bitregion_start, poly_uint64 bitregion_end,
756 : machine_mode fieldmode,
757 : rtx value, bool reverse, bool fallback_p, bool undefined_p)
758 : {
759 899486 : rtx op0 = str_rtx;
760 :
761 899492 : while (GET_CODE (op0) == SUBREG)
762 : {
763 6 : bitnum += subreg_memory_offset (op0) * BITS_PER_UNIT;
764 6 : op0 = SUBREG_REG (op0);
765 : }
766 :
767 : /* No action is needed if the target is a register and if the field
768 : lies completely outside that register. This can occur if the source
769 : code contains an out-of-bounds access to a small array. */
770 1723933 : if (REG_P (op0) && known_ge (bitnum, GET_MODE_BITSIZE (GET_MODE (op0))))
771 : return true;
772 :
773 : /* Use vec_set patterns for inserting parts of vectors whenever
774 : available. */
775 899483 : machine_mode outermode = GET_MODE (op0);
776 899483 : scalar_mode innermode = GET_MODE_INNER (outermode);
777 899483 : poly_uint64 pos;
778 897443 : if (VECTOR_MODE_P (outermode)
779 2414 : && !MEM_P (op0)
780 2402 : && optab_handler (vec_set_optab, outermode) != CODE_FOR_nothing
781 1233 : && fieldmode == innermode
782 974 : && known_eq (bitsize, GET_MODE_PRECISION (innermode))
783 900457 : && multiple_p (bitnum, GET_MODE_PRECISION (innermode), &pos))
784 : {
785 974 : class expand_operand ops[3];
786 974 : enum insn_code icode = optab_handler (vec_set_optab, outermode);
787 :
788 974 : create_fixed_operand (&ops[0], op0);
789 974 : create_input_operand (&ops[1], value, innermode);
790 974 : create_integer_operand (&ops[2], pos);
791 974 : if (maybe_expand_insn (icode, 3, ops))
792 974 : return true;
793 : }
794 :
795 : /* If the target is a register, overwriting the entire object, or storing
796 : a full-word or multi-word field can be done with just a SUBREG. */
797 898509 : if (!MEM_P (op0)
798 1721979 : && known_eq (bitsize, GET_MODE_BITSIZE (fieldmode)))
799 : {
800 : /* Use the subreg machinery either to narrow OP0 to the required
801 : words or to cope with mode punning between equal-sized modes.
802 : In the latter case, use subreg on the rhs side, not lhs. */
803 756659 : rtx sub;
804 756659 : poly_uint64 bytenum;
805 756659 : poly_uint64 regsize = REGMODE_NATURAL_SIZE (GET_MODE (op0));
806 756659 : if (known_eq (bitnum, 0U)
807 1138180 : && known_eq (bitsize, GET_MODE_BITSIZE (GET_MODE (op0))))
808 : {
809 59353 : sub = force_subreg (GET_MODE (op0), value, fieldmode, 0);
810 59353 : if (sub)
811 : {
812 59353 : if (reverse)
813 1 : sub = flip_storage_order (GET_MODE (op0), sub);
814 59353 : emit_move_insn (op0, sub);
815 59353 : return true;
816 : }
817 : }
818 889384 : else if (multiple_p (bitnum, BITS_PER_UNIT, &bytenum)
819 697299 : && (undefined_p
820 693645 : || (multiple_p (bitnum, regsize * BITS_PER_UNIT)
821 680947 : && multiple_p (bitsize, regsize * BITS_PER_UNIT)))
822 1294170 : && known_ge (GET_MODE_BITSIZE (GET_MODE (op0)), bitsize))
823 : {
824 647073 : sub = simplify_gen_subreg (fieldmode, op0, GET_MODE (op0), bytenum);
825 647073 : if (sub)
826 : {
827 647071 : if (reverse)
828 0 : value = flip_storage_order (fieldmode, value);
829 647071 : emit_move_insn (sub, value);
830 647071 : return true;
831 : }
832 : }
833 : }
834 :
835 : /* If the target is memory, storing any naturally aligned field can be
836 : done with a simple store. For targets that support fast unaligned
837 : memory, any naturally sized, unit aligned field can be done directly. */
838 192085 : poly_uint64 bytenum;
839 192085 : if (simple_mem_bitfield_p (op0, bitsize, bitnum, fieldmode, &bytenum))
840 : {
841 7053 : op0 = adjust_bitfield_address (op0, fieldmode, bytenum);
842 7053 : if (reverse)
843 0 : value = flip_storage_order (fieldmode, value);
844 7053 : emit_move_insn (op0, value);
845 7053 : return true;
846 : }
847 :
848 : /* It's possible we'll need to handle other cases here for
849 : polynomial bitnum and bitsize. */
850 :
851 : /* From here on we need to be looking at a fixed-size insertion. */
852 185032 : unsigned HOST_WIDE_INT ibitsize = bitsize.to_constant ();
853 185032 : unsigned HOST_WIDE_INT ibitnum = bitnum.to_constant ();
854 :
855 : /* Make sure we are playing with integral modes. Pun with subregs
856 : if we aren't. This must come after the entire register case above,
857 : since that case is valid for any mode. The following cases are only
858 : valid for integral modes. */
859 185032 : opt_scalar_int_mode op0_mode = int_mode_for_mode (GET_MODE (op0));
860 185032 : scalar_int_mode imode;
861 185032 : if (!op0_mode.exists (&imode) || imode != GET_MODE (op0))
862 : {
863 19881 : if (MEM_P (op0))
864 15224 : op0 = adjust_bitfield_address_size (op0, op0_mode.else_blk (),
865 : 0, MEM_SIZE (op0));
866 4657 : else if (!op0_mode.exists ())
867 : {
868 0 : if (ibitnum == 0
869 0 : && known_eq (ibitsize, GET_MODE_BITSIZE (GET_MODE (op0)))
870 0 : && MEM_P (value)
871 0 : && !reverse)
872 : {
873 0 : value = adjust_address (value, GET_MODE (op0), 0);
874 0 : emit_move_insn (op0, value);
875 0 : return true;
876 : }
877 0 : if (!fallback_p)
878 : return false;
879 0 : rtx temp = assign_stack_temp (GET_MODE (op0),
880 0 : GET_MODE_SIZE (GET_MODE (op0)));
881 0 : emit_move_insn (temp, op0);
882 0 : store_bit_field_1 (temp, bitsize, bitnum, 0, 0, fieldmode, value,
883 : reverse, fallback_p, undefined_p);
884 0 : emit_move_insn (op0, temp);
885 0 : return true;
886 : }
887 : else
888 4657 : op0 = gen_lowpart (op0_mode.require (), op0);
889 : }
890 :
891 185032 : return store_integral_bit_field (op0, op0_mode, ibitsize, ibitnum,
892 : bitregion_start, bitregion_end,
893 185032 : fieldmode, value, reverse, fallback_p);
894 : }
895 :
896 : /* Subroutine of store_bit_field_1, with the same arguments, except
897 : that BITSIZE and BITNUM are constant. Handle cases specific to
898 : integral modes. If OP0_MODE is defined, it is the mode of OP0,
899 : otherwise OP0 is a BLKmode MEM. */
900 :
901 : static bool
902 185032 : store_integral_bit_field (rtx op0, opt_scalar_int_mode op0_mode,
903 : unsigned HOST_WIDE_INT bitsize,
904 : unsigned HOST_WIDE_INT bitnum,
905 : poly_uint64 bitregion_start,
906 : poly_uint64 bitregion_end,
907 : machine_mode fieldmode,
908 : rtx value, bool reverse, bool fallback_p)
909 : {
910 : /* Storing an lsb-aligned field in a register
911 : can be done with a movstrict instruction. */
912 :
913 185032 : if (!MEM_P (op0)
914 117046 : && !reverse
915 380271 : && lowpart_bit_field_p (bitnum, bitsize, op0_mode.require ())
916 83983 : && known_eq (bitsize, GET_MODE_BITSIZE (fieldmode))
917 222562 : && optab_handler (movstrict_optab, fieldmode) != CODE_FOR_nothing)
918 : {
919 5763 : class expand_operand ops[2];
920 5763 : enum insn_code icode = optab_handler (movstrict_optab, fieldmode);
921 5763 : rtx arg0 = op0;
922 5763 : unsigned HOST_WIDE_INT subreg_off;
923 :
924 5763 : if (GET_CODE (arg0) == SUBREG)
925 : {
926 : /* Else we've got some float mode source being extracted into
927 : a different float mode destination -- this combination of
928 : subregs results in Severe Tire Damage. */
929 458 : gcc_assert (GET_MODE (SUBREG_REG (arg0)) == fieldmode
930 : || GET_MODE_CLASS (fieldmode) == MODE_INT
931 : || GET_MODE_CLASS (fieldmode) == MODE_PARTIAL_INT);
932 : arg0 = SUBREG_REG (arg0);
933 : }
934 :
935 5763 : subreg_off = bitnum / BITS_PER_UNIT;
936 5797 : if (validate_subreg (fieldmode, GET_MODE (arg0), arg0, subreg_off)
937 : /* STRICT_LOW_PART must have a non-paradoxical subreg as
938 : operand. */
939 5763 : && !paradoxical_subreg_p (fieldmode, GET_MODE (arg0)))
940 : {
941 5729 : arg0 = gen_rtx_SUBREG (fieldmode, arg0, subreg_off);
942 :
943 5729 : create_fixed_operand (&ops[0], arg0);
944 : /* Shrink the source operand to FIELDMODE. */
945 5729 : create_convert_operand_to (&ops[1], value, fieldmode, false);
946 5729 : if (maybe_expand_insn (icode, 2, ops))
947 5728 : return true;
948 : }
949 : }
950 :
951 : /* Handle fields bigger than a word. */
952 :
953 180823 : if (bitsize > BITS_PER_WORD)
954 : {
955 : /* Here we transfer the words of the field
956 : in the order least significant first.
957 : This is because the most significant word is the one which may
958 : be less than full.
959 : However, only do that if the value is not BLKmode. */
960 :
961 915 : const bool backwards = WORDS_BIG_ENDIAN && fieldmode != BLKmode;
962 915 : const int nwords = (bitsize + (BITS_PER_WORD - 1)) / BITS_PER_WORD;
963 915 : rtx_insn *last;
964 :
965 : /* This is the mode we must force value to, so that there will be enough
966 : subwords to extract. Note that fieldmode will often (always?) be
967 : VOIDmode, because that is what store_field uses to indicate that this
968 : is a bit field, but passing VOIDmode to operand_subword_force
969 : is not allowed.
970 :
971 : The mode must be fixed-size, since insertions into variable-sized
972 : objects are meant to be handled before calling this function. */
973 915 : fixed_size_mode value_mode = as_a <fixed_size_mode> (GET_MODE (value));
974 915 : if (value_mode == VOIDmode)
975 24 : value_mode
976 24 : = smallest_int_mode_for_size (nwords * BITS_PER_WORD).require ();
977 :
978 915 : last = get_last_insn ();
979 2740 : for (int i = 0; i < nwords; i++)
980 : {
981 : /* Number of bits to be stored in this iteration, i.e. BITS_PER_WORD
982 : except maybe for the last iteration. */
983 3660 : const unsigned HOST_WIDE_INT new_bitsize
984 1866 : = MIN (BITS_PER_WORD, bitsize - i * BITS_PER_WORD);
985 : /* Bit offset from the starting bit number in the target. */
986 3672 : const unsigned int bit_offset
987 : = backwards ^ reverse
988 1830 : ? MAX ((int) bitsize - (i + 1) * BITS_PER_WORD, 0)
989 : : i * BITS_PER_WORD;
990 :
991 : /* No further action is needed if the target is a register and if
992 : this field lies completely outside that register. */
993 2102 : if (REG_P (op0) && known_ge (bitnum + bit_offset,
994 : GET_MODE_BITSIZE (GET_MODE (op0))))
995 : {
996 5 : if (backwards ^ reverse)
997 0 : continue;
998 : /* For forward operation we are finished. */
999 185032 : return true;
1000 : }
1001 :
1002 : /* Starting word number in the value. */
1003 1825 : const unsigned int wordnum
1004 : = backwards
1005 1825 : ? GET_MODE_SIZE (value_mode) / UNITS_PER_WORD - (i + 1)
1006 : : i;
1007 : /* The chunk of the value in word_mode. We use bit-field extraction
1008 : in BLKmode to handle unaligned memory references and to shift the
1009 : last chunk right on big-endian machines if need be. */
1010 1825 : rtx value_word
1011 : = fieldmode == BLKmode
1012 1861 : ? extract_bit_field (value, new_bitsize, wordnum * BITS_PER_WORD,
1013 : 1, NULL_RTX, word_mode, word_mode, false,
1014 : NULL)
1015 1576 : : operand_subword_force (value, wordnum, value_mode);
1016 :
1017 1825 : if (!store_bit_field_1 (op0, new_bitsize,
1018 1825 : bitnum + bit_offset,
1019 : bitregion_start, bitregion_end,
1020 : word_mode,
1021 : value_word, reverse, fallback_p, false))
1022 : {
1023 0 : delete_insns_since (last);
1024 0 : return false;
1025 : }
1026 : }
1027 : return true;
1028 : }
1029 :
1030 : /* If VALUE has a floating-point or complex mode, access it as an
1031 : integer of the corresponding size. This can occur on a machine
1032 : with 64 bit registers that uses SFmode for float. It can also
1033 : occur for unaligned float or complex fields. */
1034 178389 : rtx orig_value = value;
1035 178389 : scalar_int_mode value_mode;
1036 178389 : if (GET_MODE (value) == VOIDmode)
1037 : /* By this point we've dealt with values that are bigger than a word,
1038 : so word_mode is a conservatively correct choice. */
1039 107049 : value_mode = word_mode;
1040 71340 : else if (!is_a <scalar_int_mode> (GET_MODE (value), &value_mode))
1041 : {
1042 1130 : value_mode = int_mode_for_mode (GET_MODE (value)).require ();
1043 1130 : value = gen_reg_rtx (value_mode);
1044 1130 : emit_move_insn (gen_lowpart (GET_MODE (orig_value), value), orig_value);
1045 : }
1046 :
1047 : /* If OP0 is a multi-word register, narrow it to the affected word.
1048 : If the region spans two words, defer to store_split_bit_field.
1049 : Don't do this if op0 is a single hard register wider than word
1050 : such as a float or vector register. */
1051 178389 : if (!MEM_P (op0)
1052 223233 : && GET_MODE_SIZE (op0_mode.require ()) > UNITS_PER_WORD
1053 213623 : && (!REG_P (op0)
1054 35213 : || !HARD_REGISTER_P (op0)
1055 143157 : || hard_regno_nregs (REGNO (op0), op0_mode.require ()) != 1))
1056 : {
1057 35307 : if (bitnum % BITS_PER_WORD + bitsize > BITS_PER_WORD)
1058 : {
1059 698 : if (!fallback_p)
1060 : return false;
1061 :
1062 71 : store_split_bit_field (op0, op0_mode, bitsize, bitnum,
1063 : bitregion_start, bitregion_end,
1064 : value, value_mode, reverse);
1065 71 : return true;
1066 : }
1067 34534 : rtx new_op0
1068 34534 : = simplify_gen_subreg (word_mode, op0, op0_mode.require (),
1069 34609 : bitnum / BITS_PER_WORD * UNITS_PER_WORD);
1070 34534 : if (!new_op0)
1071 : {
1072 : /* No valid word-mode SUBREG of op0 at this offset. Defer to
1073 : store_split_bit_field, which addresses op0 a word at a time. */
1074 0 : if (!fallback_p)
1075 : return false;
1076 0 : store_split_bit_field (op0, op0_mode, bitsize, bitnum,
1077 : bitregion_start, bitregion_end,
1078 : value, value_mode, reverse);
1079 0 : return true;
1080 : }
1081 34534 : op0 = new_op0;
1082 34534 : op0_mode = word_mode;
1083 34609 : bitnum %= BITS_PER_WORD;
1084 : }
1085 :
1086 : /* From here on we can assume that the field to be stored in fits
1087 : within a word. If the destination is a register, it too fits
1088 : in a word. */
1089 :
1090 177691 : extraction_insn insv;
1091 177691 : if (!MEM_P (op0)
1092 110481 : && !reverse
1093 110464 : && get_best_reg_extraction_insn (&insv, EP_insv,
1094 220928 : GET_MODE_BITSIZE (op0_mode.require ()),
1095 : fieldmode)
1096 288153 : && store_bit_field_using_insv (&insv, op0, op0_mode,
1097 : bitsize, bitnum, value, value_mode))
1098 2076 : return true;
1099 :
1100 : /* If OP0 is a memory, try copying it to a register and seeing if a
1101 : cheap register alternative is available. */
1102 175615 : if (MEM_P (op0) && !reverse)
1103 : {
1104 66810 : if (get_best_mem_extraction_insn (&insv, EP_insv, bitsize, bitnum,
1105 : fieldmode)
1106 66810 : && store_bit_field_using_insv (&insv, op0, op0_mode,
1107 : bitsize, bitnum, value, value_mode))
1108 0 : return true;
1109 :
1110 66810 : rtx_insn *last = get_last_insn ();
1111 :
1112 : /* Try loading part of OP0 into a register, inserting the bitfield
1113 : into that, and then copying the result back to OP0. */
1114 66810 : unsigned HOST_WIDE_INT bitpos;
1115 66810 : rtx xop0 = adjust_bit_field_mem_for_reg (EP_insv, op0, bitsize, bitnum,
1116 : bitregion_start, bitregion_end,
1117 : fieldmode, &bitpos);
1118 66810 : if (xop0)
1119 : {
1120 62973 : rtx tempreg = copy_to_reg (xop0);
1121 62973 : if (store_bit_field_1 (tempreg, bitsize, bitpos,
1122 : bitregion_start, bitregion_end,
1123 : fieldmode, orig_value, reverse, false, false))
1124 : {
1125 0 : emit_move_insn (xop0, tempreg);
1126 0 : return true;
1127 : }
1128 62973 : delete_insns_since (last);
1129 : }
1130 : }
1131 :
1132 175615 : if (!fallback_p)
1133 : return false;
1134 :
1135 113269 : store_fixed_bit_field (op0, op0_mode, bitsize, bitnum, bitregion_start,
1136 : bitregion_end, value, value_mode, reverse);
1137 113269 : return true;
1138 : }
1139 :
1140 : /* Generate code to store value from rtx VALUE
1141 : into a bit-field within structure STR_RTX
1142 : containing BITSIZE bits starting at bit BITNUM.
1143 :
1144 : BITREGION_START is bitpos of the first bitfield in this region.
1145 : BITREGION_END is the bitpos of the ending bitfield in this region.
1146 : These two fields are 0, if the C++ memory model does not apply,
1147 : or we are not interested in keeping track of bitfield regions.
1148 :
1149 : FIELDMODE is the machine-mode of the FIELD_DECL node for this field.
1150 :
1151 : If REVERSE is true, the store is to be done in reverse order.
1152 :
1153 : If UNDEFINED_P is true then STR_RTX is currently undefined. */
1154 :
1155 : void
1156 834688 : store_bit_field (rtx str_rtx, poly_uint64 bitsize, poly_uint64 bitnum,
1157 : poly_uint64 bitregion_start, poly_uint64 bitregion_end,
1158 : machine_mode fieldmode,
1159 : rtx value, bool reverse, bool undefined_p)
1160 : {
1161 : /* Handle -fstrict-volatile-bitfields in the cases where it applies. */
1162 834688 : unsigned HOST_WIDE_INT ibitsize = 0, ibitnum = 0;
1163 834688 : scalar_int_mode int_mode;
1164 834688 : if (bitsize.is_constant (&ibitsize)
1165 834688 : && bitnum.is_constant (&ibitnum)
1166 1574061 : && is_a <scalar_int_mode> (fieldmode, &int_mode)
1167 739377 : && strict_volatile_bitfield_p (str_rtx, ibitsize, ibitnum, int_mode,
1168 : bitregion_start, bitregion_end))
1169 : {
1170 : /* Storing of a full word can be done with a simple store.
1171 : We know here that the field can be accessed with one single
1172 : instruction. For targets that support unaligned memory,
1173 : an unaligned access may be necessary. */
1174 8 : if (ibitsize == GET_MODE_BITSIZE (int_mode))
1175 : {
1176 0 : str_rtx = adjust_bitfield_address (str_rtx, int_mode,
1177 : ibitnum / BITS_PER_UNIT);
1178 0 : if (reverse)
1179 0 : value = flip_storage_order (int_mode, value);
1180 0 : gcc_assert (ibitnum % BITS_PER_UNIT == 0);
1181 0 : emit_move_insn (str_rtx, value);
1182 : }
1183 : else
1184 : {
1185 4 : rtx temp;
1186 :
1187 4 : str_rtx = narrow_bit_field_mem (str_rtx, int_mode, ibitsize,
1188 : ibitnum, &ibitnum);
1189 8 : gcc_assert (ibitnum + ibitsize <= GET_MODE_BITSIZE (int_mode));
1190 4 : temp = copy_to_reg (str_rtx);
1191 4 : if (!store_bit_field_1 (temp, ibitsize, ibitnum, 0, 0,
1192 : int_mode, value, reverse, true, undefined_p))
1193 0 : gcc_unreachable ();
1194 :
1195 4 : emit_move_insn (str_rtx, temp);
1196 : }
1197 :
1198 4 : return;
1199 : }
1200 :
1201 : /* Under the C++0x memory model, we must not touch bits outside the
1202 : bit region. Adjust the address to start at the beginning of the
1203 : bit region. */
1204 834684 : if (MEM_P (str_rtx) && maybe_ne (bitregion_start, 0U))
1205 : {
1206 50946 : scalar_int_mode best_mode;
1207 50946 : machine_mode addr_mode = VOIDmode;
1208 :
1209 50946 : poly_uint64 offset = exact_div (bitregion_start, BITS_PER_UNIT);
1210 101892 : bitnum -= bitregion_start;
1211 50946 : poly_int64 size = bits_to_bytes_round_up (bitnum + bitsize);
1212 50946 : bitregion_end -= bitregion_start;
1213 50946 : bitregion_start = 0;
1214 50946 : if (bitsize.is_constant (&ibitsize)
1215 50946 : && bitnum.is_constant (&ibitnum)
1216 50946 : && get_best_mode (ibitsize, ibitnum,
1217 : bitregion_start, bitregion_end,
1218 50946 : MEM_ALIGN (str_rtx), INT_MAX,
1219 50946 : MEM_VOLATILE_P (str_rtx), &best_mode))
1220 47532 : addr_mode = best_mode;
1221 50946 : str_rtx = adjust_bitfield_address_size (str_rtx, addr_mode,
1222 : offset, size);
1223 : }
1224 :
1225 834684 : if (!store_bit_field_1 (str_rtx, bitsize, bitnum,
1226 : bitregion_start, bitregion_end,
1227 : fieldmode, value, reverse, true, undefined_p))
1228 0 : gcc_unreachable ();
1229 : }
1230 :
1231 : /* Use shifts and boolean operations to store VALUE into a bit field of
1232 : width BITSIZE in OP0, starting at bit BITNUM. If OP0_MODE is defined,
1233 : it is the mode of OP0, otherwise OP0 is a BLKmode MEM. VALUE_MODE is
1234 : the mode of VALUE.
1235 :
1236 : If REVERSE is true, the store is to be done in reverse order. */
1237 :
1238 : static void
1239 130768 : store_fixed_bit_field (rtx op0, opt_scalar_int_mode op0_mode,
1240 : unsigned HOST_WIDE_INT bitsize,
1241 : unsigned HOST_WIDE_INT bitnum,
1242 : poly_uint64 bitregion_start, poly_uint64 bitregion_end,
1243 : rtx value, scalar_int_mode value_mode, bool reverse)
1244 : {
1245 : /* There is a case not handled here:
1246 : a structure with a known alignment of just a halfword
1247 : and a field split across two aligned halfwords within the structure.
1248 : Or likewise a structure with a known alignment of just a byte
1249 : and a field split across two bytes.
1250 : Such cases are not supposed to be able to occur. */
1251 :
1252 130768 : scalar_int_mode best_mode;
1253 130768 : if (MEM_P (op0))
1254 : {
1255 84567 : unsigned int max_bitsize = BITS_PER_WORD;
1256 84567 : scalar_int_mode imode;
1257 144335 : if (op0_mode.exists (&imode) && GET_MODE_BITSIZE (imode) < max_bitsize)
1258 90648 : max_bitsize = GET_MODE_BITSIZE (imode);
1259 :
1260 84567 : if (!get_best_mode (bitsize, bitnum, bitregion_start, bitregion_end,
1261 84567 : MEM_ALIGN (op0), max_bitsize, MEM_VOLATILE_P (op0),
1262 : &best_mode))
1263 : {
1264 : /* The only way this should occur is if the field spans word
1265 : boundaries. */
1266 6569 : store_split_bit_field (op0, op0_mode, bitsize, bitnum,
1267 : bitregion_start, bitregion_end,
1268 : value, value_mode, reverse);
1269 6569 : return;
1270 : }
1271 :
1272 77998 : op0 = narrow_bit_field_mem (op0, best_mode, bitsize, bitnum, &bitnum);
1273 : }
1274 : else
1275 46201 : best_mode = op0_mode.require ();
1276 :
1277 124199 : store_fixed_bit_field_1 (op0, best_mode, bitsize, bitnum,
1278 : value, value_mode, reverse);
1279 : }
1280 :
1281 : /* Helper function for store_fixed_bit_field, stores
1282 : the bit field always using MODE, which is the mode of OP0. The other
1283 : arguments are as for store_fixed_bit_field. */
1284 :
1285 : static void
1286 124199 : store_fixed_bit_field_1 (rtx op0, scalar_int_mode mode,
1287 : unsigned HOST_WIDE_INT bitsize,
1288 : unsigned HOST_WIDE_INT bitnum,
1289 : rtx value, scalar_int_mode value_mode, bool reverse)
1290 : {
1291 124199 : rtx temp;
1292 124199 : int all_zero = 0;
1293 124199 : int all_one = 0;
1294 :
1295 : /* Note that bitsize + bitnum can be greater than GET_MODE_BITSIZE (mode)
1296 : for invalid input, such as f5 from gcc.dg/pr48335-2.c. */
1297 :
1298 124199 : if (reverse ? !BYTES_BIG_ENDIAN : BYTES_BIG_ENDIAN)
1299 : /* BITNUM is the distance between our msb
1300 : and that of the containing datum.
1301 : Convert it to the distance from the lsb. */
1302 1058 : bitnum = GET_MODE_BITSIZE (mode) - bitsize - bitnum;
1303 :
1304 : /* Now BITNUM is always the distance between our lsb
1305 : and that of OP0. */
1306 :
1307 : /* Shift VALUE left by BITNUM bits. If VALUE is not constant,
1308 : we must first convert its mode to MODE. */
1309 :
1310 124199 : if (CONST_INT_P (value))
1311 : {
1312 74657 : unsigned HOST_WIDE_INT v = UINTVAL (value);
1313 :
1314 74657 : if (bitsize < HOST_BITS_PER_WIDE_INT)
1315 74639 : v &= (HOST_WIDE_INT_1U << bitsize) - 1;
1316 :
1317 74657 : if (v == 0)
1318 : all_zero = 1;
1319 59464 : else if ((bitsize < HOST_BITS_PER_WIDE_INT
1320 59456 : && v == (HOST_WIDE_INT_1U << bitsize) - 1)
1321 50972 : || (bitsize == HOST_BITS_PER_WIDE_INT
1322 50972 : && v == HOST_WIDE_INT_M1U))
1323 8492 : all_one = 1;
1324 :
1325 74657 : value = lshift_value (mode, v, bitnum);
1326 : }
1327 : else
1328 : {
1329 49542 : int must_and = (GET_MODE_BITSIZE (value_mode) != bitsize
1330 76856 : && bitnum + bitsize != GET_MODE_BITSIZE (mode));
1331 :
1332 49542 : if (value_mode != mode)
1333 27136 : value = convert_to_mode (mode, value, 1);
1334 :
1335 49542 : if (must_and)
1336 20530 : value = expand_binop (mode, and_optab, value,
1337 : mask_rtx (mode, 0, bitsize, 0),
1338 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
1339 49542 : if (bitnum > 0)
1340 14017 : value = expand_shift (LSHIFT_EXPR, mode, value,
1341 14017 : bitnum, NULL_RTX, 1);
1342 : }
1343 :
1344 124199 : if (reverse)
1345 529 : value = flip_storage_order (mode, value);
1346 :
1347 : /* Now clear the chosen bits in OP0,
1348 : except that if VALUE is -1 we need not bother. */
1349 : /* We keep the intermediates in registers to allow CSE to combine
1350 : consecutive bitfield assignments. */
1351 :
1352 124199 : temp = force_reg (mode, op0);
1353 :
1354 124199 : if (! all_one)
1355 : {
1356 115707 : rtx mask = mask_rtx (mode, bitnum, bitsize, 1);
1357 115707 : if (reverse)
1358 517 : mask = flip_storage_order (mode, mask);
1359 115707 : temp = expand_binop (mode, and_optab, temp, mask,
1360 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
1361 115707 : temp = force_reg (mode, temp);
1362 : }
1363 :
1364 : /* Now logical-or VALUE into OP0, unless it is zero. */
1365 :
1366 124199 : if (! all_zero)
1367 : {
1368 109006 : temp = expand_binop (mode, ior_optab, temp, value,
1369 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
1370 109006 : temp = force_reg (mode, temp);
1371 : }
1372 :
1373 124199 : if (op0 != temp)
1374 : {
1375 124199 : op0 = copy_rtx (op0);
1376 124199 : emit_move_insn (op0, temp);
1377 : }
1378 124199 : }
1379 :
1380 : /* Store a bit field that is split across multiple accessible memory objects.
1381 :
1382 : OP0 is the REG, SUBREG or MEM rtx for the first of the objects.
1383 : BITSIZE is the field width; BITPOS the position of its first bit
1384 : (within the word).
1385 : VALUE is the value to store, which has mode VALUE_MODE.
1386 : If OP0_MODE is defined, it is the mode of OP0, otherwise OP0 is
1387 : a BLKmode MEM.
1388 :
1389 : If REVERSE is true, the store is to be done in reverse order.
1390 :
1391 : This does not yet handle fields wider than BITS_PER_WORD. */
1392 :
1393 : static void
1394 6640 : store_split_bit_field (rtx op0, opt_scalar_int_mode op0_mode,
1395 : unsigned HOST_WIDE_INT bitsize,
1396 : unsigned HOST_WIDE_INT bitpos,
1397 : poly_uint64 bitregion_start, poly_uint64 bitregion_end,
1398 : rtx value, scalar_int_mode value_mode, bool reverse)
1399 : {
1400 6640 : unsigned int unit, total_bits, bitsdone = 0;
1401 :
1402 : /* Make sure UNIT isn't larger than BITS_PER_WORD, we can only handle that
1403 : much at a time. */
1404 6640 : if (REG_P (op0) || GET_CODE (op0) == SUBREG)
1405 71 : unit = BITS_PER_WORD;
1406 : else
1407 6569 : unit = MIN (MEM_ALIGN (op0), BITS_PER_WORD);
1408 :
1409 : /* If OP0 is a memory with a mode, then UNIT must not be larger than
1410 : OP0's mode as well. Otherwise, store_fixed_bit_field will call us
1411 : again, and we will mutually recurse forever. */
1412 6640 : if (MEM_P (op0) && op0_mode.exists ())
1413 5111 : unit = MIN (unit, GET_MODE_BITSIZE (op0_mode.require ()));
1414 :
1415 : /* If VALUE is a constant other than a CONST_INT, get it into a register in
1416 : WORD_MODE. If we can do this using gen_lowpart_common, do so. Note
1417 : that VALUE might be a floating-point constant. */
1418 6640 : if (CONSTANT_P (value) && !CONST_INT_P (value))
1419 : {
1420 0 : rtx word = gen_lowpart_common (word_mode, value);
1421 :
1422 0 : if (word && (value != word))
1423 : value = word;
1424 : else
1425 0 : value = gen_lowpart_common (word_mode, force_reg (value_mode, value));
1426 0 : value_mode = word_mode;
1427 : }
1428 :
1429 6640 : total_bits = GET_MODE_BITSIZE (value_mode);
1430 :
1431 30941 : while (bitsdone < bitsize)
1432 : {
1433 24301 : unsigned HOST_WIDE_INT thissize;
1434 24301 : unsigned HOST_WIDE_INT thispos;
1435 24301 : unsigned HOST_WIDE_INT offset;
1436 24301 : rtx part;
1437 :
1438 24301 : offset = (bitpos + bitsdone) / unit;
1439 24301 : thispos = (bitpos + bitsdone) % unit;
1440 :
1441 : /* When region of bytes we can touch is restricted, decrease
1442 : UNIT close to the end of the region as needed. If op0 is a REG
1443 : or SUBREG of REG, don't do this, as there can't be data races
1444 : on a register and we can expand shorter code in some cases. */
1445 31103 : if (maybe_ne (bitregion_end, 0U)
1446 24301 : && unit > BITS_PER_UNIT
1447 14217 : && maybe_gt (bitpos + bitsdone - thispos + unit, bitregion_end + 1)
1448 6866 : && !REG_P (op0)
1449 31103 : && (GET_CODE (op0) != SUBREG || !REG_P (SUBREG_REG (op0))))
1450 : {
1451 6802 : unit = unit / 2;
1452 6802 : continue;
1453 : }
1454 :
1455 : /* THISSIZE must not overrun a word boundary. Otherwise,
1456 : store_fixed_bit_field will call us again, and we will mutually
1457 : recurse forever. */
1458 17499 : thissize = MIN (bitsize - bitsdone, BITS_PER_WORD);
1459 17499 : thissize = MIN (thissize, unit - thispos);
1460 :
1461 17499 : if (reverse ? !BYTES_BIG_ENDIAN : BYTES_BIG_ENDIAN)
1462 : {
1463 : /* Fetch successively less significant portions. */
1464 214 : if (CONST_INT_P (value))
1465 108 : part = GEN_INT (((unsigned HOST_WIDE_INT) (INTVAL (value))
1466 : >> (bitsize - bitsdone - thissize))
1467 : & ((HOST_WIDE_INT_1 << thissize) - 1));
1468 : /* Likewise, but the source is little-endian. */
1469 106 : else if (reverse)
1470 106 : part = extract_fixed_bit_field (word_mode, value, value_mode,
1471 : thissize,
1472 : bitsize - bitsdone - thissize,
1473 : NULL_RTX, 1, false);
1474 : else
1475 : /* The args are chosen so that the last part includes the
1476 : lsb. Give extract_bit_field the value it needs (with
1477 : endianness compensation) to fetch the piece we want. */
1478 : part = extract_fixed_bit_field (word_mode, value, value_mode,
1479 : thissize,
1480 : total_bits - bitsize + bitsdone,
1481 : NULL_RTX, 1, false);
1482 : }
1483 : else
1484 : {
1485 : /* Fetch successively more significant portions. */
1486 17285 : if (CONST_INT_P (value))
1487 12867 : part = GEN_INT (((unsigned HOST_WIDE_INT) (INTVAL (value))
1488 : >> bitsdone)
1489 : & ((HOST_WIDE_INT_1 << thissize) - 1));
1490 : /* Likewise, but the source is big-endian. */
1491 4418 : else if (reverse)
1492 : part = extract_fixed_bit_field (word_mode, value, value_mode,
1493 : thissize,
1494 : total_bits - bitsdone - thissize,
1495 : NULL_RTX, 1, false);
1496 : else
1497 4418 : part = extract_fixed_bit_field (word_mode, value, value_mode,
1498 : thissize, bitsdone, NULL_RTX,
1499 : 1, false);
1500 : }
1501 :
1502 : /* If OP0 is a register, then handle OFFSET here. */
1503 17499 : rtx op0_piece = op0;
1504 17499 : opt_scalar_int_mode op0_piece_mode = op0_mode;
1505 17499 : if (SUBREG_P (op0) || REG_P (op0))
1506 : {
1507 142 : scalar_int_mode imode;
1508 142 : if (op0_mode.exists (&imode)
1509 142 : && GET_MODE_SIZE (imode) < UNITS_PER_WORD)
1510 : {
1511 0 : if (offset)
1512 0 : op0_piece = const0_rtx;
1513 : }
1514 : else
1515 : {
1516 142 : op0_piece = operand_subword_force (op0,
1517 142 : offset * unit / BITS_PER_WORD,
1518 142 : GET_MODE (op0));
1519 142 : op0_piece_mode = word_mode;
1520 : }
1521 142 : offset &= BITS_PER_WORD / unit - 1;
1522 : }
1523 :
1524 : /* OFFSET is in UNITs, and UNIT is in bits. If WORD is const0_rtx,
1525 : it is just an out-of-bounds access. Ignore it. */
1526 17499 : if (op0_piece != const0_rtx)
1527 17499 : store_fixed_bit_field (op0_piece, op0_piece_mode, thissize,
1528 17499 : offset * unit + thispos, bitregion_start,
1529 : bitregion_end, part, word_mode, reverse);
1530 17499 : bitsdone += thissize;
1531 : }
1532 6640 : }
1533 :
1534 : /* A subroutine of extract_bit_field_1 that converts return value X
1535 : to either MODE or TMODE. MODE, TMODE and UNSIGNEDP are arguments
1536 : to extract_bit_field. */
1537 :
1538 : static rtx
1539 898927 : convert_extracted_bit_field (rtx x, machine_mode mode,
1540 : machine_mode tmode, bool unsignedp)
1541 : {
1542 898927 : if (GET_MODE (x) == tmode || GET_MODE (x) == mode)
1543 : return x;
1544 :
1545 : /* If the x mode is not a scalar integral, first convert to the
1546 : integer mode of that size and then access it as a floating-point
1547 : value via a SUBREG. */
1548 21664 : if (!SCALAR_INT_MODE_P (tmode))
1549 : {
1550 11642 : scalar_int_mode int_mode = int_mode_for_mode (tmode).require ();
1551 11642 : x = convert_to_mode (int_mode, x, unsignedp);
1552 11642 : x = force_reg (int_mode, x);
1553 11642 : return gen_lowpart (tmode, x);
1554 : }
1555 :
1556 10022 : return convert_to_mode (tmode, x, unsignedp);
1557 : }
1558 :
1559 : /* Try to use an ext(z)v pattern to extract a field from OP0.
1560 : Return the extracted value on success, otherwise return null.
1561 : EXTV describes the extraction instruction to use. If OP0_MODE
1562 : is defined, it is the mode of OP0, otherwise OP0 is a BLKmode MEM.
1563 : The other arguments are as for extract_bit_field. */
1564 :
1565 : static rtx
1566 165467 : extract_bit_field_using_extv (const extraction_insn *extv, rtx op0,
1567 : opt_scalar_int_mode op0_mode,
1568 : unsigned HOST_WIDE_INT bitsize,
1569 : unsigned HOST_WIDE_INT bitnum,
1570 : int unsignedp, rtx target,
1571 : machine_mode mode, machine_mode tmode)
1572 : {
1573 165467 : class expand_operand ops[4];
1574 165467 : rtx spec_target = target;
1575 165467 : rtx spec_target_subreg = 0;
1576 165467 : scalar_int_mode ext_mode = extv->field_mode;
1577 165467 : unsigned unit = GET_MODE_BITSIZE (ext_mode);
1578 :
1579 165467 : if (bitsize == 0 || unit < bitsize)
1580 : return NULL_RTX;
1581 :
1582 165467 : if (MEM_P (op0))
1583 : /* Get a reference to the first byte of the field. */
1584 0 : op0 = narrow_bit_field_mem (op0, extv->struct_mode, bitsize, bitnum,
1585 : &bitnum);
1586 : else
1587 : {
1588 : /* Convert from counting within OP0 to counting in EXT_MODE. */
1589 165467 : if (BYTES_BIG_ENDIAN)
1590 : bitnum += unit - GET_MODE_BITSIZE (op0_mode.require ());
1591 :
1592 : /* If op0 is a register, we need it in EXT_MODE to make it
1593 : acceptable to the format of ext(z)v. */
1594 165467 : if (GET_CODE (op0) == SUBREG && op0_mode.require () != ext_mode)
1595 0 : return NULL_RTX;
1596 165467 : if (REG_P (op0) && op0_mode.require () != ext_mode)
1597 49722 : op0 = gen_lowpart_SUBREG (ext_mode, op0);
1598 : }
1599 :
1600 : /* If BITS_BIG_ENDIAN is zero on a BYTES_BIG_ENDIAN machine, we count
1601 : "backwards" from the size of the unit we are extracting from.
1602 : Otherwise, we count bits from the most significant on a
1603 : BYTES/BITS_BIG_ENDIAN machine. */
1604 :
1605 165467 : if (BITS_BIG_ENDIAN != BYTES_BIG_ENDIAN)
1606 : bitnum = unit - bitsize - bitnum;
1607 :
1608 165467 : if (target == 0)
1609 45810 : target = spec_target = gen_reg_rtx (tmode);
1610 :
1611 165467 : if (GET_MODE (target) != ext_mode)
1612 : {
1613 98234 : rtx temp;
1614 : /* Don't use LHS paradoxical subreg if explicit truncation is needed
1615 : between the mode of the extraction (word_mode) and the target
1616 : mode. Instead, create a temporary and use convert_move to set
1617 : the target. */
1618 98234 : if (REG_P (target)
1619 97120 : && TRULY_NOOP_TRUNCATION_MODES_P (GET_MODE (target), ext_mode)
1620 195354 : && (temp = gen_lowpart_if_possible (ext_mode, target)))
1621 : {
1622 96610 : target = temp;
1623 96610 : if (partial_subreg_p (GET_MODE (spec_target), ext_mode))
1624 95752 : spec_target_subreg = target;
1625 : }
1626 : else
1627 1624 : target = gen_reg_rtx (ext_mode);
1628 : }
1629 :
1630 165467 : create_output_operand (&ops[0], target, ext_mode);
1631 165467 : create_fixed_operand (&ops[1], op0);
1632 165467 : create_integer_operand (&ops[2], bitsize);
1633 165467 : create_integer_operand (&ops[3], bitnum);
1634 165467 : if (maybe_expand_insn (extv->icode, 4, ops))
1635 : {
1636 1713 : target = ops[0].value;
1637 1713 : if (target == spec_target)
1638 : return target;
1639 1713 : if (target == spec_target_subreg)
1640 : return spec_target;
1641 56 : return convert_extracted_bit_field (target, mode, tmode, unsignedp);
1642 : }
1643 : return NULL_RTX;
1644 : }
1645 :
1646 : /* See whether it would be valid to extract the part of OP0 with
1647 : mode OP0_MODE described by BITNUM and BITSIZE into a value of
1648 : mode MODE using a subreg operation.
1649 : Return the subreg if so, otherwise return null. */
1650 :
1651 : static rtx
1652 854733 : extract_bit_field_as_subreg (machine_mode mode, rtx op0,
1653 : machine_mode op0_mode,
1654 : poly_uint64 bitsize, poly_uint64 bitnum)
1655 : {
1656 854733 : poly_uint64 bytenum;
1657 854733 : if (multiple_p (bitnum, BITS_PER_UNIT, &bytenum)
1658 816213 : && known_eq (bitsize, GET_MODE_BITSIZE (mode))
1659 854733 : && lowpart_bit_field_p (bitnum, bitsize, op0_mode)
1660 1670946 : && TRULY_NOOP_TRUNCATION_MODES_P (mode, op0_mode))
1661 680510 : return force_subreg (mode, op0, op0_mode, bytenum);
1662 : return NULL_RTX;
1663 : }
1664 :
1665 : /* A subroutine of extract_bit_field, with the same arguments.
1666 : If UNSIGNEDP is -1, the result need not be sign or zero extended.
1667 : If FALLBACK_P is true, fall back to extract_fixed_bit_field
1668 : if we can find no other means of implementing the operation.
1669 : if FALLBACK_P is false, return NULL instead. */
1670 :
1671 : static rtx
1672 1146207 : extract_bit_field_1 (rtx str_rtx, poly_uint64 bitsize, poly_uint64 bitnum,
1673 : int unsignedp, rtx target, machine_mode mode,
1674 : machine_mode tmode, bool reverse, bool fallback_p,
1675 : rtx *alt_rtl)
1676 : {
1677 1146207 : rtx op0 = str_rtx;
1678 1146207 : machine_mode mode1;
1679 :
1680 1146207 : if (tmode == VOIDmode)
1681 0 : tmode = mode;
1682 :
1683 1156677 : while (GET_CODE (op0) == SUBREG)
1684 : {
1685 10470 : bitnum += SUBREG_BYTE (op0) * BITS_PER_UNIT;
1686 10470 : op0 = SUBREG_REG (op0);
1687 : }
1688 :
1689 : /* If we have an out-of-bounds access to a register, just return an
1690 : uninitialized register of the required mode. This can occur if the
1691 : source code contains an out-of-bounds access to a small array. */
1692 2110185 : if (REG_P (op0) && known_ge (bitnum, GET_MODE_BITSIZE (GET_MODE (op0))))
1693 0 : return gen_reg_rtx (tmode);
1694 :
1695 1146207 : if (REG_P (op0)
1696 963978 : && mode == GET_MODE (op0)
1697 149820 : && known_eq (bitnum, 0U)
1698 1383873 : && known_eq (bitsize, GET_MODE_BITSIZE (GET_MODE (op0))))
1699 : {
1700 18993 : if (reverse)
1701 0 : op0 = flip_storage_order (mode, op0);
1702 : /* We're trying to extract a full register from itself. */
1703 18993 : return op0;
1704 : }
1705 :
1706 : /* First try to check for vector from vector extractions. */
1707 1054810 : if (VECTOR_MODE_P (GET_MODE (op0))
1708 93611 : && !MEM_P (op0)
1709 92189 : && VECTOR_MODE_P (tmode)
1710 13129 : && known_eq (bitsize, GET_MODE_PRECISION (tmode))
1711 2280686 : && maybe_gt (GET_MODE_SIZE (GET_MODE (op0)), GET_MODE_SIZE (tmode)))
1712 : {
1713 13129 : machine_mode new_mode = GET_MODE (op0);
1714 39387 : if (GET_MODE_INNER (new_mode) != GET_MODE_INNER (tmode))
1715 : {
1716 214 : scalar_mode inner_mode = GET_MODE_INNER (tmode);
1717 214 : poly_uint64 nunits;
1718 428 : if (!multiple_p (GET_MODE_BITSIZE (GET_MODE (op0)),
1719 214 : GET_MODE_UNIT_BITSIZE (tmode), &nunits)
1720 428 : || !related_vector_mode (tmode, inner_mode,
1721 214 : nunits).exists (&new_mode)
1722 412 : || maybe_ne (GET_MODE_SIZE (new_mode),
1723 594 : GET_MODE_SIZE (GET_MODE (op0))))
1724 16 : new_mode = VOIDmode;
1725 : }
1726 13129 : poly_uint64 pos;
1727 13129 : if (new_mode != VOIDmode
1728 13113 : && (convert_optab_handler (vec_extract_optab, new_mode, tmode)
1729 : != CODE_FOR_nothing)
1730 26242 : && multiple_p (bitnum, GET_MODE_BITSIZE (tmode), &pos))
1731 : {
1732 9005 : class expand_operand ops[3];
1733 9005 : machine_mode outermode = new_mode;
1734 9005 : machine_mode innermode = tmode;
1735 9005 : enum insn_code icode
1736 9005 : = convert_optab_handler (vec_extract_optab, outermode, innermode);
1737 :
1738 9005 : if (new_mode != GET_MODE (op0))
1739 33 : op0 = gen_lowpart (new_mode, op0);
1740 9005 : create_output_operand (&ops[0], target, innermode);
1741 9005 : ops[0].target = 1;
1742 9005 : create_input_operand (&ops[1], op0, outermode);
1743 9005 : create_integer_operand (&ops[2], pos);
1744 9005 : if (maybe_expand_insn (icode, 3, ops))
1745 : {
1746 9005 : if (alt_rtl && ops[0].target)
1747 205 : *alt_rtl = target;
1748 9005 : target = ops[0].value;
1749 9005 : if (GET_MODE (target) != mode)
1750 9005 : return gen_lowpart (tmode, target);
1751 : return target;
1752 : }
1753 : }
1754 : }
1755 :
1756 : /* See if we can get a better vector mode before extracting. */
1757 1047413 : if (VECTOR_MODE_P (GET_MODE (op0))
1758 84606 : && !MEM_P (op0)
1759 1284577 : && GET_MODE_INNER (GET_MODE (op0)) != tmode)
1760 : {
1761 9796 : machine_mode new_mode;
1762 :
1763 9796 : if (GET_MODE_CLASS (tmode) == MODE_FLOAT)
1764 557 : new_mode = MIN_MODE_VECTOR_FLOAT;
1765 : else if (GET_MODE_CLASS (tmode) == MODE_FRACT)
1766 0 : new_mode = MIN_MODE_VECTOR_FRACT;
1767 : else if (GET_MODE_CLASS (tmode) == MODE_UFRACT)
1768 0 : new_mode = MIN_MODE_VECTOR_UFRACT;
1769 : else if (GET_MODE_CLASS (tmode) == MODE_ACCUM)
1770 0 : new_mode = MIN_MODE_VECTOR_ACCUM;
1771 : else if (GET_MODE_CLASS (tmode) == MODE_UACCUM)
1772 0 : new_mode = MIN_MODE_VECTOR_UACCUM;
1773 : else
1774 9239 : new_mode = MIN_MODE_VECTOR_INT;
1775 :
1776 154916 : FOR_EACH_MODE_FROM (new_mode, new_mode)
1777 309116 : if (known_eq (GET_MODE_SIZE (new_mode), GET_MODE_SIZE (GET_MODE (op0)))
1778 64892 : && known_eq (GET_MODE_UNIT_SIZE (new_mode), GET_MODE_SIZE (tmode))
1779 174150 : && known_eq (bitsize, GET_MODE_UNIT_PRECISION (new_mode))
1780 19592 : && multiple_p (bitnum, GET_MODE_UNIT_PRECISION (new_mode))
1781 9759 : && targetm.vector_mode_supported_p (new_mode)
1782 164299 : && targetm.modes_tieable_p (GET_MODE (op0), new_mode))
1783 : break;
1784 9796 : if (new_mode != VOIDmode)
1785 9438 : op0 = gen_lowpart (new_mode, op0);
1786 : }
1787 :
1788 : /* Use vec_extract patterns for extracting parts of vectors whenever
1789 : available. If that fails, see whether the current modes and bitregion
1790 : give a natural subreg. */
1791 1118209 : machine_mode outermode = GET_MODE (op0);
1792 1118209 : if (VECTOR_MODE_P (outermode) && !MEM_P (op0))
1793 : {
1794 83184 : scalar_mode innermode = GET_MODE_INNER (outermode);
1795 :
1796 83184 : enum insn_code icode
1797 83184 : = convert_optab_handler (vec_extract_optab, outermode, innermode);
1798 :
1799 83184 : poly_uint64 pos;
1800 83184 : if (icode != CODE_FOR_nothing
1801 83187 : && known_eq (bitsize, GET_MODE_PRECISION (innermode))
1802 164425 : && multiple_p (bitnum, GET_MODE_PRECISION (innermode), &pos))
1803 : {
1804 81238 : class expand_operand ops[3];
1805 :
1806 81238 : create_output_operand (&ops[0], target,
1807 81238 : insn_data[icode].operand[0].mode);
1808 81238 : ops[0].target = 1;
1809 81238 : create_input_operand (&ops[1], op0, outermode);
1810 81238 : create_integer_operand (&ops[2], pos);
1811 81238 : if (maybe_expand_insn (icode, 3, ops))
1812 : {
1813 81238 : if (alt_rtl && ops[0].target)
1814 19341 : *alt_rtl = target;
1815 81238 : target = ops[0].value;
1816 81238 : if (GET_MODE (target) != mode)
1817 81238 : return gen_lowpart (tmode, target);
1818 : return target;
1819 : }
1820 : }
1821 : /* Using subregs is useful if we're extracting one register vector
1822 : from a multi-register vector. extract_bit_field_as_subreg checks
1823 : for valid bitsize and bitnum, so we don't need to do that here. */
1824 1946 : if (VECTOR_MODE_P (mode))
1825 : {
1826 47 : rtx sub = extract_bit_field_as_subreg (mode, op0, outermode,
1827 : bitsize, bitnum);
1828 47 : if (sub)
1829 : return sub;
1830 : }
1831 : }
1832 :
1833 : /* Make sure we are playing with integral modes. Pun with subregs
1834 : if we aren't. */
1835 1036954 : opt_scalar_int_mode op0_mode = int_mode_for_mode (GET_MODE (op0));
1836 1036954 : scalar_int_mode imode;
1837 1036954 : if (!op0_mode.exists (&imode) || imode != GET_MODE (op0))
1838 : {
1839 195092 : if (MEM_P (op0))
1840 156855 : op0 = adjust_bitfield_address_size (op0, op0_mode.else_blk (),
1841 : 0, MEM_SIZE (op0));
1842 38237 : else if (op0_mode.exists (&imode))
1843 : {
1844 38206 : op0 = gen_lowpart (imode, op0);
1845 :
1846 : /* If we got a SUBREG, force it into a register since we
1847 : aren't going to be able to do another SUBREG on it. */
1848 38206 : if (GET_CODE (op0) == SUBREG)
1849 37929 : op0 = force_reg (imode, op0);
1850 : }
1851 : else
1852 : {
1853 62 : poly_int64 size = GET_MODE_SIZE (GET_MODE (op0));
1854 31 : rtx mem = assign_stack_temp (GET_MODE (op0), size);
1855 31 : emit_move_insn (mem, op0);
1856 31 : op0 = adjust_bitfield_address_size (mem, BLKmode, 0, size);
1857 : }
1858 : }
1859 :
1860 : /* ??? We currently assume TARGET is at least as big as BITSIZE.
1861 : If that's wrong, the solution is to test for it and set TARGET to 0
1862 : if needed. */
1863 :
1864 : /* Get the mode of the field to use for atomic access or subreg
1865 : conversion. */
1866 1036954 : if (!SCALAR_INT_MODE_P (tmode)
1867 1036954 : || !mode_for_size (bitsize, GET_MODE_CLASS (tmode), 0).exists (&mode1))
1868 310389 : mode1 = mode;
1869 1036954 : gcc_assert (mode1 != BLKmode);
1870 :
1871 : /* Extraction of a full MODE1 value can be done with a subreg as long
1872 : as the least significant bit of the value is the least significant
1873 : bit of either OP0 or a word of OP0. */
1874 1036954 : if (!MEM_P (op0) && !reverse && op0_mode.exists (&imode))
1875 : {
1876 854686 : rtx sub = extract_bit_field_as_subreg (mode1, op0, imode,
1877 : bitsize, bitnum);
1878 854686 : if (sub)
1879 674272 : return convert_extracted_bit_field (sub, mode, tmode, unsignedp);
1880 : }
1881 :
1882 : /* Extraction of a full MODE1 value can be done with a load as long as
1883 : the field is on a byte boundary and is sufficiently aligned. */
1884 362682 : poly_uint64 bytenum;
1885 362682 : if (simple_mem_bitfield_p (op0, bitsize, bitnum, mode1, &bytenum))
1886 : {
1887 43031 : op0 = adjust_bitfield_address (op0, mode1, bytenum);
1888 43031 : if (reverse)
1889 51 : op0 = flip_storage_order (mode1, op0);
1890 43031 : return convert_extracted_bit_field (op0, mode, tmode, unsignedp);
1891 : }
1892 :
1893 : /* If we have a memory source and a non-constant bit offset, restrict
1894 : the memory to the referenced bytes. This is a worst-case fallback
1895 : but is useful for things like vector booleans. */
1896 319651 : if (MEM_P (op0) && !bitnum.is_constant ())
1897 : {
1898 : bytenum = bits_to_bytes_round_down (bitnum);
1899 : bitnum = num_trailing_bits (bitnum);
1900 : poly_uint64 bytesize = bits_to_bytes_round_up (bitnum + bitsize);
1901 : op0 = adjust_bitfield_address_size (op0, BLKmode, bytenum, bytesize);
1902 : op0_mode = opt_scalar_int_mode ();
1903 : }
1904 :
1905 : /* It's possible we'll need to handle other cases here for
1906 : polynomial bitnum and bitsize. */
1907 :
1908 : /* From here on we need to be looking at a fixed-size insertion. */
1909 319651 : return extract_integral_bit_field (op0, op0_mode, bitsize.to_constant (),
1910 : bitnum.to_constant (), unsignedp,
1911 319651 : target, mode, tmode, reverse, fallback_p);
1912 : }
1913 :
1914 : /* Subroutine of extract_bit_field_1, with the same arguments, except
1915 : that BITSIZE and BITNUM are constant. Handle cases specific to
1916 : integral modes. If OP0_MODE is defined, it is the mode of OP0,
1917 : otherwise OP0 is a BLKmode MEM. */
1918 :
1919 : static rtx
1920 319651 : extract_integral_bit_field (rtx op0, opt_scalar_int_mode op0_mode,
1921 : unsigned HOST_WIDE_INT bitsize,
1922 : unsigned HOST_WIDE_INT bitnum, int unsignedp,
1923 : rtx target, machine_mode mode, machine_mode tmode,
1924 : bool reverse, bool fallback_p)
1925 : {
1926 : /* Handle fields bigger than a word. */
1927 :
1928 322917 : if (bitsize > BITS_PER_WORD)
1929 : {
1930 : /* Here we transfer the words of the field
1931 : in the order least significant first.
1932 : This is because the most significant word is the one which may
1933 : be less than full. */
1934 :
1935 1516 : const bool backwards = WORDS_BIG_ENDIAN;
1936 1516 : unsigned int nwords = (bitsize + (BITS_PER_WORD - 1)) / BITS_PER_WORD;
1937 1516 : unsigned int i;
1938 1516 : rtx_insn *last;
1939 :
1940 1516 : if (target == 0 || !REG_P (target) || !valid_multiword_target_p (target))
1941 1492 : target = gen_reg_rtx (mode);
1942 :
1943 : /* In case we're about to clobber a base register or something
1944 : (see gcc.c-torture/execute/20040625-1.c). */
1945 1516 : if (reg_mentioned_p (target, op0))
1946 0 : target = gen_reg_rtx (mode);
1947 :
1948 : /* Indicate for flow that the entire target reg is being set. */
1949 1516 : emit_clobber (target);
1950 :
1951 : /* The mode must be fixed-size, since extract_bit_field_1 handles
1952 : extractions from variable-sized objects before calling this
1953 : function. */
1954 1516 : unsigned int target_size
1955 1516 : = GET_MODE_SIZE (GET_MODE (target)).to_constant ();
1956 1516 : last = get_last_insn ();
1957 4548 : for (i = 0; i < nwords; i++)
1958 : {
1959 : /* If I is 0, use the low-order word in both field and target;
1960 : if I is 1, use the next to lowest word; and so on. */
1961 : /* Word number in TARGET to use. */
1962 3032 : unsigned int wordnum
1963 : = (backwards ? target_size / UNITS_PER_WORD - i - 1 : i);
1964 : /* Offset from start of field in OP0. */
1965 6064 : unsigned int bit_offset = (backwards ^ reverse
1966 3032 : ? MAX ((int) bitsize - ((int) i + 1)
1967 : * BITS_PER_WORD,
1968 : 0)
1969 3100 : : (int) i * BITS_PER_WORD);
1970 3032 : rtx target_part = operand_subword (target, wordnum, 1, VOIDmode);
1971 3032 : rtx result_part
1972 3208 : = extract_bit_field_1 (op0, MIN (BITS_PER_WORD,
1973 : bitsize - i * BITS_PER_WORD),
1974 3032 : bitnum + bit_offset,
1975 : (unsignedp ? 1 : -1), target_part,
1976 : mode, word_mode, reverse, fallback_p, NULL);
1977 :
1978 3032 : gcc_assert (target_part);
1979 3032 : if (!result_part)
1980 : {
1981 0 : delete_insns_since (last);
1982 0 : return NULL;
1983 : }
1984 :
1985 3032 : if (result_part != target_part)
1986 2916 : emit_move_insn (target_part, result_part);
1987 : }
1988 :
1989 1516 : if (unsignedp)
1990 : {
1991 : /* Unless we've filled TARGET, the upper regs in a multi-reg value
1992 : need to be zero'd out. */
1993 1530 : if (target_size > nwords * UNITS_PER_WORD)
1994 : {
1995 0 : unsigned int i, total_words;
1996 :
1997 0 : total_words = target_size / UNITS_PER_WORD;
1998 0 : for (i = nwords; i < total_words; i++)
1999 0 : emit_move_insn
2000 0 : (operand_subword (target,
2001 0 : backwards ? total_words - i - 1 : i,
2002 : 1, VOIDmode),
2003 : const0_rtx);
2004 : }
2005 1496 : return target;
2006 : }
2007 :
2008 : /* Signed bit field: sign-extend with two arithmetic shifts. */
2009 40 : target = expand_shift (LSHIFT_EXPR, mode, target,
2010 20 : GET_MODE_BITSIZE (mode) - bitsize, NULL_RTX, 0);
2011 40 : return expand_shift (RSHIFT_EXPR, mode, target,
2012 20 : GET_MODE_BITSIZE (mode) - bitsize, NULL_RTX, 0);
2013 : }
2014 :
2015 : /* If OP0 is a multi-word register, narrow it to the affected word.
2016 : If the region spans two words, defer to extract_split_bit_field. */
2017 501083 : if (!MEM_P (op0) && GET_MODE_SIZE (op0_mode.require ()) > UNITS_PER_WORD)
2018 : {
2019 3908 : if (bitnum % BITS_PER_WORD + bitsize > BITS_PER_WORD)
2020 : {
2021 1146 : if (!fallback_p)
2022 : return NULL_RTX;
2023 64 : target = extract_split_bit_field (op0, op0_mode, bitsize, bitnum,
2024 : unsignedp, reverse);
2025 64 : return convert_extracted_bit_field (target, mode, tmode, unsignedp);
2026 : }
2027 : /* If OP0 is a hard register, copy it to a pseudo before calling
2028 : force_subreg. */
2029 2762 : if (REG_P (op0) && HARD_REGISTER_P (op0))
2030 1 : op0 = copy_to_reg (op0);
2031 2762 : op0 = force_subreg (word_mode, op0, op0_mode.require (),
2032 3226 : bitnum / BITS_PER_WORD * UNITS_PER_WORD);
2033 2762 : op0_mode = word_mode;
2034 2994 : bitnum %= BITS_PER_WORD;
2035 : }
2036 :
2037 : /* From here on we know the desired field is smaller than a word.
2038 : If OP0 is a register, it too fits within a word. */
2039 316989 : enum extraction_pattern pattern = unsignedp ? EP_extzv : EP_extv;
2040 316989 : extraction_insn extv;
2041 316989 : if (!MEM_P (op0)
2042 179267 : && !reverse
2043 : /* ??? We could limit the structure size to the part of OP0 that
2044 : contains the field, with appropriate checks for endianness
2045 : and TARGET_TRULY_NOOP_TRUNCATION. */
2046 496248 : && get_best_reg_extraction_insn (&extv, pattern,
2047 510040 : GET_MODE_BITSIZE (op0_mode.require ()),
2048 : tmode))
2049 : {
2050 165467 : rtx result = extract_bit_field_using_extv (&extv, op0, op0_mode,
2051 : bitsize, bitnum,
2052 : unsignedp, target, mode,
2053 : tmode);
2054 165467 : if (result)
2055 : return result;
2056 : }
2057 :
2058 : /* If OP0 is a memory, try copying it to a register and seeing if a
2059 : cheap register alternative is available. */
2060 315276 : if (MEM_P (op0) & !reverse)
2061 : {
2062 137520 : if (get_best_mem_extraction_insn (&extv, pattern, bitsize, bitnum,
2063 : tmode))
2064 : {
2065 0 : rtx result = extract_bit_field_using_extv (&extv, op0, op0_mode,
2066 : bitsize, bitnum,
2067 : unsignedp, target, mode,
2068 : tmode);
2069 0 : if (result)
2070 0 : return result;
2071 : }
2072 :
2073 137520 : rtx_insn *last = get_last_insn ();
2074 :
2075 : /* Try loading part of OP0 into a register and extracting the
2076 : bitfield from that. */
2077 137520 : unsigned HOST_WIDE_INT bitpos;
2078 137520 : rtx xop0 = adjust_bit_field_mem_for_reg (pattern, op0, bitsize, bitnum,
2079 : 0, 0, tmode, &bitpos);
2080 137520 : if (xop0)
2081 : {
2082 134854 : xop0 = copy_to_reg (xop0);
2083 134854 : rtx result = extract_bit_field_1 (xop0, bitsize, bitpos,
2084 : unsignedp, target,
2085 : mode, tmode, reverse, false, NULL);
2086 134854 : if (result)
2087 : return result;
2088 134854 : delete_insns_since (last);
2089 : }
2090 : }
2091 :
2092 315276 : if (!fallback_p)
2093 : return NULL;
2094 :
2095 : /* Find a correspondingly-sized integer field, so we can apply
2096 : shifts and masks to it. */
2097 181504 : scalar_int_mode int_mode;
2098 181504 : if (!int_mode_for_mode (tmode).exists (&int_mode))
2099 : /* If this fails, we should probably push op0 out to memory and then
2100 : do a load. */
2101 0 : int_mode = int_mode_for_mode (mode).require ();
2102 :
2103 181504 : target = extract_fixed_bit_field (int_mode, op0, op0_mode, bitsize,
2104 : bitnum, target, unsignedp, reverse);
2105 :
2106 : /* Complex values must be reversed piecewise, so we need to undo the global
2107 : reversal, convert to the complex mode and reverse again. */
2108 181504 : if (reverse && COMPLEX_MODE_P (tmode))
2109 : {
2110 0 : target = flip_storage_order (int_mode, target);
2111 0 : target = convert_extracted_bit_field (target, mode, tmode, unsignedp);
2112 0 : target = flip_storage_order (tmode, target);
2113 : }
2114 : else
2115 181504 : target = convert_extracted_bit_field (target, mode, tmode, unsignedp);
2116 :
2117 : return target;
2118 : }
2119 :
2120 : /* Generate code to extract a byte-field from STR_RTX
2121 : containing BITSIZE bits, starting at BITNUM,
2122 : and put it in TARGET if possible (if TARGET is nonzero).
2123 : Regardless of TARGET, we return the rtx for where the value is placed.
2124 :
2125 : STR_RTX is the structure containing the byte (a REG or MEM).
2126 : UNSIGNEDP is nonzero if this is an unsigned bit field.
2127 : MODE is the natural mode of the field value once extracted.
2128 : TMODE is the mode the caller would like the value to have;
2129 : but the value may be returned with type MODE instead.
2130 :
2131 : If REVERSE is true, the extraction is to be done in reverse order.
2132 :
2133 : If a TARGET is specified and we can store in it at no extra cost,
2134 : we do so, and return TARGET.
2135 : Otherwise, we return a REG of mode TMODE or MODE, with TMODE preferred
2136 : if they are equally easy.
2137 :
2138 : If the result can be stored at TARGET, and ALT_RTL is non-NULL,
2139 : then *ALT_RTL is set to TARGET (before legitimziation). */
2140 :
2141 : rtx
2142 1008321 : extract_bit_field (rtx str_rtx, poly_uint64 bitsize, poly_uint64 bitnum,
2143 : int unsignedp, rtx target, machine_mode mode,
2144 : machine_mode tmode, bool reverse, rtx *alt_rtl)
2145 : {
2146 1008321 : machine_mode mode1;
2147 :
2148 : /* Handle -fstrict-volatile-bitfields in the cases where it applies. */
2149 2016642 : if (maybe_ne (GET_MODE_BITSIZE (GET_MODE (str_rtx)), 0))
2150 : mode1 = GET_MODE (str_rtx);
2151 256426 : else if (target && maybe_ne (GET_MODE_BITSIZE (GET_MODE (target)), 0))
2152 : mode1 = GET_MODE (target);
2153 : else
2154 : mode1 = tmode;
2155 :
2156 1008321 : unsigned HOST_WIDE_INT ibitsize, ibitnum;
2157 1008321 : scalar_int_mode int_mode;
2158 1008321 : if (bitsize.is_constant (&ibitsize)
2159 1008321 : && bitnum.is_constant (&ibitnum)
2160 1859460 : && is_a <scalar_int_mode> (mode1, &int_mode)
2161 851146 : && strict_volatile_bitfield_p (str_rtx, ibitsize, ibitnum,
2162 : int_mode, 0, 0))
2163 : {
2164 : /* Extraction of a full INT_MODE value can be done with a simple load.
2165 : We know here that the field can be accessed with one single
2166 : instruction. For targets that support unaligned memory,
2167 : an unaligned access may be necessary. */
2168 14 : if (ibitsize == GET_MODE_BITSIZE (int_mode))
2169 : {
2170 0 : rtx result = adjust_bitfield_address (str_rtx, int_mode,
2171 : ibitnum / BITS_PER_UNIT);
2172 0 : if (reverse)
2173 0 : result = flip_storage_order (int_mode, result);
2174 0 : gcc_assert (ibitnum % BITS_PER_UNIT == 0);
2175 0 : return convert_extracted_bit_field (result, mode, tmode, unsignedp);
2176 : }
2177 :
2178 7 : str_rtx = narrow_bit_field_mem (str_rtx, int_mode, ibitsize, ibitnum,
2179 : &ibitnum);
2180 14 : gcc_assert (ibitnum + ibitsize <= GET_MODE_BITSIZE (int_mode));
2181 7 : str_rtx = copy_to_reg (str_rtx);
2182 7 : return extract_bit_field_1 (str_rtx, ibitsize, ibitnum, unsignedp,
2183 : target, mode, tmode, reverse, true, alt_rtl);
2184 : }
2185 :
2186 1008314 : return extract_bit_field_1 (str_rtx, bitsize, bitnum, unsignedp,
2187 1008314 : target, mode, tmode, reverse, true, alt_rtl);
2188 : }
2189 :
2190 : /* Use shifts and boolean operations to extract a field of BITSIZE bits
2191 : from bit BITNUM of OP0. If OP0_MODE is defined, it is the mode of OP0,
2192 : otherwise OP0 is a BLKmode MEM.
2193 :
2194 : UNSIGNEDP is nonzero for an unsigned bit field (don't sign-extend value).
2195 : If REVERSE is true, the extraction is to be done in reverse order.
2196 :
2197 : If TARGET is nonzero, attempts to store the value there
2198 : and return TARGET, but this is not guaranteed.
2199 : If TARGET is not used, create a pseudo-reg of mode TMODE for the value. */
2200 :
2201 : static rtx
2202 199859 : extract_fixed_bit_field (machine_mode tmode, rtx op0,
2203 : opt_scalar_int_mode op0_mode,
2204 : unsigned HOST_WIDE_INT bitsize,
2205 : unsigned HOST_WIDE_INT bitnum, rtx target,
2206 : int unsignedp, bool reverse)
2207 : {
2208 199859 : scalar_int_mode mode;
2209 199859 : if (MEM_P (op0))
2210 : {
2211 151449 : if (!get_best_mode (bitsize, bitnum, 0, 0, MEM_ALIGN (op0),
2212 151449 : BITS_PER_WORD, MEM_VOLATILE_P (op0), &mode))
2213 : /* The only way this should occur is if the field spans word
2214 : boundaries. */
2215 4302 : return extract_split_bit_field (op0, op0_mode, bitsize, bitnum,
2216 4302 : unsignedp, reverse);
2217 :
2218 147147 : op0 = narrow_bit_field_mem (op0, mode, bitsize, bitnum, &bitnum);
2219 : }
2220 : else
2221 48410 : mode = op0_mode.require ();
2222 :
2223 195557 : return extract_fixed_bit_field_1 (tmode, op0, mode, bitsize, bitnum,
2224 195557 : target, unsignedp, reverse);
2225 : }
2226 :
2227 : /* Helper function for extract_fixed_bit_field, extracts
2228 : the bit field always using MODE, which is the mode of OP0.
2229 : If UNSIGNEDP is -1, the result need not be sign or zero extended.
2230 : The other arguments are as for extract_fixed_bit_field. */
2231 :
2232 : static rtx
2233 195557 : extract_fixed_bit_field_1 (machine_mode tmode, rtx op0, scalar_int_mode mode,
2234 : unsigned HOST_WIDE_INT bitsize,
2235 : unsigned HOST_WIDE_INT bitnum, rtx target,
2236 : int unsignedp, bool reverse)
2237 : {
2238 : /* Note that bitsize + bitnum can be greater than GET_MODE_BITSIZE (mode)
2239 : for invalid input, such as extract equivalent of f5 from
2240 : gcc.dg/pr48335-2.c. */
2241 :
2242 195557 : if (reverse ? !BYTES_BIG_ENDIAN : BYTES_BIG_ENDIAN)
2243 : /* BITNUM is the distance between our msb and that of OP0.
2244 : Convert it to the distance from the lsb. */
2245 424 : bitnum = GET_MODE_BITSIZE (mode) - bitsize - bitnum;
2246 :
2247 : /* Now BITNUM is always the distance between the field's lsb and that of OP0.
2248 : We have reduced the big-endian case to the little-endian case. */
2249 195557 : if (reverse)
2250 212 : op0 = flip_storage_order (mode, op0);
2251 :
2252 195557 : if (unsignedp)
2253 : {
2254 136177 : if (bitnum)
2255 : {
2256 : /* If the field does not already start at the lsb,
2257 : shift it so it does. */
2258 : /* Maybe propagate the target for the shift. */
2259 61159 : rtx subtarget = (target != 0 && REG_P (target) ? target : 0);
2260 61159 : if (tmode != mode)
2261 40731 : subtarget = 0;
2262 61159 : op0 = expand_shift (RSHIFT_EXPR, mode, op0, bitnum, subtarget, 1);
2263 : }
2264 : /* Convert the value to the desired mode. TMODE must also be a
2265 : scalar integer for this conversion to make sense, since we
2266 : shouldn't reinterpret the bits. */
2267 136177 : scalar_int_mode new_mode = as_a <scalar_int_mode> (tmode);
2268 136177 : if (mode != new_mode)
2269 60072 : op0 = convert_to_mode (new_mode, op0, 1);
2270 :
2271 : /* Unless the msb of the field used to be the msb when we shifted,
2272 : mask out the upper bits. */
2273 :
2274 136177 : if (GET_MODE_BITSIZE (mode) != bitnum + bitsize
2275 136177 : && unsignedp != -1)
2276 92887 : return expand_binop (new_mode, and_optab, op0,
2277 : mask_rtx (new_mode, 0, bitsize, 0),
2278 92887 : target, 1, OPTAB_LIB_WIDEN);
2279 : return op0;
2280 : }
2281 :
2282 : /* To extract a signed bit-field, first shift its msb to the msb of the word,
2283 : then arithmetic-shift its lsb to the lsb of the word. */
2284 59380 : op0 = force_reg (mode, op0);
2285 :
2286 : /* Find the narrowest integer mode that contains the field. */
2287 :
2288 59380 : opt_scalar_int_mode mode_iter;
2289 147325 : FOR_EACH_MODE_IN_CLASS (mode_iter, MODE_INT)
2290 294650 : if (GET_MODE_BITSIZE (mode_iter.require ()) >= bitsize + bitnum)
2291 : break;
2292 :
2293 59380 : mode = mode_iter.require ();
2294 59380 : op0 = convert_to_mode (mode, op0, 0);
2295 :
2296 59380 : if (mode != tmode)
2297 4823 : target = 0;
2298 :
2299 118760 : if (GET_MODE_BITSIZE (mode) != (bitsize + bitnum))
2300 : {
2301 54413 : int amount = GET_MODE_BITSIZE (mode) - (bitsize + bitnum);
2302 : /* Maybe propagate the target for the shift. */
2303 54413 : rtx subtarget = (target != 0 && REG_P (target) ? target : 0);
2304 54413 : op0 = expand_shift (LSHIFT_EXPR, mode, op0, amount, subtarget, 1);
2305 : }
2306 :
2307 118760 : return expand_shift (RSHIFT_EXPR, mode, op0,
2308 59380 : GET_MODE_BITSIZE (mode) - bitsize, target, 0);
2309 : }
2310 :
2311 : /* Return a constant integer (CONST_INT or CONST_DOUBLE) rtx with the value
2312 : VALUE << BITPOS. */
2313 :
2314 : static rtx
2315 74657 : lshift_value (machine_mode mode, unsigned HOST_WIDE_INT value,
2316 : int bitpos)
2317 : {
2318 74657 : return immed_wide_int_const (wi::lshift (value, bitpos), mode);
2319 : }
2320 :
2321 : /* Extract a bit field that is split across two words
2322 : and return an RTX for the result.
2323 :
2324 : OP0 is the REG, SUBREG or MEM rtx for the first of the two words.
2325 : BITSIZE is the field width; BITPOS, position of its first bit, in the word.
2326 : UNSIGNEDP is 1 if should zero-extend the contents; else sign-extend.
2327 : If OP0_MODE is defined, it is the mode of OP0, otherwise OP0 is
2328 : a BLKmode MEM.
2329 :
2330 : If REVERSE is true, the extraction is to be done in reverse order. */
2331 :
2332 : static rtx
2333 4366 : extract_split_bit_field (rtx op0, opt_scalar_int_mode op0_mode,
2334 : unsigned HOST_WIDE_INT bitsize,
2335 : unsigned HOST_WIDE_INT bitpos, int unsignedp,
2336 : bool reverse)
2337 : {
2338 4366 : unsigned int unit;
2339 4366 : unsigned int bitsdone = 0;
2340 4366 : rtx result = NULL_RTX;
2341 4366 : int first = 1;
2342 :
2343 : /* Make sure UNIT isn't larger than BITS_PER_WORD, we can only handle that
2344 : much at a time. */
2345 4366 : if (REG_P (op0) || GET_CODE (op0) == SUBREG)
2346 64 : unit = BITS_PER_WORD;
2347 : else
2348 6385 : unit = MIN (MEM_ALIGN (op0), BITS_PER_WORD);
2349 :
2350 18197 : while (bitsdone < bitsize)
2351 : {
2352 13831 : unsigned HOST_WIDE_INT thissize;
2353 13831 : rtx part;
2354 13831 : unsigned HOST_WIDE_INT thispos;
2355 13831 : unsigned HOST_WIDE_INT offset;
2356 :
2357 13831 : offset = (bitpos + bitsdone) / unit;
2358 13831 : thispos = (bitpos + bitsdone) % unit;
2359 :
2360 : /* THISSIZE must not overrun a word boundary. Otherwise,
2361 : extract_fixed_bit_field will call us again, and we will mutually
2362 : recurse forever. */
2363 13831 : thissize = MIN (bitsize - bitsdone, BITS_PER_WORD);
2364 13831 : thissize = MIN (thissize, unit - thispos);
2365 :
2366 : /* If OP0 is a register, then handle OFFSET here. */
2367 13831 : rtx op0_piece = op0;
2368 13831 : opt_scalar_int_mode op0_piece_mode = op0_mode;
2369 13831 : if (SUBREG_P (op0) || REG_P (op0))
2370 : {
2371 128 : op0_piece = operand_subword_force (op0, offset, op0_mode.require ());
2372 128 : op0_piece_mode = word_mode;
2373 128 : offset = 0;
2374 : }
2375 :
2376 : /* Extract the parts in bit-counting order,
2377 : whose meaning is determined by BYTES_PER_UNIT.
2378 : OFFSET is in UNITs, and UNIT is in bits. */
2379 27662 : part = extract_fixed_bit_field (word_mode, op0_piece, op0_piece_mode,
2380 13831 : thissize, offset * unit + thispos,
2381 : 0, 1, reverse);
2382 13831 : bitsdone += thissize;
2383 :
2384 : /* Shift this part into place for the result. */
2385 13831 : if (reverse ? !BYTES_BIG_ENDIAN : BYTES_BIG_ENDIAN)
2386 : {
2387 4 : if (bitsize != bitsdone)
2388 2 : part = expand_shift (LSHIFT_EXPR, word_mode, part,
2389 2 : bitsize - bitsdone, 0, 1);
2390 : }
2391 : else
2392 : {
2393 13827 : if (bitsdone != thissize)
2394 9463 : part = expand_shift (LSHIFT_EXPR, word_mode, part,
2395 9463 : bitsdone - thissize, 0, 1);
2396 : }
2397 :
2398 13831 : if (first)
2399 : result = part;
2400 : else
2401 : /* Combine the parts with bitwise or. This works
2402 : because we extracted each part as an unsigned bit field. */
2403 9465 : result = expand_binop (word_mode, ior_optab, part, result, NULL_RTX, 1,
2404 : OPTAB_LIB_WIDEN);
2405 :
2406 13831 : first = 0;
2407 : }
2408 :
2409 : /* Unsigned bit field: we are done. */
2410 4366 : if (unsignedp)
2411 : return result;
2412 : /* Signed bit field: sign-extend with two arithmetic shifts. */
2413 1464 : result = expand_shift (LSHIFT_EXPR, word_mode, result,
2414 1464 : BITS_PER_WORD - bitsize, NULL_RTX, 0);
2415 1464 : return expand_shift (RSHIFT_EXPR, word_mode, result,
2416 1464 : BITS_PER_WORD - bitsize, NULL_RTX, 0);
2417 : }
2418 :
2419 : /* Try to read the low bits of SRC as an rvalue of mode MODE, preserving
2420 : the bit pattern. SRC_MODE is the mode of SRC; if this is smaller than
2421 : MODE, fill the upper bits with zeros. Fail if the layout of either
2422 : mode is unknown (as for CC modes) or if the extraction would involve
2423 : unprofitable mode punning. Return the value on success, otherwise
2424 : return null.
2425 :
2426 : This is different from gen_lowpart* in these respects:
2427 :
2428 : - the returned value must always be considered an rvalue
2429 :
2430 : - when MODE is wider than SRC_MODE, the extraction involves
2431 : a zero extension
2432 :
2433 : - when MODE is smaller than SRC_MODE, the extraction involves
2434 : a truncation (and is thus subject to TARGET_TRULY_NOOP_TRUNCATION).
2435 :
2436 : In other words, this routine performs a computation, whereas the
2437 : gen_lowpart* routines are conceptually lvalue or rvalue subreg
2438 : operations. */
2439 :
2440 : rtx
2441 117485 : extract_low_bits (machine_mode mode, machine_mode src_mode, rtx src)
2442 : {
2443 117485 : scalar_int_mode int_mode, src_int_mode;
2444 :
2445 117485 : if (mode == src_mode)
2446 : return src;
2447 :
2448 80939 : if (CONSTANT_P (src))
2449 : {
2450 : /* simplify_gen_subreg can't be used here, as if simplify_subreg
2451 : fails, it will happily create (subreg (symbol_ref)) or similar
2452 : invalid SUBREGs. */
2453 16538 : poly_uint64 byte = subreg_lowpart_offset (mode, src_mode);
2454 16538 : rtx ret = simplify_subreg (mode, src, src_mode, byte);
2455 16538 : if (ret)
2456 : return ret;
2457 :
2458 22 : if (GET_MODE (src) == VOIDmode
2459 22 : || !validate_subreg (mode, src_mode, src, byte))
2460 5 : return NULL_RTX;
2461 :
2462 17 : src = force_reg (GET_MODE (src), src);
2463 17 : return gen_rtx_SUBREG (mode, src, byte);
2464 : }
2465 :
2466 64401 : if (GET_MODE_CLASS (mode) == MODE_CC || GET_MODE_CLASS (src_mode) == MODE_CC)
2467 : return NULL_RTX;
2468 :
2469 128802 : if (known_eq (GET_MODE_BITSIZE (mode), GET_MODE_BITSIZE (src_mode))
2470 64401 : && targetm.modes_tieable_p (mode, src_mode))
2471 : {
2472 4158 : rtx x = gen_lowpart_common (mode, src);
2473 4158 : if (x)
2474 : return x;
2475 : }
2476 :
2477 60254 : if (!int_mode_for_mode (src_mode).exists (&src_int_mode)
2478 60241 : || !int_mode_for_mode (mode).exists (&int_mode))
2479 13 : return NULL_RTX;
2480 :
2481 60241 : if (!targetm.modes_tieable_p (src_int_mode, src_mode))
2482 : return NULL_RTX;
2483 59154 : if (!targetm.modes_tieable_p (int_mode, mode))
2484 : return NULL_RTX;
2485 :
2486 57089 : src = gen_lowpart (src_int_mode, src);
2487 57089 : if (!validate_subreg (int_mode, src_int_mode, src,
2488 : subreg_lowpart_offset (int_mode, src_int_mode)))
2489 : return NULL_RTX;
2490 :
2491 57077 : src = convert_modes (int_mode, src_int_mode, src, true);
2492 57077 : src = gen_lowpart (mode, src);
2493 57077 : return src;
2494 : }
2495 :
2496 : /* Add INC into TARGET. */
2497 :
2498 : void
2499 1185 : expand_inc (rtx target, rtx inc)
2500 : {
2501 1185 : rtx value = expand_binop (GET_MODE (target), add_optab,
2502 : target, inc,
2503 : target, 0, OPTAB_LIB_WIDEN);
2504 1185 : if (value != target)
2505 61 : emit_move_insn (target, value);
2506 1185 : }
2507 :
2508 : /* Subtract DEC from TARGET. */
2509 :
2510 : void
2511 1220 : expand_dec (rtx target, rtx dec)
2512 : {
2513 1220 : rtx value = expand_binop (GET_MODE (target), sub_optab,
2514 : target, dec,
2515 : target, 0, OPTAB_LIB_WIDEN);
2516 1220 : if (value != target)
2517 0 : emit_move_insn (target, value);
2518 1220 : }
2519 :
2520 : /* Output a shift instruction for expression code CODE,
2521 : with SHIFTED being the rtx for the value to shift,
2522 : and AMOUNT the rtx for the amount to shift by.
2523 : Store the result in the rtx TARGET, if that is convenient.
2524 : If UNSIGNEDP is nonzero, do a logical shift; otherwise, arithmetic.
2525 : Return the rtx for where the value is.
2526 : If that cannot be done, abort the compilation unless MAY_FAIL is true,
2527 : in which case 0 is returned. */
2528 :
2529 : static rtx
2530 1532680 : expand_shift_1 (enum tree_code code, machine_mode mode, rtx shifted,
2531 : rtx amount, rtx target, int unsignedp, bool may_fail = false)
2532 : {
2533 1532680 : rtx op1, temp = 0;
2534 1532680 : int left = (code == LSHIFT_EXPR || code == LROTATE_EXPR);
2535 1532680 : int rotate = (code == LROTATE_EXPR || code == RROTATE_EXPR);
2536 1532680 : optab lshift_optab = ashl_optab;
2537 1532680 : optab rshift_arith_optab = ashr_optab;
2538 1532680 : optab rshift_uns_optab = lshr_optab;
2539 1532680 : optab lrotate_optab = rotl_optab;
2540 1532680 : optab rrotate_optab = rotr_optab;
2541 1532680 : machine_mode op1_mode;
2542 1532680 : scalar_mode scalar_mode = GET_MODE_INNER (mode);
2543 1532680 : int attempt;
2544 1532680 : bool speed = optimize_insn_for_speed_p ();
2545 :
2546 1532680 : op1 = amount;
2547 1532680 : op1_mode = GET_MODE (op1);
2548 :
2549 : /* Determine whether the shift/rotate amount is a vector, or scalar. If the
2550 : shift amount is a vector, use the vector/vector shift patterns. */
2551 1532680 : if (VECTOR_MODE_P (mode) && VECTOR_MODE_P (op1_mode))
2552 : {
2553 1532680 : lshift_optab = vashl_optab;
2554 1532680 : rshift_arith_optab = vashr_optab;
2555 1532680 : rshift_uns_optab = vlshr_optab;
2556 1532680 : lrotate_optab = vrotl_optab;
2557 1532680 : rrotate_optab = vrotr_optab;
2558 : }
2559 :
2560 : /* Previously detected shift-counts computed by NEGATE_EXPR
2561 : and shifted in the other direction; but that does not work
2562 : on all machines. */
2563 :
2564 1532680 : if (SHIFT_COUNT_TRUNCATED)
2565 : {
2566 : if (CONST_INT_P (op1)
2567 : && ((unsigned HOST_WIDE_INT) INTVAL (op1) >=
2568 : (unsigned HOST_WIDE_INT) GET_MODE_BITSIZE (scalar_mode)))
2569 : op1 = gen_int_shift_amount (mode,
2570 : (unsigned HOST_WIDE_INT) INTVAL (op1)
2571 : % GET_MODE_BITSIZE (scalar_mode));
2572 : else if (GET_CODE (op1) == SUBREG
2573 : && subreg_lowpart_p (op1)
2574 : && SCALAR_INT_MODE_P (GET_MODE (SUBREG_REG (op1)))
2575 : && SCALAR_INT_MODE_P (GET_MODE (op1)))
2576 : op1 = SUBREG_REG (op1);
2577 : }
2578 :
2579 : /* Canonicalize rotates by constant amount. We may canonicalize
2580 : to reduce the immediate or if the ISA can rotate by constants
2581 : in only on direction. */
2582 1532680 : if (rotate && reverse_rotate_by_imm_p (scalar_mode, left, op1))
2583 : {
2584 3124 : op1 = gen_int_shift_amount (mode, (GET_MODE_BITSIZE (scalar_mode)
2585 3124 : - INTVAL (op1)));
2586 3124 : left = !left;
2587 3124 : code = left ? LROTATE_EXPR : RROTATE_EXPR;
2588 : }
2589 :
2590 : /* Rotation of 16bit values by 8 bits is effectively equivalent to a bswaphi.
2591 : Note that this is not the case for bigger values. For instance a rotation
2592 : of 0x01020304 by 16 bits gives 0x03040102 which is different from
2593 : 0x04030201 (bswapsi). */
2594 1532680 : if (rotate
2595 8423 : && CONST_INT_P (op1)
2596 5281 : && INTVAL (op1) == BITS_PER_UNIT
2597 1056 : && GET_MODE_SIZE (scalar_mode) == 2
2598 1533561 : && optab_handler (bswap_optab, mode) != CODE_FOR_nothing)
2599 880 : return expand_unop (mode, bswap_optab, shifted, NULL_RTX, unsignedp);
2600 :
2601 1531800 : if (op1 == const0_rtx)
2602 : return shifted;
2603 :
2604 : /* Check whether its cheaper to implement a left shift by a constant
2605 : bit count by a sequence of additions. */
2606 1484111 : if (code == LSHIFT_EXPR
2607 885050 : && CONST_INT_P (op1)
2608 854709 : && INTVAL (op1) > 0
2609 854684 : && INTVAL (op1) < GET_MODE_PRECISION (scalar_mode)
2610 854684 : && INTVAL (op1) < MAX_BITS_PER_WORD
2611 850033 : && (shift_cost (speed, mode, INTVAL (op1))
2612 850033 : > INTVAL (op1) * add_cost (speed, mode))
2613 1486934 : && shift_cost (speed, mode, INTVAL (op1)) != MAX_COST)
2614 : {
2615 : int i;
2616 5824 : for (i = 0; i < INTVAL (op1); i++)
2617 : {
2618 3001 : temp = force_reg (mode, shifted);
2619 3001 : shifted = expand_binop (mode, add_optab, temp, temp, NULL_RTX,
2620 : unsignedp, OPTAB_LIB_WIDEN);
2621 : }
2622 : return shifted;
2623 : }
2624 :
2625 2962610 : for (attempt = 0; temp == 0 && attempt < 3; attempt++)
2626 : {
2627 1481356 : enum optab_methods methods;
2628 :
2629 1481356 : if (attempt == 0)
2630 : methods = OPTAB_DIRECT;
2631 68 : else if (attempt == 1)
2632 : methods = OPTAB_WIDEN;
2633 : else
2634 34 : methods = OPTAB_LIB_WIDEN;
2635 :
2636 1481356 : if (rotate)
2637 : {
2638 : /* Widening does not work for rotation. */
2639 7611 : if (methods == OPTAB_WIDEN)
2640 34 : continue;
2641 7577 : else if (methods == OPTAB_LIB_WIDEN)
2642 : {
2643 : /* If we have been unable to open-code this by a rotation,
2644 : do it as the IOR or PLUS of two shifts. I.e., to rotate
2645 : A by N bits, compute
2646 : (A << N) | ((unsigned) A >> ((-N) & (C - 1)))
2647 : where C is the bitsize of A. If N cannot be zero,
2648 : use PLUS instead of IOR.
2649 :
2650 : It is theoretically possible that the target machine might
2651 : not be able to perform either shift and hence we would
2652 : be making two libcalls rather than just the one for the
2653 : shift (similarly if IOR could not be done). We will allow
2654 : this extremely unlikely lossage to avoid complicating the
2655 : code below. */
2656 :
2657 34 : rtx subtarget = target == shifted ? 0 : target;
2658 34 : rtx new_amount, other_amount;
2659 34 : rtx temp1;
2660 :
2661 34 : new_amount = op1;
2662 34 : if (op1 == const0_rtx)
2663 : return shifted;
2664 34 : else if (CONST_INT_P (op1))
2665 23 : other_amount = gen_int_shift_amount
2666 23 : (mode, GET_MODE_BITSIZE (scalar_mode) - INTVAL (op1));
2667 : else
2668 : {
2669 11 : other_amount
2670 22 : = simplify_gen_unary (NEG, GET_MODE (op1),
2671 11 : op1, GET_MODE (op1));
2672 11 : HOST_WIDE_INT mask = GET_MODE_PRECISION (scalar_mode) - 1;
2673 11 : other_amount
2674 11 : = simplify_gen_binary (AND, GET_MODE (op1), other_amount,
2675 11 : gen_int_mode (mask, GET_MODE (op1)));
2676 : }
2677 :
2678 34 : shifted = force_reg (mode, shifted);
2679 :
2680 45 : temp = expand_shift_1 (left ? LSHIFT_EXPR : RSHIFT_EXPR,
2681 : mode, shifted, new_amount, 0, 1);
2682 45 : temp1 = expand_shift_1 (left ? RSHIFT_EXPR : LSHIFT_EXPR,
2683 : mode, shifted, other_amount,
2684 : subtarget, 1);
2685 34 : return expand_binop (mode,
2686 34 : CONST_INT_P (op1) ? add_optab : ior_optab,
2687 34 : temp, temp1, target, unsignedp, methods);
2688 : }
2689 :
2690 11116 : temp = expand_binop (mode,
2691 : left ? lrotate_optab : rrotate_optab,
2692 : shifted, op1, target, unsignedp, methods);
2693 : }
2694 1473745 : else if (unsignedp)
2695 1228011 : temp = expand_binop (mode,
2696 : left ? lshift_optab : rshift_uns_optab,
2697 : shifted, op1, target, unsignedp, methods);
2698 :
2699 : /* Do arithmetic shifts.
2700 : Also, if we are going to widen the operand, we can just as well
2701 : use an arithmetic right-shift instead of a logical one. */
2702 1481288 : if (temp == 0 && ! rotate
2703 591167 : && (! unsignedp || (! left && methods == OPTAB_WIDEN)))
2704 : {
2705 : enum optab_methods methods1 = methods;
2706 :
2707 : /* If trying to widen a log shift to an arithmetic shift,
2708 : don't accept an arithmetic shift of the same size. */
2709 : if (unsignedp)
2710 : methods1 = OPTAB_MUST_WIDEN;
2711 :
2712 : /* Arithmetic shift */
2713 :
2714 837252 : temp = expand_binop (mode,
2715 : left ? lshift_optab : rshift_arith_optab,
2716 : shifted, op1, target, unsignedp, methods1);
2717 : }
2718 :
2719 : /* We used to try extzv here for logical right shifts, but that was
2720 : only useful for one machine, the VAX, and caused poor code
2721 : generation there for lshrdi3, so the code was deleted and a
2722 : define_expand for lshrsi3 was added to vax.md. */
2723 : }
2724 :
2725 1481254 : gcc_assert (temp != NULL_RTX || may_fail);
2726 : return temp;
2727 : }
2728 :
2729 : /* Output a shift instruction for expression code CODE,
2730 : with SHIFTED being the rtx for the value to shift,
2731 : and AMOUNT the amount to shift by.
2732 : Store the result in the rtx TARGET, if that is convenient.
2733 : If UNSIGNEDP is nonzero, do a logical shift; otherwise, arithmetic.
2734 : Return the rtx for where the value is. */
2735 :
2736 : rtx
2737 1237881 : expand_shift (enum tree_code code, machine_mode mode, rtx shifted,
2738 : poly_int64 amount, rtx target, int unsignedp)
2739 : {
2740 1237881 : return expand_shift_1 (code, mode, shifted,
2741 : gen_int_shift_amount (mode, amount),
2742 1237881 : target, unsignedp);
2743 : }
2744 :
2745 : /* Likewise, but return 0 if that cannot be done. */
2746 :
2747 : rtx
2748 322 : maybe_expand_shift (enum tree_code code, machine_mode mode, rtx shifted,
2749 : int amount, rtx target, int unsignedp)
2750 : {
2751 322 : return expand_shift_1 (code, mode,
2752 322 : shifted, GEN_INT (amount), target, unsignedp, true);
2753 : }
2754 :
2755 : /* Output a shift instruction for expression code CODE,
2756 : with SHIFTED being the rtx for the value to shift,
2757 : and AMOUNT the tree for the amount to shift by.
2758 : Store the result in the rtx TARGET, if that is convenient.
2759 : If UNSIGNEDP is nonzero, do a logical shift; otherwise, arithmetic.
2760 : Return the rtx for where the value is. */
2761 :
2762 : rtx
2763 294409 : expand_variable_shift (enum tree_code code, machine_mode mode, rtx shifted,
2764 : tree amount, rtx target, int unsignedp)
2765 : {
2766 294409 : return expand_shift_1 (code, mode,
2767 294409 : shifted, expand_normal (amount), target, unsignedp);
2768 : }
2769 :
2770 :
2771 : static void synth_mult (struct algorithm *, unsigned HOST_WIDE_INT,
2772 : const struct mult_cost *, machine_mode mode);
2773 : static rtx expand_mult_const (machine_mode, rtx, HOST_WIDE_INT, rtx,
2774 : const struct algorithm *, enum mult_variant);
2775 : static unsigned HOST_WIDE_INT invert_mod2n (unsigned HOST_WIDE_INT, int);
2776 : static rtx extract_high_half (scalar_int_mode, rtx);
2777 : static rtx expmed_mult_highpart (scalar_int_mode, rtx, rtx, rtx, int, int);
2778 :
2779 : /* Compute and return the best algorithm for multiplying by T.
2780 : The algorithm must cost less than cost_limit
2781 : If retval.cost >= COST_LIMIT, no algorithm was found and all
2782 : other field of the returned struct are undefined.
2783 : MODE is the machine mode of the multiplication. */
2784 :
2785 : static void
2786 34960195 : synth_mult (struct algorithm *alg_out, unsigned HOST_WIDE_INT t,
2787 : const struct mult_cost *cost_limit, machine_mode mode)
2788 : {
2789 34960195 : int m;
2790 34960195 : struct algorithm *alg_in, *best_alg;
2791 34960195 : struct mult_cost best_cost;
2792 34960195 : struct mult_cost new_limit;
2793 34960195 : int op_cost, op_latency;
2794 34960195 : unsigned HOST_WIDE_INT orig_t = t;
2795 34960195 : unsigned HOST_WIDE_INT q;
2796 34960195 : int maxm, hash_index;
2797 34960195 : bool cache_hit = false;
2798 34960195 : enum alg_code cache_alg = alg_zero;
2799 34960195 : bool speed = optimize_insn_for_speed_p ();
2800 34960195 : scalar_int_mode imode;
2801 34960195 : struct alg_hash_entry *entry_ptr;
2802 :
2803 : /* Indicate that no algorithm is yet found. If no algorithm
2804 : is found, this value will be returned and indicate failure. */
2805 34960195 : alg_out->cost.cost = cost_limit->cost + 1;
2806 34960195 : alg_out->cost.latency = cost_limit->latency + 1;
2807 :
2808 34960195 : if (cost_limit->cost < 0
2809 28839100 : || (cost_limit->cost == 0 && cost_limit->latency <= 0))
2810 27551310 : return;
2811 :
2812 : /* Be prepared for vector modes. */
2813 47781322 : imode = as_a <scalar_int_mode> (GET_MODE_INNER (mode));
2814 :
2815 71101065 : maxm = MIN (BITS_PER_WORD, GET_MODE_BITSIZE (imode));
2816 :
2817 : /* Restrict the bits of "t" to the multiplication's mode. */
2818 23890661 : t &= GET_MODE_MASK (imode);
2819 :
2820 : /* t == 1 can be done in zero cost. */
2821 23890661 : if (t == 1)
2822 : {
2823 6038708 : alg_out->ops = 1;
2824 6038708 : alg_out->cost.cost = 0;
2825 6038708 : alg_out->cost.latency = 0;
2826 6038708 : alg_out->op[0] = alg_m;
2827 6038708 : return;
2828 : }
2829 :
2830 : /* t == 0 sometimes has a cost. If it does and it exceeds our limit,
2831 : fail now. */
2832 17851953 : if (t == 0)
2833 : {
2834 551823 : if (MULT_COST_LESS (cost_limit, zero_cost (speed)))
2835 : return;
2836 : else
2837 : {
2838 551823 : alg_out->ops = 1;
2839 551823 : alg_out->cost.cost = zero_cost (speed);
2840 551823 : alg_out->cost.latency = zero_cost (speed);
2841 551823 : alg_out->op[0] = alg_zero;
2842 551823 : return;
2843 : }
2844 : }
2845 :
2846 : /* We'll be needing a couple extra algorithm structures now. */
2847 :
2848 17300130 : alg_in = XALLOCA (struct algorithm);
2849 17300130 : best_alg = XALLOCA (struct algorithm);
2850 17300130 : best_cost = *cost_limit;
2851 :
2852 : /* Compute the hash index. */
2853 17300130 : hash_index = (t ^ (unsigned int) mode ^ (speed * 256)) % NUM_ALG_HASH_ENTRIES;
2854 :
2855 : /* See if we already know what to do for T. */
2856 17300130 : entry_ptr = alg_hash_entry_ptr (hash_index);
2857 17300130 : if (entry_ptr->t == t
2858 14382905 : && entry_ptr->mode == mode
2859 14382905 : && entry_ptr->speed == speed
2860 14382905 : && entry_ptr->alg != alg_unknown)
2861 : {
2862 14382905 : cache_alg = entry_ptr->alg;
2863 :
2864 14382905 : if (cache_alg == alg_impossible)
2865 : {
2866 : /* The cache tells us that it's impossible to synthesize
2867 : multiplication by T within entry_ptr->cost. */
2868 6610332 : if (!CHEAPER_MULT_COST (&entry_ptr->cost, cost_limit))
2869 : /* COST_LIMIT is at least as restrictive as the one
2870 : recorded in the hash table, in which case we have no
2871 : hope of synthesizing a multiplication. Just
2872 : return. */
2873 : return;
2874 :
2875 : /* If we get here, COST_LIMIT is less restrictive than the
2876 : one recorded in the hash table, so we may be able to
2877 : synthesize a multiplication. Proceed as if we didn't
2878 : have the cache entry. */
2879 : }
2880 : else
2881 : {
2882 7772573 : if (CHEAPER_MULT_COST (cost_limit, &entry_ptr->cost))
2883 : /* The cached algorithm shows that this multiplication
2884 : requires more cost than COST_LIMIT. Just return. This
2885 : way, we don't clobber this cache entry with
2886 : alg_impossible but retain useful information. */
2887 : return;
2888 :
2889 7034450 : cache_hit = true;
2890 :
2891 7034450 : switch (cache_alg)
2892 : {
2893 4651762 : case alg_shift:
2894 4651762 : goto do_alg_shift;
2895 :
2896 954750 : case alg_add_t_m2:
2897 954750 : case alg_sub_t_m2:
2898 954750 : goto do_alg_addsub_t_m2;
2899 :
2900 115747 : case alg_add_factor:
2901 115747 : case alg_sub_factor:
2902 115747 : goto do_alg_addsub_factor;
2903 :
2904 1312183 : case alg_add_t2_m:
2905 1312183 : goto do_alg_add_t2_m;
2906 :
2907 8 : case alg_sub_t2_m:
2908 8 : goto do_alg_sub_t2_m;
2909 :
2910 0 : default:
2911 0 : gcc_unreachable ();
2912 : }
2913 : }
2914 : }
2915 :
2916 : /* If we have a group of zero bits at the low-order part of T, try
2917 : multiplying by the remaining bits and then doing a shift. */
2918 :
2919 3713201 : if ((t & 1) == 0)
2920 : {
2921 1897818 : do_alg_shift:
2922 6549580 : m = ctz_or_zero (t); /* m = number of low zero bits */
2923 6549580 : if (m < maxm)
2924 : {
2925 6548594 : q = t >> m;
2926 : /* The function expand_shift will choose between a shift and
2927 : a sequence of additions, so the observed cost is given as
2928 : MIN (m * add_cost(speed, mode), shift_cost(speed, mode, m)). */
2929 6548594 : op_cost = m * add_cost (speed, mode);
2930 6548594 : if (shift_cost (speed, mode, m) < op_cost)
2931 : op_cost = shift_cost (speed, mode, m);
2932 6548594 : new_limit.cost = best_cost.cost - op_cost;
2933 6548594 : new_limit.latency = best_cost.latency - op_cost;
2934 6548594 : synth_mult (alg_in, q, &new_limit, mode);
2935 :
2936 6548594 : alg_in->cost.cost += op_cost;
2937 6548594 : alg_in->cost.latency += op_cost;
2938 6548594 : if (CHEAPER_MULT_COST (&alg_in->cost, &best_cost))
2939 : {
2940 4253903 : best_cost = alg_in->cost;
2941 4253903 : std::swap (alg_in, best_alg);
2942 4253903 : best_alg->log[best_alg->ops] = m;
2943 4253903 : best_alg->op[best_alg->ops] = alg_shift;
2944 : }
2945 :
2946 : /* See if treating ORIG_T as a signed number yields a better
2947 : sequence. Try this sequence only for a negative ORIG_T
2948 : as it would be useless for a non-negative ORIG_T. */
2949 6548594 : if ((HOST_WIDE_INT) orig_t < 0)
2950 : {
2951 : /* Shift ORIG_T as follows because a right shift of a
2952 : negative-valued signed type is implementation
2953 : defined. */
2954 646959 : q = ~(~orig_t >> m);
2955 : /* The function expand_shift will choose between a shift
2956 : and a sequence of additions, so the observed cost is
2957 : given as MIN (m * add_cost(speed, mode),
2958 : shift_cost(speed, mode, m)). */
2959 646959 : op_cost = m * add_cost (speed, mode);
2960 646959 : if (shift_cost (speed, mode, m) < op_cost)
2961 : op_cost = shift_cost (speed, mode, m);
2962 646959 : new_limit.cost = best_cost.cost - op_cost;
2963 646959 : new_limit.latency = best_cost.latency - op_cost;
2964 646959 : synth_mult (alg_in, q, &new_limit, mode);
2965 :
2966 646959 : alg_in->cost.cost += op_cost;
2967 646959 : alg_in->cost.latency += op_cost;
2968 646959 : if (CHEAPER_MULT_COST (&alg_in->cost, &best_cost))
2969 : {
2970 616626 : best_cost = alg_in->cost;
2971 616626 : std::swap (alg_in, best_alg);
2972 616626 : best_alg->log[best_alg->ops] = m;
2973 616626 : best_alg->op[best_alg->ops] = alg_shift;
2974 : }
2975 : }
2976 : }
2977 986 : else if (2 * BITS_PER_WORD <= HOST_BITS_PER_WIDE_INT
2978 986 : && GET_MODE_BITSIZE (imode) == 2 * BITS_PER_WORD
2979 986 : && m >= BITS_PER_WORD
2980 1972 : && imode == mode)
2981 : {
2982 986 : q = t >> m;
2983 986 : int op1_cost = shift_cost (speed, mode, m - BITS_PER_WORD);
2984 986 : int op2_cost = zero_cost (speed);
2985 986 : op_latency = MAX (op1_cost, op2_cost);
2986 986 : op_cost = op1_cost + op2_cost;
2987 :
2988 986 : new_limit.cost = best_cost.cost - op_cost;
2989 986 : new_limit.latency = best_cost.latency - op_latency;
2990 986 : synth_mult (alg_in, q, &new_limit, mode);
2991 986 : alg_in->cost.cost += op_cost;
2992 986 : alg_in->cost.latency += op_latency;
2993 986 : if (CHEAPER_MULT_COST (&alg_in->cost, &best_cost))
2994 : {
2995 912 : best_cost = alg_in->cost;
2996 912 : std::swap (alg_in, best_alg);
2997 912 : best_alg->log[best_alg->ops] = m;
2998 912 : best_alg->op[best_alg->ops] = alg_shift;
2999 : }
3000 : }
3001 6549580 : if (cache_hit)
3002 4651762 : goto done;
3003 : }
3004 :
3005 : /* If we have an odd number, add or subtract one. */
3006 1897818 : if ((t & 1) != 0)
3007 : {
3008 2770133 : unsigned HOST_WIDE_INT w;
3009 :
3010 0 : do_alg_addsub_t_m2:
3011 40833223 : for (w = 1; (w & t) != 0; w <<= 1)
3012 : ;
3013 : /* If T was -1, then W will be zero after the loop. This is another
3014 : case where T ends with ...111. Handling this with (T + 1) and
3015 : subtract 1 produces slightly better code and results in algorithm
3016 : selection much faster than treating it like the ...0111 case
3017 : below. */
3018 2770133 : if (w == 0
3019 2345255 : || (w > 2
3020 : /* Reject the case where t is 3.
3021 : Thus we prefer addition in that case. */
3022 2345255 : && t != 3))
3023 : {
3024 : /* T ends with ...111. Multiply by (T + 1) and subtract T. */
3025 :
3026 1569017 : op_cost = add_cost (speed, mode);
3027 1569017 : new_limit.cost = best_cost.cost - op_cost;
3028 1569017 : new_limit.latency = best_cost.latency - op_cost;
3029 1569017 : synth_mult (alg_in, t + 1, &new_limit, mode);
3030 :
3031 1569017 : alg_in->cost.cost += op_cost;
3032 1569017 : alg_in->cost.latency += op_cost;
3033 1569017 : if (CHEAPER_MULT_COST (&alg_in->cost, &best_cost))
3034 : {
3035 726025 : best_cost = alg_in->cost;
3036 726025 : std::swap (alg_in, best_alg);
3037 726025 : best_alg->log[best_alg->ops] = 0;
3038 726025 : best_alg->op[best_alg->ops] = alg_sub_t_m2;
3039 : }
3040 : }
3041 : else
3042 : {
3043 : /* T ends with ...01 or ...011. Multiply by (T - 1) and add T. */
3044 :
3045 1201116 : op_cost = add_cost (speed, mode);
3046 1201116 : new_limit.cost = best_cost.cost - op_cost;
3047 1201116 : new_limit.latency = best_cost.latency - op_cost;
3048 1201116 : synth_mult (alg_in, t - 1, &new_limit, mode);
3049 :
3050 1201116 : alg_in->cost.cost += op_cost;
3051 1201116 : alg_in->cost.latency += op_cost;
3052 1201116 : if (CHEAPER_MULT_COST (&alg_in->cost, &best_cost))
3053 : {
3054 183623 : best_cost = alg_in->cost;
3055 183623 : std::swap (alg_in, best_alg);
3056 183623 : best_alg->log[best_alg->ops] = 0;
3057 183623 : best_alg->op[best_alg->ops] = alg_add_t_m2;
3058 : }
3059 : }
3060 :
3061 : /* We may be able to calculate a * -7, a * -15, a * -31, etc
3062 : quickly with a - a * n for some appropriate constant n. */
3063 2770133 : m = exact_log2 (-orig_t + 1);
3064 2770133 : if (m >= 0 && m < maxm)
3065 : {
3066 759178 : op_cost = add_cost (speed, mode) + shift_cost (speed, mode, m);
3067 : /* If the target has a cheap shift-and-subtract insn use
3068 : that in preference to a shift insn followed by a sub insn.
3069 : Assume that the shift-and-sub is "atomic" with a latency
3070 : equal to it's cost, otherwise assume that on superscalar
3071 : hardware the shift may be executed concurrently with the
3072 : earlier steps in the algorithm. */
3073 759178 : if (shiftsub1_cost (speed, mode, m) <= op_cost)
3074 : {
3075 : op_cost = shiftsub1_cost (speed, mode, m);
3076 : op_latency = op_cost;
3077 : }
3078 : else
3079 753431 : op_latency = add_cost (speed, mode);
3080 :
3081 759178 : new_limit.cost = best_cost.cost - op_cost;
3082 759178 : new_limit.latency = best_cost.latency - op_latency;
3083 759178 : synth_mult (alg_in, (unsigned HOST_WIDE_INT) (-orig_t + 1) >> m,
3084 : &new_limit, mode);
3085 :
3086 759178 : alg_in->cost.cost += op_cost;
3087 759178 : alg_in->cost.latency += op_latency;
3088 759178 : if (CHEAPER_MULT_COST (&alg_in->cost, &best_cost))
3089 : {
3090 191911 : best_cost = alg_in->cost;
3091 191911 : std::swap (alg_in, best_alg);
3092 191911 : best_alg->log[best_alg->ops] = m;
3093 191911 : best_alg->op[best_alg->ops] = alg_sub_t_m2;
3094 : }
3095 : }
3096 :
3097 2770133 : if (cache_hit)
3098 954750 : goto done;
3099 : }
3100 :
3101 : /* Look for factors of t of the form
3102 : t = q(2**m +- 1), 2 <= m <= floor(log2(t - 1)).
3103 : If we find such a factor, we can multiply by t using an algorithm that
3104 : multiplies by q, shift the result by m and add/subtract it to itself.
3105 :
3106 : We search for large factors first and loop down, even if large factors
3107 : are less probable than small; if we find a large factor we will find a
3108 : good sequence quickly, and therefore be able to prune (by decreasing
3109 : COST_LIMIT) the search. */
3110 :
3111 1897818 : do_alg_addsub_factor:
3112 72399474 : for (m = floor_log2 (t - 1); m >= 2; m--)
3113 : {
3114 70532623 : unsigned HOST_WIDE_INT d;
3115 :
3116 70532623 : d = (HOST_WIDE_INT_1U << m) + 1;
3117 70532623 : if (t % d == 0 && t > d && m < maxm
3118 940521 : && (!cache_hit || cache_alg == alg_add_factor))
3119 : {
3120 940521 : op_cost = add_cost (speed, mode) + shift_cost (speed, mode, m);
3121 940521 : if (shiftadd_cost (speed, mode, m) <= op_cost)
3122 : op_cost = shiftadd_cost (speed, mode, m);
3123 :
3124 940521 : op_latency = op_cost;
3125 :
3126 :
3127 940521 : new_limit.cost = best_cost.cost - op_cost;
3128 940521 : new_limit.latency = best_cost.latency - op_latency;
3129 940521 : synth_mult (alg_in, t / d, &new_limit, mode);
3130 :
3131 940521 : alg_in->cost.cost += op_cost;
3132 940521 : alg_in->cost.latency += op_latency;
3133 940521 : if (alg_in->cost.latency < op_cost)
3134 196377 : alg_in->cost.latency = op_cost;
3135 940521 : if (CHEAPER_MULT_COST (&alg_in->cost, &best_cost))
3136 : {
3137 114665 : best_cost = alg_in->cost;
3138 114665 : std::swap (alg_in, best_alg);
3139 114665 : best_alg->log[best_alg->ops] = m;
3140 114665 : best_alg->op[best_alg->ops] = alg_add_factor;
3141 : }
3142 : /* Other factors will have been taken care of in the recursion. */
3143 : break;
3144 : }
3145 :
3146 69592102 : d = (HOST_WIDE_INT_1U << m) - 1;
3147 69592102 : if (t % d == 0 && t > d && m < maxm
3148 1021576 : && (!cache_hit || cache_alg == alg_sub_factor))
3149 : {
3150 1021576 : op_cost = add_cost (speed, mode) + shift_cost (speed, mode, m);
3151 1021576 : if (shiftsub0_cost (speed, mode, m) <= op_cost)
3152 : op_cost = shiftsub0_cost (speed, mode, m);
3153 :
3154 1021576 : op_latency = op_cost;
3155 :
3156 1021576 : new_limit.cost = best_cost.cost - op_cost;
3157 1021576 : new_limit.latency = best_cost.latency - op_latency;
3158 1021576 : synth_mult (alg_in, t / d, &new_limit, mode);
3159 :
3160 1021576 : alg_in->cost.cost += op_cost;
3161 1021576 : alg_in->cost.latency += op_latency;
3162 1021576 : if (alg_in->cost.latency < op_cost)
3163 269511 : alg_in->cost.latency = op_cost;
3164 1021576 : if (CHEAPER_MULT_COST (&alg_in->cost, &best_cost))
3165 : {
3166 30909 : best_cost = alg_in->cost;
3167 30909 : std::swap (alg_in, best_alg);
3168 30909 : best_alg->log[best_alg->ops] = m;
3169 30909 : best_alg->op[best_alg->ops] = alg_sub_factor;
3170 : }
3171 : break;
3172 : }
3173 : }
3174 3828948 : if (cache_hit)
3175 115747 : goto done;
3176 :
3177 : /* Try shift-and-add (load effective address) instructions,
3178 : i.e. do a*3, a*5, a*9. */
3179 3713201 : if ((t & 1) != 0)
3180 : {
3181 1815383 : do_alg_add_t2_m:
3182 3127566 : q = t - 1;
3183 3127566 : m = ctz_hwi (q);
3184 3127566 : if (q && m < maxm)
3185 : {
3186 3127550 : op_cost = shiftadd_cost (speed, mode, m);
3187 3127550 : new_limit.cost = best_cost.cost - op_cost;
3188 3127550 : new_limit.latency = best_cost.latency - op_cost;
3189 3127550 : synth_mult (alg_in, (t - 1) >> m, &new_limit, mode);
3190 :
3191 3127550 : alg_in->cost.cost += op_cost;
3192 3127550 : alg_in->cost.latency += op_cost;
3193 3127550 : if (CHEAPER_MULT_COST (&alg_in->cost, &best_cost))
3194 : {
3195 1365693 : best_cost = alg_in->cost;
3196 1365693 : std::swap (alg_in, best_alg);
3197 1365693 : best_alg->log[best_alg->ops] = m;
3198 1365693 : best_alg->op[best_alg->ops] = alg_add_t2_m;
3199 : }
3200 : }
3201 3127566 : if (cache_hit)
3202 1312183 : goto done;
3203 :
3204 1815383 : do_alg_sub_t2_m:
3205 1815391 : q = t + 1;
3206 1815391 : m = ctz_hwi (q);
3207 1815391 : if (q && m < maxm)
3208 : {
3209 1793329 : op_cost = shiftsub0_cost (speed, mode, m);
3210 1793329 : new_limit.cost = best_cost.cost - op_cost;
3211 1793329 : new_limit.latency = best_cost.latency - op_cost;
3212 1793329 : synth_mult (alg_in, (t + 1) >> m, &new_limit, mode);
3213 :
3214 1793329 : alg_in->cost.cost += op_cost;
3215 1793329 : alg_in->cost.latency += op_cost;
3216 1793329 : if (CHEAPER_MULT_COST (&alg_in->cost, &best_cost))
3217 : {
3218 64 : best_cost = alg_in->cost;
3219 64 : std::swap (alg_in, best_alg);
3220 64 : best_alg->log[best_alg->ops] = m;
3221 64 : best_alg->op[best_alg->ops] = alg_sub_t2_m;
3222 : }
3223 : }
3224 1815391 : if (cache_hit)
3225 : goto done;
3226 : }
3227 :
3228 1897818 : done:
3229 : /* If best_cost has not decreased, we have not found any algorithm. */
3230 10747651 : if (!CHEAPER_MULT_COST (&best_cost, cost_limit))
3231 : {
3232 : /* We failed to find an algorithm. Record alg_impossible for
3233 : this case (that is, <T, MODE, COST_LIMIT>) so that next time
3234 : we are asked to find an algorithm for T within the same or
3235 : lower COST_LIMIT, we can immediately return to the
3236 : caller. */
3237 3338766 : entry_ptr->t = t;
3238 3338766 : entry_ptr->mode = mode;
3239 3338766 : entry_ptr->speed = speed;
3240 3338766 : entry_ptr->alg = alg_impossible;
3241 3338766 : entry_ptr->cost = *cost_limit;
3242 3338766 : return;
3243 : }
3244 :
3245 : /* Cache the result. */
3246 7408885 : if (!cache_hit)
3247 : {
3248 684778 : entry_ptr->t = t;
3249 684778 : entry_ptr->mode = mode;
3250 684778 : entry_ptr->speed = speed;
3251 684778 : entry_ptr->alg = best_alg->op[best_alg->ops];
3252 684778 : entry_ptr->cost.cost = best_cost.cost;
3253 684778 : entry_ptr->cost.latency = best_cost.latency;
3254 : }
3255 :
3256 : /* If we are getting a too long sequence for `struct algorithm'
3257 : to record, make this search fail. */
3258 7408885 : if (best_alg->ops == MAX_BITS_PER_WORD)
3259 : return;
3260 :
3261 : /* Copy the algorithm from temporary space to the space at alg_out.
3262 : We avoid using structure assignment because the majority of
3263 : best_alg is normally undefined, and this is a critical function. */
3264 7408885 : alg_out->ops = best_alg->ops + 1;
3265 7408885 : alg_out->cost = best_cost;
3266 7408885 : memcpy (alg_out->op, best_alg->op,
3267 7408885 : alg_out->ops * sizeof *alg_out->op);
3268 7408885 : memcpy (alg_out->log, best_alg->log,
3269 : alg_out->ops * sizeof *alg_out->log);
3270 : }
3271 :
3272 : /* Find the cheapest way of multiplying a value of mode MODE by VAL.
3273 : Try three variations:
3274 :
3275 : - a shift/add sequence based on VAL itself
3276 : - a shift/add sequence based on -VAL, followed by a negation
3277 : - a shift/add sequence based on VAL - 1, followed by an addition.
3278 :
3279 : Return true if the cheapest of these cost less than MULT_COST,
3280 : describing the algorithm in *ALG and final fixup in *VARIANT. */
3281 :
3282 : bool
3283 7350700 : choose_mult_variant (machine_mode mode, HOST_WIDE_INT val,
3284 : struct algorithm *alg, enum mult_variant *variant,
3285 : int mult_cost)
3286 : {
3287 7350700 : struct algorithm alg2;
3288 7350700 : struct mult_cost limit;
3289 7350700 : int op_cost;
3290 7350700 : bool speed = optimize_insn_for_speed_p ();
3291 :
3292 : /* Fail quickly for impossible bounds. */
3293 7350700 : if (mult_cost < 0)
3294 : return false;
3295 :
3296 : /* Ensure that mult_cost provides a reasonable upper bound.
3297 : Any constant multiplication can be performed with less
3298 : than 2 * bits additions. */
3299 14698210 : op_cost = 2 * GET_MODE_UNIT_BITSIZE (mode) * add_cost (speed, mode);
3300 7349105 : if (mult_cost > op_cost)
3301 : mult_cost = op_cost;
3302 :
3303 7349105 : *variant = basic_variant;
3304 7349105 : limit.cost = mult_cost;
3305 7349105 : limit.latency = mult_cost;
3306 7349105 : synth_mult (alg, val, &limit, mode);
3307 :
3308 : /* This works only if the inverted value actually fits in an
3309 : `unsigned int' */
3310 14698210 : if (HOST_BITS_PER_INT >= GET_MODE_UNIT_BITSIZE (mode))
3311 : {
3312 2653159 : op_cost = neg_cost (speed, mode);
3313 2653159 : if (MULT_COST_LESS (&alg->cost, mult_cost))
3314 : {
3315 2549209 : limit.cost = alg->cost.cost - op_cost;
3316 2549209 : limit.latency = alg->cost.latency - op_cost;
3317 : }
3318 : else
3319 : {
3320 103950 : limit.cost = mult_cost - op_cost;
3321 103950 : limit.latency = mult_cost - op_cost;
3322 : }
3323 :
3324 2653159 : synth_mult (&alg2, -val, &limit, mode);
3325 2653159 : alg2.cost.cost += op_cost;
3326 2653159 : alg2.cost.latency += op_cost;
3327 2653159 : if (CHEAPER_MULT_COST (&alg2.cost, &alg->cost))
3328 11927 : *alg = alg2, *variant = negate_variant;
3329 : }
3330 :
3331 : /* This proves very useful for division-by-constant. */
3332 7349105 : op_cost = add_cost (speed, mode);
3333 7349105 : if (MULT_COST_LESS (&alg->cost, mult_cost))
3334 : {
3335 6510947 : limit.cost = alg->cost.cost - op_cost;
3336 6510947 : limit.latency = alg->cost.latency - op_cost;
3337 : }
3338 : else
3339 : {
3340 838158 : limit.cost = mult_cost - op_cost;
3341 838158 : limit.latency = mult_cost - op_cost;
3342 : }
3343 :
3344 7349105 : if (val != HOST_WIDE_INT_MIN
3345 7349115 : || GET_MODE_UNIT_PRECISION (mode) == HOST_BITS_PER_WIDE_INT)
3346 : {
3347 7349105 : synth_mult (&alg2, val - HOST_WIDE_INT_1U, &limit, mode);
3348 7349105 : alg2.cost.cost += op_cost;
3349 7349105 : alg2.cost.latency += op_cost;
3350 7349105 : if (CHEAPER_MULT_COST (&alg2.cost, &alg->cost))
3351 2781 : *alg = alg2, *variant = add_variant;
3352 : }
3353 :
3354 7349105 : return MULT_COST_LESS (&alg->cost, mult_cost);
3355 : }
3356 :
3357 : /* A subroutine of expand_mult, used for constant multiplications.
3358 : Multiply OP0 by VAL in mode MODE, storing the result in TARGET if
3359 : convenient. Use the shift/add sequence described by ALG and apply
3360 : the final fixup specified by VARIANT. */
3361 :
3362 : static rtx
3363 137966 : expand_mult_const (machine_mode mode, rtx op0, HOST_WIDE_INT val,
3364 : rtx target, const struct algorithm *alg,
3365 : enum mult_variant variant)
3366 : {
3367 137966 : unsigned HOST_WIDE_INT val_so_far;
3368 137966 : rtx_insn *insn;
3369 137966 : rtx accum, tem;
3370 137966 : int opno;
3371 137966 : machine_mode nmode;
3372 :
3373 : /* Avoid referencing memory over and over and invalid sharing
3374 : on SUBREGs. */
3375 137966 : op0 = force_reg (mode, op0);
3376 :
3377 : /* ACCUM starts out either as OP0 or as a zero, depending on
3378 : the first operation. */
3379 :
3380 137966 : if (alg->op[0] == alg_zero)
3381 : {
3382 5730 : accum = copy_to_mode_reg (mode, CONST0_RTX (mode));
3383 5730 : val_so_far = 0;
3384 : }
3385 132236 : else if (alg->op[0] == alg_m)
3386 : {
3387 132236 : accum = copy_to_mode_reg (mode, op0);
3388 132236 : val_so_far = 1;
3389 : }
3390 : else
3391 0 : gcc_unreachable ();
3392 :
3393 390318 : for (opno = 1; opno < alg->ops; opno++)
3394 : {
3395 252352 : int log = alg->log[opno];
3396 252352 : rtx shift_subtarget = optimize ? 0 : accum;
3397 236239 : rtx add_target
3398 137966 : = (opno == alg->ops - 1 && target != 0 && variant != add_variant
3399 40448 : && !optimize)
3400 252352 : ? target : 0;
3401 252352 : rtx accum_target = optimize ? 0 : accum;
3402 252352 : rtx accum_inner;
3403 :
3404 252352 : switch (alg->op[opno])
3405 : {
3406 110023 : case alg_shift:
3407 110023 : tem = expand_shift (LSHIFT_EXPR, mode, accum, log, NULL_RTX, 0);
3408 : /* REG_EQUAL note will be attached to the following insn. */
3409 110023 : emit_move_insn (accum, tem);
3410 110023 : val_so_far <<= log;
3411 110023 : break;
3412 :
3413 6085 : case alg_add_t_m2:
3414 6085 : tem = expand_shift (LSHIFT_EXPR, mode, op0, log, NULL_RTX, 0);
3415 12170 : accum = force_operand (gen_rtx_PLUS (mode, accum, tem),
3416 : add_target ? add_target : accum_target);
3417 6085 : val_so_far += HOST_WIDE_INT_1U << log;
3418 6085 : break;
3419 :
3420 19677 : case alg_sub_t_m2:
3421 19677 : tem = expand_shift (LSHIFT_EXPR, mode, op0, log, NULL_RTX, 0);
3422 39354 : accum = force_operand (gen_rtx_MINUS (mode, accum, tem),
3423 : add_target ? add_target : accum_target);
3424 19677 : val_so_far -= HOST_WIDE_INT_1U << log;
3425 19677 : break;
3426 :
3427 115423 : case alg_add_t2_m:
3428 115423 : accum = expand_shift (LSHIFT_EXPR, mode, accum,
3429 115423 : log, shift_subtarget, 0);
3430 230846 : accum = force_operand (gen_rtx_PLUS (mode, accum, op0),
3431 : add_target ? add_target : accum_target);
3432 115423 : val_so_far = (val_so_far << log) + 1;
3433 115423 : break;
3434 :
3435 0 : case alg_sub_t2_m:
3436 0 : accum = expand_shift (LSHIFT_EXPR, mode, accum,
3437 0 : log, shift_subtarget, 0);
3438 0 : accum = force_operand (gen_rtx_MINUS (mode, accum, op0),
3439 : add_target ? add_target : accum_target);
3440 0 : val_so_far = (val_so_far << log) - 1;
3441 0 : break;
3442 :
3443 1051 : case alg_add_factor:
3444 1051 : tem = expand_shift (LSHIFT_EXPR, mode, accum, log, NULL_RTX, 0);
3445 2102 : accum = force_operand (gen_rtx_PLUS (mode, accum, tem),
3446 : add_target ? add_target : accum_target);
3447 1051 : val_so_far += val_so_far << log;
3448 1051 : break;
3449 :
3450 93 : case alg_sub_factor:
3451 93 : tem = expand_shift (LSHIFT_EXPR, mode, accum, log, NULL_RTX, 0);
3452 186 : accum = force_operand (gen_rtx_MINUS (mode, tem, accum),
3453 : (add_target
3454 93 : ? add_target : (optimize ? 0 : tem)));
3455 93 : val_so_far = (val_so_far << log) - val_so_far;
3456 93 : break;
3457 :
3458 0 : default:
3459 0 : gcc_unreachable ();
3460 : }
3461 :
3462 252352 : if (SCALAR_INT_MODE_P (mode))
3463 : {
3464 : /* Write a REG_EQUAL note on the last insn so that we can cse
3465 : multiplication sequences. Note that if ACCUM is a SUBREG,
3466 : we've set the inner register and must properly indicate that. */
3467 245717 : tem = op0, nmode = mode;
3468 245717 : accum_inner = accum;
3469 245717 : if (GET_CODE (accum) == SUBREG)
3470 : {
3471 0 : accum_inner = SUBREG_REG (accum);
3472 0 : nmode = GET_MODE (accum_inner);
3473 0 : tem = gen_lowpart (nmode, op0);
3474 : }
3475 :
3476 : /* Don't add a REG_EQUAL note if tem is a paradoxical SUBREG.
3477 : In that case, only the low bits of accum would be guaranteed to
3478 : be equal to the content of the REG_EQUAL note, the upper bits
3479 : can be anything. */
3480 245717 : if (!paradoxical_subreg_p (tem))
3481 : {
3482 245717 : insn = get_last_insn ();
3483 245717 : wide_int wval_so_far
3484 245717 : = wi::uhwi (val_so_far,
3485 245717 : GET_MODE_PRECISION (as_a <scalar_mode> (nmode)));
3486 245717 : rtx c = immed_wide_int_const (wval_so_far, nmode);
3487 245717 : set_dst_reg_note (insn, REG_EQUAL, gen_rtx_MULT (nmode, tem, c),
3488 : accum_inner);
3489 245717 : }
3490 : }
3491 : }
3492 :
3493 137966 : if (variant == negate_variant)
3494 : {
3495 510 : val_so_far = -val_so_far;
3496 510 : accum = expand_unop (mode, neg_optab, accum, target, 0);
3497 : }
3498 137456 : else if (variant == add_variant)
3499 : {
3500 25 : val_so_far = val_so_far + 1;
3501 25 : accum = force_operand (gen_rtx_PLUS (mode, accum, op0), target);
3502 : }
3503 :
3504 : /* Compare only the bits of val and val_so_far that are significant
3505 : in the result mode, to avoid sign-/zero-extension confusion. */
3506 137966 : nmode = GET_MODE_INNER (mode);
3507 137966 : val &= GET_MODE_MASK (nmode);
3508 137966 : val_so_far &= GET_MODE_MASK (nmode);
3509 137966 : gcc_assert (val == (HOST_WIDE_INT) val_so_far);
3510 :
3511 137966 : return accum;
3512 : }
3513 :
3514 : /* Perform a multiplication and return an rtx for the result.
3515 : MODE is mode of value; OP0 and OP1 are what to multiply (rtx's);
3516 : TARGET is a suggestion for where to store the result (an rtx).
3517 :
3518 : We check specially for a constant integer as OP1.
3519 : If you want this check for OP0 as well, then before calling
3520 : you should swap the two operands if OP0 would be constant. */
3521 :
3522 : rtx
3523 1114846 : expand_mult (machine_mode mode, rtx op0, rtx op1, rtx target,
3524 : int unsignedp, bool no_libcall)
3525 : {
3526 1114846 : enum mult_variant variant;
3527 1114846 : struct algorithm algorithm;
3528 1114846 : rtx scalar_op1;
3529 1114846 : int max_cost;
3530 1114846 : bool speed = optimize_insn_for_speed_p ();
3531 1114846 : bool do_trapv = flag_trapv && SCALAR_INT_MODE_P (mode) && !unsignedp;
3532 :
3533 1114846 : if (CONSTANT_P (op0))
3534 275 : std::swap (op0, op1);
3535 :
3536 : /* For vectors, there are several simplifications that can be made if
3537 : all elements of the vector constant are identical. */
3538 1114846 : scalar_op1 = unwrap_const_vec_duplicate (op1);
3539 :
3540 1114846 : if (INTEGRAL_MODE_P (mode))
3541 : {
3542 996470 : rtx fake_reg;
3543 996470 : HOST_WIDE_INT coeff;
3544 996470 : bool is_neg;
3545 996470 : int mode_bitsize;
3546 :
3547 996470 : if (op1 == CONST0_RTX (mode))
3548 : return op1;
3549 996470 : if (op1 == CONST1_RTX (mode))
3550 : return op0;
3551 951270 : if (op1 == CONSTM1_RTX (mode))
3552 2822 : return expand_unop (mode, do_trapv ? negv_optab : neg_optab,
3553 1411 : op0, target, 0);
3554 :
3555 949859 : if (do_trapv)
3556 32 : goto skip_synth;
3557 :
3558 : /* If mode is integer vector mode, check if the backend supports
3559 : vector lshift (by scalar or vector) at all. If not, we can't use
3560 : synthesized multiply. */
3561 949827 : if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT
3562 14505 : && optab_handler (vashl_optab, mode) == CODE_FOR_nothing
3563 961935 : && optab_handler (ashl_optab, mode) == CODE_FOR_nothing)
3564 0 : goto skip_synth;
3565 :
3566 : /* These are the operations that are potentially turned into
3567 : a sequence of shifts and additions. */
3568 949827 : mode_bitsize = GET_MODE_UNIT_BITSIZE (mode);
3569 :
3570 : /* synth_mult does an `unsigned int' multiply. As long as the mode is
3571 : less than or equal in size to `unsigned int' this doesn't matter.
3572 : If the mode is larger than `unsigned int', then synth_mult works
3573 : only if the constant value exactly fits in an `unsigned int' without
3574 : any truncation. This means that multiplying by negative values does
3575 : not work; results are off by 2^32 on a 32 bit machine. */
3576 949827 : if (CONST_INT_P (scalar_op1))
3577 : {
3578 694454 : coeff = INTVAL (scalar_op1);
3579 694454 : is_neg = coeff < 0;
3580 : }
3581 : #if TARGET_SUPPORTS_WIDE_INT
3582 255373 : else if (CONST_WIDE_INT_P (scalar_op1))
3583 : #else
3584 : else if (CONST_DOUBLE_AS_INT_P (scalar_op1))
3585 : #endif
3586 : {
3587 1140 : int shift = wi::exact_log2 (rtx_mode_t (scalar_op1, mode));
3588 : /* Perfect power of 2 (other than 1, which is handled above). */
3589 1140 : if (shift > 0)
3590 106 : return expand_shift (LSHIFT_EXPR, mode, op0,
3591 106 : shift, target, unsignedp);
3592 : else
3593 1034 : goto skip_synth;
3594 : }
3595 : else
3596 254233 : goto skip_synth;
3597 :
3598 : /* We used to test optimize here, on the grounds that it's better to
3599 : produce a smaller program when -O is not used. But this causes
3600 : such a terrible slowdown sometimes that it seems better to always
3601 : use synth_mult. */
3602 :
3603 : /* Special case powers of two. */
3604 694454 : if (EXACT_POWER_OF_2_OR_ZERO_P (coeff)
3605 468412 : && !(is_neg && mode_bitsize > HOST_BITS_PER_WIDE_INT))
3606 468404 : return expand_shift (LSHIFT_EXPR, mode, op0,
3607 936808 : floor_log2 (coeff), target, unsignedp);
3608 :
3609 226050 : fake_reg = gen_raw_REG (mode, LAST_VIRTUAL_REGISTER + 1);
3610 :
3611 : /* Attempt to handle multiplication of DImode values by negative
3612 : coefficients, by performing the multiplication by a positive
3613 : multiplier and then inverting the result. */
3614 226050 : if (is_neg && mode_bitsize > HOST_BITS_PER_WIDE_INT)
3615 : {
3616 : /* Its safe to use -coeff even for INT_MIN, as the
3617 : result is interpreted as an unsigned coefficient.
3618 : Exclude cost of op0 from max_cost to match the cost
3619 : calculation of the synth_mult. */
3620 216 : coeff = -(unsigned HOST_WIDE_INT) coeff;
3621 216 : max_cost = (set_src_cost (gen_rtx_MULT (mode, fake_reg, op1),
3622 : mode, speed)
3623 216 : - neg_cost (speed, mode));
3624 216 : if (max_cost <= 0)
3625 0 : goto skip_synth;
3626 :
3627 : /* Special case powers of two. */
3628 216 : if (EXACT_POWER_OF_2_OR_ZERO_P (coeff))
3629 : {
3630 342 : rtx temp = expand_shift (LSHIFT_EXPR, mode, op0,
3631 171 : floor_log2 (coeff), target, unsignedp);
3632 171 : return expand_unop (mode, neg_optab, temp, target, 0);
3633 : }
3634 :
3635 45 : if (choose_mult_variant (mode, coeff, &algorithm, &variant,
3636 : max_cost))
3637 : {
3638 44 : rtx temp = expand_mult_const (mode, op0, coeff, NULL_RTX,
3639 : &algorithm, variant);
3640 44 : return expand_unop (mode, neg_optab, temp, target, 0);
3641 : }
3642 1 : goto skip_synth;
3643 : }
3644 :
3645 : /* Exclude cost of op0 from max_cost to match the cost
3646 : calculation of the synth_mult. */
3647 225834 : max_cost = set_src_cost (gen_rtx_MULT (mode, fake_reg, op1), mode, speed);
3648 225834 : if (choose_mult_variant (mode, coeff, &algorithm, &variant, max_cost))
3649 137525 : return expand_mult_const (mode, op0, coeff, target,
3650 137525 : &algorithm, variant);
3651 : }
3652 88309 : skip_synth:
3653 :
3654 : /* Expand x*2.0 as x+x. */
3655 37916 : if (CONST_DOUBLE_AS_FLOAT_P (scalar_op1)
3656 499901 : && real_equal (CONST_DOUBLE_REAL_VALUE (scalar_op1), &dconst2))
3657 : {
3658 5928 : op0 = force_reg (GET_MODE (op0), op0);
3659 11856 : return expand_binop (mode, add_optab, op0, op0,
3660 : target, unsignedp,
3661 5928 : no_libcall ? OPTAB_WIDEN : OPTAB_LIB_WIDEN);
3662 : }
3663 :
3664 : /* This used to use umul_optab if unsigned, but for non-widening multiply
3665 : there is no difference between signed and unsigned. */
3666 1368139 : op0 = expand_binop (mode, do_trapv ? smulv_optab : smul_optab,
3667 : op0, op1, target, unsignedp,
3668 : no_libcall ? OPTAB_WIDEN : OPTAB_LIB_WIDEN);
3669 456057 : gcc_assert (op0 || no_libcall);
3670 : return op0;
3671 : }
3672 :
3673 : /* Return a cost estimate for multiplying a register by the given
3674 : COEFFicient in the given MODE and SPEED. */
3675 :
3676 : int
3677 6795792 : mult_by_coeff_cost (HOST_WIDE_INT coeff, machine_mode mode, bool speed)
3678 : {
3679 6795792 : int max_cost;
3680 6795792 : struct algorithm algorithm;
3681 6795792 : enum mult_variant variant;
3682 :
3683 6795792 : rtx fake_reg = gen_raw_REG (mode, LAST_VIRTUAL_REGISTER + 1);
3684 6795792 : max_cost = set_src_cost (gen_rtx_MULT (mode, fake_reg, fake_reg),
3685 : mode, speed);
3686 6795792 : if (choose_mult_variant (mode, coeff, &algorithm, &variant, max_cost))
3687 6050020 : return algorithm.cost.cost;
3688 : else
3689 : return max_cost;
3690 : }
3691 :
3692 : /* Perform a widening multiplication and return an rtx for the result.
3693 : MODE is mode of value; OP0 and OP1 are what to multiply (rtx's);
3694 : TARGET is a suggestion for where to store the result (an rtx).
3695 : THIS_OPTAB is the optab we should use, it must be either umul_widen_optab
3696 : or smul_widen_optab.
3697 :
3698 : We check specially for a constant integer as OP1, comparing the
3699 : cost of a widening multiply against the cost of a sequence of shifts
3700 : and adds. */
3701 :
3702 : rtx
3703 18476 : expand_widening_mult (machine_mode mode, rtx op0, rtx op1, rtx target,
3704 : int unsignedp, optab this_optab)
3705 : {
3706 18476 : bool speed = optimize_insn_for_speed_p ();
3707 18476 : rtx cop1;
3708 :
3709 18476 : if (CONST_INT_P (op1)
3710 4254 : && GET_MODE (op0) != VOIDmode
3711 4254 : && (cop1 = convert_modes (mode, GET_MODE (op0), op1,
3712 : this_optab == umul_widen_optab))
3713 4254 : && CONST_INT_P (cop1)
3714 22209 : && (INTVAL (cop1) >= 0
3715 20963 : || HWI_COMPUTABLE_MODE_P (mode)))
3716 : {
3717 3500 : HOST_WIDE_INT coeff = INTVAL (cop1);
3718 3500 : int max_cost;
3719 3500 : enum mult_variant variant;
3720 3500 : struct algorithm algorithm;
3721 :
3722 3500 : if (coeff == 0)
3723 1013 : return CONST0_RTX (mode);
3724 :
3725 : /* Special case powers of two. */
3726 3392 : if (EXACT_POWER_OF_2_OR_ZERO_P (coeff))
3727 : {
3728 520 : op0 = convert_to_mode (mode, op0, this_optab == umul_widen_optab);
3729 520 : return expand_shift (LSHIFT_EXPR, mode, op0,
3730 520 : floor_log2 (coeff), target, unsignedp);
3731 : }
3732 :
3733 : /* Exclude cost of op0 from max_cost to match the cost
3734 : calculation of the synth_mult. */
3735 2872 : max_cost = mul_widen_cost (speed, mode);
3736 2872 : if (choose_mult_variant (mode, coeff, &algorithm, &variant,
3737 : max_cost))
3738 : {
3739 385 : op0 = convert_to_mode (mode, op0, this_optab == umul_widen_optab);
3740 385 : return expand_mult_const (mode, op0, coeff, target,
3741 385 : &algorithm, variant);
3742 : }
3743 : }
3744 17463 : return expand_binop (mode, this_optab, op0, op1, target,
3745 17463 : unsignedp, OPTAB_LIB_WIDEN);
3746 : }
3747 :
3748 : /* Choose a minimal N + 1 bit approximation to 2**K / D that can be used to
3749 : replace division by D, put the least significant N bits of the result in
3750 : *MULTIPLIER_PTR, the value K - N in *POST_SHIFT_PTR, and return the most
3751 : significant bit.
3752 :
3753 : The width of operations is N (should be <= HOST_BITS_PER_WIDE_INT), the
3754 : needed precision is PRECISION (should be <= N).
3755 :
3756 : PRECISION should be as small as possible so this function can choose the
3757 : multiplier more freely. If PRECISION is <= N - 1, the most significant
3758 : bit returned by the function will be zero.
3759 :
3760 : Using this function, x / D is equal to (x*m) / 2**N >> (*POST_SHIFT_PTR),
3761 : where m is the full N + 1 bit multiplier. */
3762 :
3763 : unsigned HOST_WIDE_INT
3764 64890 : choose_multiplier (unsigned HOST_WIDE_INT d, int n, int precision,
3765 : unsigned HOST_WIDE_INT *multiplier_ptr,
3766 : int *post_shift_ptr)
3767 : {
3768 64890 : int lgup, post_shift;
3769 64890 : int pow1, pow2;
3770 :
3771 : /* lgup = ceil(log2(d)) */
3772 : /* Assuming d > 1, we have d >= 2^(lgup-1) + 1 */
3773 64890 : lgup = ceil_log2 (d);
3774 :
3775 64890 : gcc_assert (lgup <= n);
3776 64890 : gcc_assert (lgup <= precision);
3777 :
3778 64890 : pow1 = n + lgup;
3779 64890 : pow2 = n + lgup - precision;
3780 :
3781 : /* mlow = 2^(n + lgup)/d */
3782 : /* Trivially from above we have mlow < 2^(n+1) */
3783 64890 : wide_int val = wi::set_bit_in_zero (pow1, HOST_BITS_PER_DOUBLE_INT);
3784 64890 : wide_int mlow = wi::udiv_trunc (val, d);
3785 :
3786 : /* mhigh = (2^(n + lgup) + 2^(n + lgup - precision))/d */
3787 : /* From above we have mhigh < 2^(n+1) assuming lgup <= precision */
3788 : /* From precision <= n, the difference between the numerators of mhigh and
3789 : mlow is >= 2^lgup >= d. Therefore the difference of the quotients in
3790 : the Euclidean division by d is at least 1, so we have mlow < mhigh and
3791 : the exact value of 2^(n + lgup)/d lies in the interval [mlow; mhigh). */
3792 64890 : val |= wi::set_bit_in_zero (pow2, HOST_BITS_PER_DOUBLE_INT);
3793 64890 : wide_int mhigh = wi::udiv_trunc (val, d);
3794 :
3795 : /* Reduce to lowest terms. */
3796 : /* If precision <= n - 1, then the difference between the numerators of
3797 : mhigh and mlow is >= 2^(lgup + 1) >= 2 * 2^lgup >= 2 * d. Therefore
3798 : the difference of the quotients in the Euclidean division by d is at
3799 : least 2, which means that mhigh and mlow differ by at least one bit
3800 : not in the last place. The conclusion is that the first iteration of
3801 : the loop below completes and shifts mhigh and mlow by 1 bit, which in
3802 : particular means that mhigh < 2^n, that is to say, the most significant
3803 : bit in the n + 1 bit value is zero. */
3804 171777 : for (post_shift = lgup; post_shift > 0; post_shift--)
3805 : {
3806 166240 : unsigned HOST_WIDE_INT ml_lo = wi::extract_uhwi (mlow, 1,
3807 : HOST_BITS_PER_WIDE_INT);
3808 166240 : unsigned HOST_WIDE_INT mh_lo = wi::extract_uhwi (mhigh, 1,
3809 : HOST_BITS_PER_WIDE_INT);
3810 166240 : if (ml_lo >= mh_lo)
3811 : break;
3812 :
3813 106887 : mlow = wi::uhwi (ml_lo, HOST_BITS_PER_DOUBLE_INT);
3814 106887 : mhigh = wi::uhwi (mh_lo, HOST_BITS_PER_DOUBLE_INT);
3815 : }
3816 :
3817 64890 : *post_shift_ptr = post_shift;
3818 :
3819 64890 : if (n < HOST_BITS_PER_WIDE_INT)
3820 : {
3821 41769 : unsigned HOST_WIDE_INT mask = (HOST_WIDE_INT_1U << n) - 1;
3822 41769 : *multiplier_ptr = mhigh.to_uhwi () & mask;
3823 41769 : return mhigh.to_uhwi () > mask;
3824 : }
3825 : else
3826 : {
3827 23121 : *multiplier_ptr = mhigh.to_uhwi ();
3828 23121 : return wi::extract_uhwi (mhigh, HOST_BITS_PER_WIDE_INT, 1);
3829 : }
3830 64890 : }
3831 :
3832 : /* Compute the inverse of X mod 2**N, i.e., find Y such that X * Y is congruent
3833 : to 1 modulo 2**N, assuming that X is odd. Bézout's lemma guarantees that Y
3834 : exists for any given positive N. */
3835 :
3836 : static unsigned HOST_WIDE_INT
3837 51963 : invert_mod2n (unsigned HOST_WIDE_INT x, int n)
3838 : {
3839 51963 : gcc_assert ((x & 1) == 1);
3840 :
3841 : /* The algorithm notes that the choice Y = X satisfies X*Y == 1 mod 2^3,
3842 : since X is odd. Then each Newton-Raphson iteration doubles the number
3843 : of bits of significance in Y (Hensel's lemma). */
3844 :
3845 53271 : const unsigned HOST_WIDE_INT mask
3846 : = (n == HOST_BITS_PER_WIDE_INT
3847 51963 : ? HOST_WIDE_INT_M1U
3848 1308 : : (HOST_WIDE_INT_1U << n) - 1);
3849 51963 : unsigned HOST_WIDE_INT y = x;
3850 51963 : int nbit = 3;
3851 :
3852 310440 : while (nbit < n)
3853 : {
3854 258477 : y = y * (2 - x*y) & mask; /* Modulo 2^N */
3855 258477 : nbit *= 2;
3856 : }
3857 :
3858 51963 : return y;
3859 : }
3860 :
3861 : /* Emit code to adjust ADJ_OPERAND after multiplication of wrong signedness
3862 : flavor of OP0 and OP1. ADJ_OPERAND is already the high half of the
3863 : product OP0 x OP1. If UNSIGNEDP is nonzero, adjust the signed product
3864 : to become unsigned, if UNSIGNEDP is zero, adjust the unsigned product to
3865 : become signed.
3866 :
3867 : The result is put in TARGET if that is convenient.
3868 :
3869 : MODE is the mode of operation. */
3870 :
3871 : rtx
3872 0 : expand_mult_highpart_adjust (scalar_int_mode mode, rtx adj_operand, rtx op0,
3873 : rtx op1, rtx target, int unsignedp)
3874 : {
3875 0 : rtx tem;
3876 0 : enum rtx_code adj_code = unsignedp ? PLUS : MINUS;
3877 :
3878 0 : tem = expand_shift (RSHIFT_EXPR, mode, op0,
3879 0 : GET_MODE_BITSIZE (mode) - 1, NULL_RTX, 0);
3880 0 : tem = expand_and (mode, tem, op1, NULL_RTX);
3881 0 : adj_operand
3882 0 : = force_operand (gen_rtx_fmt_ee (adj_code, mode, adj_operand, tem),
3883 : adj_operand);
3884 :
3885 0 : tem = expand_shift (RSHIFT_EXPR, mode, op1,
3886 0 : GET_MODE_BITSIZE (mode) - 1, NULL_RTX, 0);
3887 0 : tem = expand_and (mode, tem, op0, NULL_RTX);
3888 0 : target = force_operand (gen_rtx_fmt_ee (adj_code, mode, adj_operand, tem),
3889 : target);
3890 :
3891 0 : return target;
3892 : }
3893 :
3894 : /* Subroutine of expmed_mult_highpart. Return the MODE high part of OP. */
3895 :
3896 : static rtx
3897 19494 : extract_high_half (scalar_int_mode mode, rtx op)
3898 : {
3899 19494 : if (mode == word_mode)
3900 0 : return gen_highpart (mode, op);
3901 :
3902 19494 : scalar_int_mode wider_mode = GET_MODE_WIDER_MODE (mode).require ();
3903 :
3904 38988 : op = expand_shift (RSHIFT_EXPR, wider_mode, op,
3905 19494 : GET_MODE_BITSIZE (mode), 0, 1);
3906 19494 : return convert_modes (mode, wider_mode, op, 0);
3907 : }
3908 :
3909 : /* Like expmed_mult_highpart, but only consider using multiplication optab. */
3910 :
3911 : rtx
3912 45872 : expmed_mult_highpart_optab (scalar_int_mode mode, rtx op0, rtx op1,
3913 : rtx target, int unsignedp, int max_cost)
3914 : {
3915 45872 : const scalar_int_mode wider_mode = GET_MODE_WIDER_MODE (mode).require ();
3916 45872 : const bool speed = optimize_insn_for_speed_p ();
3917 45872 : const int size = GET_MODE_BITSIZE (mode);
3918 45872 : optab moptab;
3919 45872 : rtx tem;
3920 :
3921 : /* Firstly, try using a multiplication insn that only generates the needed
3922 : high part of the product, and in the sign flavor of unsignedp. */
3923 45872 : if (mul_highpart_cost (speed, mode) < max_cost)
3924 : {
3925 43828 : moptab = unsignedp ? umul_highpart_optab : smul_highpart_optab;
3926 43828 : tem = expand_binop (mode, moptab, op0, op1, target, unsignedp,
3927 : OPTAB_DIRECT);
3928 43828 : if (tem)
3929 : return tem;
3930 : }
3931 :
3932 : /* Secondly, same as above, but use sign flavor opposite of unsignedp.
3933 : Need to adjust the result after the multiplication. */
3934 21668 : if (size - 1 < BITS_PER_WORD
3935 43114 : && (mul_highpart_cost (speed, mode)
3936 21446 : + 2 * shift_cost (speed, mode, size-1)
3937 21446 : + 4 * add_cost (speed, mode) < max_cost))
3938 : {
3939 5122 : moptab = unsignedp ? smul_highpart_optab : umul_highpart_optab;
3940 5122 : tem = expand_binop (mode, moptab, op0, op1, target, !unsignedp,
3941 : OPTAB_DIRECT);
3942 5122 : if (tem)
3943 : /* We used the wrong signedness. Adjust the result. */
3944 0 : return expand_mult_highpart_adjust (mode, tem, op0, op1, tem,
3945 0 : unsignedp);
3946 : }
3947 :
3948 : /* Try widening multiplication. */
3949 21668 : moptab = unsignedp ? umul_widen_optab : smul_widen_optab;
3950 21668 : if (convert_optab_handler (moptab, wider_mode, mode) != CODE_FOR_nothing
3951 21668 : && mul_widen_cost (speed, wider_mode) < max_cost)
3952 : {
3953 381 : tem = expand_binop (wider_mode, moptab, op0, op1, NULL_RTX, unsignedp,
3954 : OPTAB_WIDEN);
3955 381 : if (tem)
3956 381 : return extract_high_half (mode, tem);
3957 : }
3958 :
3959 : /* Try widening the mode and perform a non-widening multiplication. */
3960 21287 : if (optab_handler (smul_optab, wider_mode) != CODE_FOR_nothing
3961 20721 : && size - 1 < BITS_PER_WORD
3962 42004 : && (mul_cost (speed, wider_mode) + shift_cost (speed, mode, size-1)
3963 : < max_cost))
3964 : {
3965 19101 : rtx_insn *insns;
3966 19101 : rtx wop0, wop1;
3967 :
3968 : /* We need to widen the operands, for example to ensure the
3969 : constant multiplier is correctly sign or zero extended.
3970 : Use a sequence to clean-up any instructions emitted by
3971 : the conversions if things don't work out. */
3972 19101 : start_sequence ();
3973 19101 : wop0 = convert_modes (wider_mode, mode, op0, unsignedp);
3974 19101 : wop1 = convert_modes (wider_mode, mode, op1, unsignedp);
3975 19101 : tem = expand_binop (wider_mode, smul_optab, wop0, wop1, 0,
3976 : unsignedp, OPTAB_WIDEN);
3977 19101 : insns = end_sequence ();
3978 :
3979 19101 : if (tem)
3980 : {
3981 19101 : emit_insn (insns);
3982 19101 : return extract_high_half (mode, tem);
3983 : }
3984 : }
3985 :
3986 : /* Try widening multiplication of opposite signedness, and adjust. */
3987 2186 : moptab = unsignedp ? smul_widen_optab : umul_widen_optab;
3988 2186 : if (convert_optab_handler (moptab, wider_mode, mode) != CODE_FOR_nothing
3989 448 : && size - 1 < BITS_PER_WORD
3990 2970 : && (mul_widen_cost (speed, wider_mode)
3991 392 : + 2 * shift_cost (speed, mode, size-1)
3992 392 : + 4 * add_cost (speed, mode) < max_cost))
3993 : {
3994 0 : tem = expand_binop (wider_mode, moptab, op0, op1, NULL_RTX, !unsignedp,
3995 : OPTAB_WIDEN);
3996 0 : if (tem != 0)
3997 : {
3998 0 : tem = extract_high_half (mode, tem);
3999 : /* We used the wrong signedness. Adjust the result. */
4000 0 : return expand_mult_highpart_adjust (mode, tem, op0, op1, target,
4001 0 : unsignedp);
4002 : }
4003 : }
4004 :
4005 : return 0;
4006 : }
4007 :
4008 : /* Emit code to multiply OP0 and OP1 (where OP1 is an integer constant),
4009 : putting the high half of the result in TARGET if that is convenient,
4010 : and return where the result is. If the operation cannot be performed,
4011 : 0 is returned.
4012 :
4013 : MODE is the mode of operation and result.
4014 :
4015 : UNSIGNEDP nonzero means unsigned multiply.
4016 :
4017 : MAX_COST is the total allowed cost for the expanded RTL. */
4018 :
4019 : static rtx
4020 45872 : expmed_mult_highpart (scalar_int_mode mode, rtx op0, rtx op1,
4021 : rtx target, int unsignedp, int max_cost)
4022 : {
4023 45872 : const bool speed = optimize_insn_for_speed_p ();
4024 45872 : unsigned HOST_WIDE_INT cnst1;
4025 45872 : int extra_cost;
4026 45872 : bool sign_adjust = false;
4027 45872 : enum mult_variant variant;
4028 45872 : struct algorithm alg;
4029 45872 : rtx narrow_op1, tem;
4030 :
4031 : /* We can't support modes wider than HOST_BITS_PER_INT. */
4032 45872 : gcc_assert (HWI_COMPUTABLE_MODE_P (mode));
4033 :
4034 45872 : cnst1 = INTVAL (op1) & GET_MODE_MASK (mode);
4035 45872 : narrow_op1 = gen_int_mode (INTVAL (op1), mode);
4036 :
4037 : /* We can't optimize modes wider than BITS_PER_WORD.
4038 : ??? We might be able to perform double-word arithmetic if
4039 : mode == word_mode, however all the cost calculations in
4040 : synth_mult etc. assume single-word operations. */
4041 45872 : scalar_int_mode wider_mode = GET_MODE_WIDER_MODE (mode).require ();
4042 94853 : if (GET_MODE_BITSIZE (wider_mode) > BITS_PER_WORD)
4043 24774 : return expmed_mult_highpart_optab (mode, op0, narrow_op1, target,
4044 24774 : unsignedp, max_cost);
4045 :
4046 42196 : extra_cost = shift_cost (speed, mode, GET_MODE_BITSIZE (mode) - 1);
4047 :
4048 : /* Check whether we try to multiply by a negative constant. */
4049 31411 : if (!unsignedp && ((cnst1 >> (GET_MODE_BITSIZE (mode) - 1)) & 1))
4050 : {
4051 2237 : sign_adjust = true;
4052 2237 : extra_cost += add_cost (speed, mode);
4053 : }
4054 :
4055 : /* See whether shift/add multiplication is cheap enough. */
4056 21098 : if (choose_mult_variant (wider_mode, cnst1, &alg, &variant,
4057 : max_cost - extra_cost))
4058 : {
4059 : /* See whether the specialized multiplication optabs are
4060 : cheaper than the shift/add version. */
4061 38828 : tem = expmed_mult_highpart_optab (mode, op0, narrow_op1, target,
4062 : unsignedp,
4063 19414 : alg.cost.cost + extra_cost);
4064 19414 : if (tem)
4065 : return tem;
4066 :
4067 12 : tem = convert_to_mode (wider_mode, op0, unsignedp);
4068 12 : tem = expand_mult_const (wider_mode, tem, cnst1, 0, &alg, variant);
4069 12 : tem = extract_high_half (mode, tem);
4070 :
4071 : /* Adjust result for signedness. */
4072 12 : if (sign_adjust)
4073 0 : tem = force_operand (gen_rtx_MINUS (mode, tem, op0), tem);
4074 :
4075 12 : return tem;
4076 : }
4077 1684 : return expmed_mult_highpart_optab (mode, op0, narrow_op1, target,
4078 1684 : unsignedp, max_cost);
4079 : }
4080 :
4081 :
4082 : /* Expand signed modulus of OP0 by a power of two D in mode MODE. */
4083 :
4084 : static rtx
4085 2520 : expand_smod_pow2 (scalar_int_mode mode, rtx op0, HOST_WIDE_INT d)
4086 : {
4087 2520 : rtx result, temp, shift;
4088 2520 : rtx_code_label *label;
4089 2520 : int logd;
4090 2520 : int prec = GET_MODE_PRECISION (mode);
4091 :
4092 2520 : logd = floor_log2 (d);
4093 2520 : result = gen_reg_rtx (mode);
4094 :
4095 : /* Avoid conditional branches when they're expensive. */
4096 2520 : if (BRANCH_COST (optimize_insn_for_speed_p (), false) >= 2
4097 2520 : && optimize_insn_for_speed_p ())
4098 : {
4099 2516 : rtx signmask = emit_store_flag (result, LT, op0, const0_rtx,
4100 : mode, 0, -1);
4101 2516 : if (signmask)
4102 : {
4103 2516 : HOST_WIDE_INT masklow = (HOST_WIDE_INT_1 << logd) - 1;
4104 2516 : signmask = force_reg (mode, signmask);
4105 5032 : shift = gen_int_shift_amount (mode, GET_MODE_BITSIZE (mode) - logd);
4106 :
4107 : /* Use the rtx_cost of a LSHIFTRT instruction to determine
4108 : which instruction sequence to use. If logical right shifts
4109 : are expensive the use 2 XORs, 2 SUBs and an AND, otherwise
4110 : use a LSHIFTRT, 1 ADD, 1 SUB and an AND. */
4111 :
4112 2516 : temp = gen_rtx_LSHIFTRT (mode, result, shift);
4113 2516 : if (optab_handler (lshr_optab, mode) == CODE_FOR_nothing
4114 2516 : || (set_src_cost (temp, mode, optimize_insn_for_speed_p ())
4115 : > COSTS_N_INSNS (2)))
4116 : {
4117 91 : temp = expand_binop (mode, xor_optab, op0, signmask,
4118 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4119 91 : temp = expand_binop (mode, sub_optab, temp, signmask,
4120 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4121 91 : temp = expand_binop (mode, and_optab, temp,
4122 91 : gen_int_mode (masklow, mode),
4123 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4124 91 : temp = expand_binop (mode, xor_optab, temp, signmask,
4125 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4126 91 : temp = expand_binop (mode, sub_optab, temp, signmask,
4127 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4128 : }
4129 : else
4130 : {
4131 2425 : signmask = expand_binop (mode, lshr_optab, signmask, shift,
4132 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4133 2425 : signmask = force_reg (mode, signmask);
4134 :
4135 2425 : temp = expand_binop (mode, add_optab, op0, signmask,
4136 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4137 2425 : temp = expand_binop (mode, and_optab, temp,
4138 2425 : gen_int_mode (masklow, mode),
4139 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4140 2425 : temp = expand_binop (mode, sub_optab, temp, signmask,
4141 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4142 : }
4143 2516 : return temp;
4144 : }
4145 : }
4146 :
4147 : /* Mask contains the mode's signbit and the significant bits of the
4148 : modulus. By including the signbit in the operation, many targets
4149 : can avoid an explicit compare operation in the following comparison
4150 : against zero. */
4151 4 : wide_int mask = wi::mask (logd, false, prec);
4152 4 : mask = wi::set_bit (mask, prec - 1);
4153 :
4154 8 : temp = expand_binop (mode, and_optab, op0,
4155 4 : immed_wide_int_const (mask, mode),
4156 : result, 1, OPTAB_LIB_WIDEN);
4157 4 : if (temp != result)
4158 0 : emit_move_insn (result, temp);
4159 :
4160 4 : label = gen_label_rtx ();
4161 4 : do_cmp_and_jump (result, const0_rtx, GE, mode, label);
4162 :
4163 4 : temp = expand_binop (mode, sub_optab, result, const1_rtx, result,
4164 : 0, OPTAB_LIB_WIDEN);
4165 :
4166 4 : mask = wi::mask (logd, true, prec);
4167 8 : temp = expand_binop (mode, ior_optab, temp,
4168 4 : immed_wide_int_const (mask, mode),
4169 : result, 1, OPTAB_LIB_WIDEN);
4170 4 : temp = expand_binop (mode, add_optab, temp, const1_rtx, result,
4171 : 0, OPTAB_LIB_WIDEN);
4172 4 : if (temp != result)
4173 0 : emit_move_insn (result, temp);
4174 4 : emit_label (label);
4175 4 : return result;
4176 4 : }
4177 :
4178 : /* Expand signed division of OP0 by a power of two D in mode MODE.
4179 : This routine is only called for positive values of D. */
4180 :
4181 : static rtx
4182 10206 : expand_sdiv_pow2 (scalar_int_mode mode, rtx op0, HOST_WIDE_INT d)
4183 : {
4184 10206 : rtx temp;
4185 10206 : rtx_code_label *label;
4186 10206 : int logd;
4187 :
4188 10206 : logd = floor_log2 (d);
4189 :
4190 10206 : if (d == 2
4191 10206 : && BRANCH_COST (optimize_insn_for_speed_p (),
4192 : false) >= 1)
4193 : {
4194 6780 : temp = gen_reg_rtx (mode);
4195 6780 : temp = emit_store_flag (temp, LT, op0, const0_rtx, mode, 0, 1);
4196 6780 : if (temp != NULL_RTX)
4197 : {
4198 6780 : temp = expand_binop (mode, add_optab, temp, op0, NULL_RTX,
4199 : 0, OPTAB_LIB_WIDEN);
4200 6780 : return expand_shift (RSHIFT_EXPR, mode, temp, logd, NULL_RTX, 0);
4201 : }
4202 : }
4203 :
4204 6851 : if (HAVE_conditional_move
4205 3426 : && BRANCH_COST (optimize_insn_for_speed_p (), false) >= 2)
4206 : {
4207 3426 : rtx temp2;
4208 :
4209 3426 : start_sequence ();
4210 3426 : temp2 = copy_to_mode_reg (mode, op0);
4211 3426 : temp = expand_binop (mode, add_optab, temp2, gen_int_mode (d - 1, mode),
4212 : NULL_RTX, 0, OPTAB_LIB_WIDEN);
4213 3426 : temp = force_reg (mode, temp);
4214 :
4215 : /* Construct "temp2 = (temp2 < 0) ? temp : temp2". */
4216 3426 : temp2 = emit_conditional_move (temp2, { LT, temp2, const0_rtx, mode },
4217 : temp, temp2, mode, 0);
4218 3426 : if (temp2)
4219 : {
4220 3376 : rtx_insn *seq = end_sequence ();
4221 3376 : emit_insn (seq);
4222 3376 : return expand_shift (RSHIFT_EXPR, mode, temp2, logd, NULL_RTX, 0);
4223 : }
4224 50 : end_sequence ();
4225 : }
4226 :
4227 50 : if (BRANCH_COST (optimize_insn_for_speed_p (),
4228 : false) >= 2)
4229 : {
4230 50 : int ushift = GET_MODE_BITSIZE (mode) - logd;
4231 :
4232 50 : temp = gen_reg_rtx (mode);
4233 50 : temp = emit_store_flag (temp, LT, op0, const0_rtx, mode, 0, -1);
4234 50 : if (temp != NULL_RTX)
4235 : {
4236 100 : if (GET_MODE_BITSIZE (mode) >= BITS_PER_WORD
4237 50 : || shift_cost (optimize_insn_for_speed_p (), mode, ushift)
4238 : > COSTS_N_INSNS (1))
4239 50 : temp = expand_binop (mode, and_optab, temp,
4240 50 : gen_int_mode (d - 1, mode),
4241 : NULL_RTX, 0, OPTAB_LIB_WIDEN);
4242 : else
4243 0 : temp = expand_shift (RSHIFT_EXPR, mode, temp,
4244 0 : ushift, NULL_RTX, 1);
4245 50 : temp = expand_binop (mode, add_optab, temp, op0, NULL_RTX,
4246 : 0, OPTAB_LIB_WIDEN);
4247 50 : return expand_shift (RSHIFT_EXPR, mode, temp, logd, NULL_RTX, 0);
4248 : }
4249 : }
4250 :
4251 0 : label = gen_label_rtx ();
4252 0 : temp = copy_to_mode_reg (mode, op0);
4253 0 : do_cmp_and_jump (temp, const0_rtx, GE, mode, label);
4254 0 : expand_inc (temp, gen_int_mode (d - 1, mode));
4255 0 : emit_label (label);
4256 0 : return expand_shift (RSHIFT_EXPR, mode, temp, logd, NULL_RTX, 0);
4257 : }
4258 :
4259 : /* Emit the code to divide OP0 by OP1, putting the result in TARGET
4260 : if that is convenient, and returning where the result is.
4261 : You may request either the quotient or the remainder as the result;
4262 : specify REM_FLAG nonzero to get the remainder.
4263 :
4264 : CODE is the expression code for which kind of division this is;
4265 : it controls how rounding is done. MODE is the machine mode to use.
4266 : UNSIGNEDP nonzero means do unsigned division. */
4267 :
4268 : /* ??? For CEIL_MOD_EXPR, can compute incorrect remainder with ANDI
4269 : and then correct it by or'ing in missing high bits
4270 : if result of ANDI is nonzero.
4271 : For ROUND_MOD_EXPR, can use ANDI and then sign-extend the result.
4272 : This could optimize to a bfexts instruction.
4273 : But C doesn't use these operations, so their optimizations are
4274 : left for later. */
4275 : /* ??? For modulo, we don't actually need the highpart of the first product,
4276 : the low part will do nicely. And for small divisors, the second multiply
4277 : can also be a low-part only multiply or even be completely left out.
4278 : E.g. to calculate the remainder of a division by 3 with a 32 bit
4279 : multiply, multiply with 0x55555556 and extract the upper two bits;
4280 : the result is exact for inputs up to 0x1fffffff.
4281 : The input range can be reduced by using cross-sum rules.
4282 : For odd divisors >= 3, the following table gives right shift counts
4283 : so that if a number is shifted by an integer multiple of the given
4284 : amount, the remainder stays the same:
4285 : 2, 4, 3, 6, 10, 12, 4, 8, 18, 6, 11, 20, 18, 0, 5, 10, 12, 0, 12, 20,
4286 : 14, 12, 23, 21, 8, 0, 20, 18, 0, 0, 6, 12, 0, 22, 0, 18, 20, 30, 0, 0,
4287 : 0, 8, 0, 11, 12, 10, 36, 0, 30, 0, 0, 12, 0, 0, 0, 0, 44, 12, 24, 0,
4288 : 20, 0, 7, 14, 0, 18, 36, 0, 0, 46, 60, 0, 42, 0, 15, 24, 20, 0, 0, 33,
4289 : 0, 20, 0, 0, 18, 0, 60, 0, 0, 0, 0, 0, 40, 18, 0, 0, 12
4290 :
4291 : Cross-sum rules for even numbers can be derived by leaving as many bits
4292 : to the right alone as the divisor has zeros to the right.
4293 : E.g. if x is an unsigned 32 bit number:
4294 : (x mod 12) == (((x & 1023) + ((x >> 8) & ~3)) * 0x15555558 >> 2 * 3) >> 28
4295 : */
4296 :
4297 : /* Helper for expand_divmod's unsigned constant division. For OP0 in
4298 : INT_MODE divided by a constant needing a (SIZE+1)-bit multiplier ML
4299 : with right shift POST_SHIFT (the mh != 0 case), try to obtain
4300 : the quotient from the high part of a single multiply in a mode twice
4301 : as wide as INT_MODE. Return the quotient in INT_MODE, having emitted
4302 : the insns, or NULL_RTX when the transformation is unavailable or not
4303 : cheaper than the classic sub/shift/add sequence. EXTRA_COST is the
4304 : cost of that sequence's follow-up ops, MAX_COST bounds the multiply
4305 : and SPEED selects the cost model.
4306 :
4307 : The magic constant occupies at most 2*SIZE bits and so must fit in a
4308 : HOST_WIDE_INT (always 64 bits today; checked below). A wider INT_MODE
4309 : such as DImode -- which would need a 128-bit magic and a single-word
4310 : high-part multiply in a 2x-wide mode that common targets lack -- is
4311 : therefore excluded. */
4312 :
4313 : static rtx
4314 2322 : expand_wide_mulh_udiv (scalar_int_mode int_mode, rtx op0,
4315 : unsigned HOST_WIDE_INT ml, int size, int post_shift,
4316 : int extra_cost, int max_cost, bool speed)
4317 : {
4318 2322 : scalar_int_mode wide_mode;
4319 :
4320 : /* We need POST_SHIFT >= 1, a wider integer mode that still fits in a
4321 : word, and the pre-shifted magic constant to fit in a HOST_WIDE_INT. */
4322 2322 : if (post_shift < 1
4323 3686 : || !GET_MODE_2XWIDER_MODE (int_mode).exists (&wide_mode)
4324 2415 : || GET_MODE_BITSIZE (wide_mode) > BITS_PER_WORD
4325 4717 : || GET_MODE_BITSIZE (wide_mode) > HOST_BITS_PER_WIDE_INT)
4326 : return NULL_RTX;
4327 :
4328 : /* The caller obtained ML and POST_SHIFT from choose_multiplier, which
4329 : guarantees POST_SHIFT <= ceil (log2 (d)) <= SIZE for a SIZE-bit
4330 : divisor d, so the shift count below is non-negative. */
4331 1031 : gcc_checking_assert (post_shift <= size);
4332 :
4333 : /* Pre-shift the (SIZE+1)-bit magic constant (2^SIZE + ML) by
4334 : (SIZE - POST_SHIFT) so that the quotient ends up in the high part
4335 : of the widened product. Since ML < 2^SIZE and POST_SHIFT >= 1, the
4336 : result is below 2^(2*SIZE) and thus fits in both WIDE_MODE and an
4337 : unsigned HOST_WIDE_INT (2*SIZE <= HOST_BITS_PER_WIDE_INT was
4338 : checked above). */
4339 1031 : unsigned HOST_WIDE_INT magic
4340 1031 : = ((HOST_WIDE_INT_1U << size) + ml) << (size - post_shift);
4341 :
4342 1031 : start_sequence ();
4343 1031 : rtx x_wide = convert_to_mode (wide_mode, op0, 1);
4344 1031 : rtx hi = expmed_mult_highpart (wide_mode, x_wide,
4345 1031 : gen_int_mode (magic, wide_mode),
4346 : NULL_RTX, 1, max_cost);
4347 1031 : rtx quotient = hi ? convert_to_mode (int_mode, hi, 1) : NULL_RTX;
4348 1031 : rtx_insn *insns = end_sequence ();
4349 :
4350 : /* Use the widened multiply only when it is no more expensive than
4351 : the classic sub/shift/add sequence. */
4352 1031 : unsigned classic_cost = mul_highpart_cost (speed, int_mode) + extra_cost;
4353 1031 : if (quotient == NULL_RTX || seq_cost (insns, speed) > classic_cost)
4354 73 : return NULL_RTX;
4355 :
4356 958 : emit_insn (insns);
4357 958 : return quotient;
4358 : }
4359 :
4360 : rtx
4361 248378 : expand_divmod (int rem_flag, enum tree_code code, machine_mode mode,
4362 : rtx op0, rtx op1, rtx target, int unsignedp,
4363 : enum optab_methods methods)
4364 : {
4365 248378 : machine_mode compute_mode;
4366 248378 : rtx tquotient;
4367 248378 : rtx quotient = 0, remainder = 0;
4368 248378 : rtx_insn *last;
4369 248378 : rtx_insn *insn;
4370 248378 : optab optab1, optab2;
4371 248378 : int op1_is_constant, op1_is_pow2 = 0;
4372 248378 : int max_cost, extra_cost;
4373 248378 : static HOST_WIDE_INT last_div_const = 0;
4374 248378 : bool speed = optimize_insn_for_speed_p ();
4375 :
4376 248378 : op1_is_constant = CONST_INT_P (op1);
4377 248378 : if (op1_is_constant)
4378 : {
4379 147897 : wide_int ext_op1 = rtx_mode_t (op1, mode);
4380 147897 : op1_is_pow2 = (wi::popcount (ext_op1) == 1
4381 295794 : || (! unsignedp
4382 177764 : && wi::popcount (wi::neg (ext_op1)) == 1));
4383 147897 : }
4384 :
4385 : /*
4386 : This is the structure of expand_divmod:
4387 :
4388 : First comes code to fix up the operands so we can perform the operations
4389 : correctly and efficiently.
4390 :
4391 : Second comes a switch statement with code specific for each rounding mode.
4392 : For some special operands this code emits all RTL for the desired
4393 : operation, for other cases, it generates only a quotient and stores it in
4394 : QUOTIENT. The case for trunc division/remainder might leave quotient = 0,
4395 : to indicate that it has not done anything.
4396 :
4397 : Last comes code that finishes the operation. If QUOTIENT is set and
4398 : REM_FLAG is set, the remainder is computed as OP0 - QUOTIENT * OP1. If
4399 : QUOTIENT is not set, it is computed using trunc rounding.
4400 :
4401 : We try to generate special code for division and remainder when OP1 is a
4402 : constant. If |OP1| = 2**n we can use shifts and some other fast
4403 : operations. For other values of OP1, we compute a carefully selected
4404 : fixed-point approximation m = 1/OP1, and generate code that multiplies OP0
4405 : by m.
4406 :
4407 : In all cases but EXACT_DIV_EXPR, this multiplication requires the upper
4408 : half of the product. Different strategies for generating the product are
4409 : implemented in expmed_mult_highpart.
4410 :
4411 : If what we actually want is the remainder, we generate that by another
4412 : by-constant multiplication and a subtraction. */
4413 :
4414 : /* We shouldn't be called with OP1 == const1_rtx, but some of the
4415 : code below will malfunction if we are, so check here and handle
4416 : the special case if so. */
4417 248378 : if (op1 == const1_rtx)
4418 0 : return rem_flag ? const0_rtx : op0;
4419 :
4420 : /* When dividing by -1, we could get an overflow.
4421 : negv_optab can handle overflows. */
4422 248378 : if (! unsignedp && op1 == constm1_rtx)
4423 : {
4424 0 : if (rem_flag)
4425 0 : return const0_rtx;
4426 0 : return expand_unop (mode, flag_trapv && GET_MODE_CLASS (mode) == MODE_INT
4427 0 : ? negv_optab : neg_optab, op0, target, 0);
4428 : }
4429 :
4430 248378 : if (target
4431 : /* Don't use the function value register as a target
4432 : since we have to read it as well as write it,
4433 : and function-inlining gets confused by this. */
4434 248378 : && ((REG_P (target) && REG_FUNCTION_VALUE_P (target))
4435 : /* Don't clobber an operand while doing a multi-step calculation. */
4436 100863 : || ((rem_flag || op1_is_constant)
4437 80823 : && (reg_mentioned_p (target, op0)
4438 78402 : || (MEM_P (op0) && MEM_P (target))))
4439 97718 : || reg_mentioned_p (target, op1)
4440 97639 : || (MEM_P (op1) && MEM_P (target))))
4441 : target = 0;
4442 :
4443 : /* Get the mode in which to perform this computation. Normally it will
4444 : be MODE, but sometimes we can't do the desired operation in MODE.
4445 : If so, pick a wider mode in which we can do the operation. Convert
4446 : to that mode at the start to avoid repeated conversions.
4447 :
4448 : First see what operations we need. These depend on the expression
4449 : we are evaluating. (We assume that divxx3 insns exist under the
4450 : same conditions that modxx3 insns and that these insns don't normally
4451 : fail. If these assumptions are not correct, we may generate less
4452 : efficient code in some cases.)
4453 :
4454 : Then see if we find a mode in which we can open-code that operation
4455 : (either a division, modulus, or shift). Finally, check for the smallest
4456 : mode for which we can do the operation with a library call. */
4457 :
4458 : /* We might want to refine this now that we have division-by-constant
4459 : optimization. Since expmed_mult_highpart tries so many variants, it is
4460 : not straightforward to generalize this. Maybe we should make an array
4461 : of possible modes in init_expmed? Save this for GCC 2.7. */
4462 :
4463 138641 : optab1 = (op1_is_pow2
4464 248378 : ? (unsignedp ? lshr_optab : ashr_optab)
4465 157057 : : (unsignedp ? udiv_optab : sdiv_optab));
4466 327830 : optab2 = (op1_is_pow2 ? optab1
4467 157057 : : (unsignedp ? udivmod_optab : sdivmod_optab));
4468 :
4469 248378 : if (methods == OPTAB_WIDEN || methods == OPTAB_LIB_WIDEN)
4470 : {
4471 259965 : FOR_EACH_MODE_FROM (compute_mode, mode)
4472 255802 : if (optab_handler (optab1, compute_mode) != CODE_FOR_nothing
4473 255802 : || optab_handler (optab2, compute_mode) != CODE_FOR_nothing)
4474 : break;
4475 :
4476 246670 : if (compute_mode == VOIDmode && methods == OPTAB_LIB_WIDEN)
4477 4163 : FOR_EACH_MODE_FROM (compute_mode, mode)
4478 4163 : if (optab_libfunc (optab1, compute_mode)
4479 4163 : || optab_libfunc (optab2, compute_mode))
4480 : break;
4481 : }
4482 : else
4483 : compute_mode = mode;
4484 :
4485 : /* If we still couldn't find a mode, use MODE, but expand_binop will
4486 : probably die. */
4487 5871 : if (compute_mode == VOIDmode)
4488 0 : compute_mode = mode;
4489 :
4490 248378 : if (target && GET_MODE (target) == compute_mode)
4491 : tquotient = target;
4492 : else
4493 150906 : tquotient = gen_reg_rtx (compute_mode);
4494 :
4495 : #if 0
4496 : /* It should be possible to restrict the precision to GET_MODE_BITSIZE
4497 : (mode), and thereby get better code when OP1 is a constant. Do that
4498 : later. It will require going over all usages of SIZE below. */
4499 : size = GET_MODE_BITSIZE (mode);
4500 : #endif
4501 :
4502 : /* Only deduct something for a REM if the last divide done was
4503 : for a different constant. Then set the constant of the last
4504 : divide. */
4505 248378 : max_cost = (unsignedp
4506 358115 : ? udiv_cost (speed, compute_mode)
4507 138641 : : sdiv_cost (speed, compute_mode));
4508 248378 : if (rem_flag && ! (last_div_const != 0 && op1_is_constant
4509 7723 : && INTVAL (op1) == last_div_const))
4510 60856 : max_cost -= (mul_cost (speed, compute_mode)
4511 60856 : + add_cost (speed, compute_mode));
4512 :
4513 248378 : last_div_const = ! rem_flag && op1_is_constant ? INTVAL (op1) : 0;
4514 :
4515 : /* Now convert to the best mode to use. */
4516 248378 : if (compute_mode != mode)
4517 : {
4518 0 : op0 = convert_modes (compute_mode, mode, op0, unsignedp);
4519 0 : op1 = convert_modes (compute_mode, mode, op1, unsignedp);
4520 :
4521 : /* convert_modes may have placed op1 into a register, so we
4522 : must recompute the following. */
4523 0 : op1_is_constant = CONST_INT_P (op1);
4524 0 : if (op1_is_constant)
4525 : {
4526 0 : wide_int ext_op1 = rtx_mode_t (op1, compute_mode);
4527 0 : op1_is_pow2 = (wi::popcount (ext_op1) == 1
4528 0 : || (! unsignedp
4529 0 : && wi::popcount (wi::neg (ext_op1)) == 1));
4530 0 : }
4531 : else
4532 : op1_is_pow2 = 0;
4533 : }
4534 :
4535 : /* If one of the operands is a volatile MEM, copy it into a register. */
4536 :
4537 248378 : if (MEM_P (op0) && MEM_VOLATILE_P (op0))
4538 0 : op0 = force_reg (compute_mode, op0);
4539 248378 : if (MEM_P (op1) && MEM_VOLATILE_P (op1))
4540 0 : op1 = force_reg (compute_mode, op1);
4541 :
4542 : /* If we need the remainder or if OP1 is constant, we need to
4543 : put OP0 in a register in case it has any queued subexpressions. */
4544 248378 : if (rem_flag || op1_is_constant)
4545 185950 : op0 = force_reg (compute_mode, op0);
4546 :
4547 248378 : last = get_last_insn ();
4548 :
4549 : /* Promote floor rounding to trunc rounding for unsigned operations. */
4550 248378 : if (unsignedp)
4551 : {
4552 109737 : if (code == FLOOR_DIV_EXPR)
4553 : code = TRUNC_DIV_EXPR;
4554 109680 : if (code == FLOOR_MOD_EXPR)
4555 156 : code = TRUNC_MOD_EXPR;
4556 109737 : if (code == EXACT_DIV_EXPR && op1_is_pow2)
4557 4835 : code = TRUNC_DIV_EXPR;
4558 : }
4559 :
4560 248378 : if (op1 != const0_rtx)
4561 247977 : switch (code)
4562 : {
4563 193855 : case TRUNC_MOD_EXPR:
4564 193855 : case TRUNC_DIV_EXPR:
4565 193855 : if (op1_is_constant)
4566 : {
4567 94487 : scalar_int_mode int_mode = as_a <scalar_int_mode> (compute_mode);
4568 94487 : int size = GET_MODE_BITSIZE (int_mode);
4569 94487 : if (unsignedp)
4570 : {
4571 57472 : unsigned HOST_WIDE_INT mh, ml;
4572 57472 : int pre_shift, post_shift;
4573 57472 : wide_int wd = rtx_mode_t (op1, int_mode);
4574 57472 : unsigned HOST_WIDE_INT d = wd.to_uhwi ();
4575 :
4576 57472 : if (wi::popcount (wd) == 1)
4577 : {
4578 32132 : pre_shift = floor_log2 (d);
4579 32132 : if (rem_flag)
4580 : {
4581 268 : unsigned HOST_WIDE_INT mask
4582 268 : = (HOST_WIDE_INT_1U << pre_shift) - 1;
4583 268 : remainder
4584 268 : = expand_binop (int_mode, and_optab, op0,
4585 268 : gen_int_mode (mask, int_mode),
4586 : remainder, 1, methods);
4587 268 : if (remainder)
4588 268 : return gen_lowpart (mode, remainder);
4589 : }
4590 31864 : quotient = expand_shift (RSHIFT_EXPR, int_mode, op0,
4591 31864 : pre_shift, tquotient, 1);
4592 : }
4593 25340 : else if (size <= HOST_BITS_PER_WIDE_INT)
4594 : {
4595 23743 : if (d >= (HOST_WIDE_INT_1U << (size - 1)))
4596 : {
4597 : /* Most significant bit of divisor is set; emit an scc
4598 : insn. */
4599 161 : quotient = emit_store_flag_force (tquotient, GEU, op0, op1,
4600 : int_mode, 1, 1);
4601 : }
4602 : else
4603 : {
4604 : /* Find a suitable multiplier and right shift count
4605 : instead of directly dividing by D. */
4606 23582 : mh = choose_multiplier (d, size, size,
4607 : &ml, &post_shift);
4608 :
4609 : /* If the suggested multiplier is more than SIZE bits,
4610 : we can do better for even divisors, using an
4611 : initial right shift. */
4612 23582 : if (mh != 0 && (d & 1) == 0)
4613 : {
4614 2306 : pre_shift = ctz_or_zero (d);
4615 2306 : mh = choose_multiplier (d >> pre_shift, size,
4616 : size - pre_shift,
4617 : &ml, &post_shift);
4618 2306 : gcc_assert (!mh);
4619 : }
4620 : else
4621 : pre_shift = 0;
4622 :
4623 2322 : if (mh != 0)
4624 : {
4625 2322 : rtx t1, t2, t3, t4;
4626 :
4627 2415 : if (post_shift - 1 >= BITS_PER_WORD)
4628 0 : goto fail1;
4629 :
4630 2322 : extra_cost
4631 2322 : = (shift_cost (speed, int_mode, post_shift - 1)
4632 2322 : + shift_cost (speed, int_mode, 1)
4633 2322 : + 2 * add_cost (speed, int_mode));
4634 :
4635 : /* Try a single widened multiply first; use it when
4636 : it is no more expensive. */
4637 2322 : quotient
4638 2322 : = expand_wide_mulh_udiv (int_mode, op0, ml, size,
4639 : post_shift, extra_cost,
4640 : max_cost, speed);
4641 2322 : if (quotient == NULL_RTX)
4642 : {
4643 1364 : t1 = expmed_mult_highpart
4644 1364 : (int_mode, op0, gen_int_mode (ml, int_mode),
4645 : NULL_RTX, 1, max_cost - extra_cost);
4646 1364 : if (t1 == 0)
4647 103 : goto fail1;
4648 1261 : t2 = force_operand (gen_rtx_MINUS (int_mode,
4649 : op0, t1),
4650 : NULL_RTX);
4651 1261 : t3 = expand_shift (RSHIFT_EXPR, int_mode,
4652 : t2, 1, NULL_RTX, 1);
4653 1261 : t4 = force_operand (gen_rtx_PLUS (int_mode,
4654 : t1, t3),
4655 : NULL_RTX);
4656 1261 : quotient = expand_shift
4657 1261 : (RSHIFT_EXPR, int_mode, t4,
4658 1261 : post_shift - 1, tquotient, 1);
4659 : }
4660 : }
4661 : else
4662 : {
4663 21260 : rtx t1, t2;
4664 :
4665 23053 : if (pre_shift >= BITS_PER_WORD
4666 21260 : || post_shift >= BITS_PER_WORD)
4667 3 : goto fail1;
4668 :
4669 21257 : t1 = expand_shift
4670 42514 : (RSHIFT_EXPR, int_mode, op0,
4671 21257 : pre_shift, NULL_RTX, 1);
4672 21257 : extra_cost
4673 21257 : = (shift_cost (speed, int_mode, pre_shift)
4674 21257 : + shift_cost (speed, int_mode, post_shift));
4675 21257 : t2 = expmed_mult_highpart
4676 21257 : (int_mode, t1,
4677 21257 : gen_int_mode (ml, int_mode),
4678 : NULL_RTX, 1, max_cost - extra_cost);
4679 21257 : if (t2 == 0)
4680 921 : goto fail1;
4681 20336 : quotient = expand_shift
4682 20336 : (RSHIFT_EXPR, int_mode, t2,
4683 20336 : post_shift, tquotient, 1);
4684 : }
4685 : }
4686 : }
4687 : else /* Too wide mode to use tricky code */
4688 : break;
4689 :
4690 54580 : insn = get_last_insn ();
4691 54580 : if (insn != last)
4692 54580 : set_dst_reg_note (insn, REG_EQUAL,
4693 : gen_rtx_UDIV (int_mode, op0, op1),
4694 : quotient);
4695 55875 : }
4696 : else /* TRUNC_DIV, signed */
4697 : {
4698 37015 : unsigned HOST_WIDE_INT ml;
4699 37015 : int post_shift;
4700 37015 : rtx mlr;
4701 37015 : HOST_WIDE_INT d = INTVAL (op1);
4702 37015 : unsigned HOST_WIDE_INT abs_d;
4703 :
4704 : /* Not prepared to handle division/remainder by
4705 : 0xffffffffffffffff8000000000000000 etc. */
4706 37015 : if (d == HOST_WIDE_INT_MIN && size > HOST_BITS_PER_WIDE_INT)
4707 : break;
4708 :
4709 : /* Since d might be INT_MIN, we have to cast to
4710 : unsigned HOST_WIDE_INT before negating to avoid
4711 : undefined signed overflow. */
4712 37015 : abs_d = (d >= 0
4713 37015 : ? (unsigned HOST_WIDE_INT) d
4714 : : - (unsigned HOST_WIDE_INT) d);
4715 :
4716 : /* n rem d = n rem -d */
4717 37015 : if (rem_flag && d < 0)
4718 : {
4719 141 : d = abs_d;
4720 141 : op1 = gen_int_mode (abs_d, int_mode);
4721 : }
4722 :
4723 37015 : if (d == 1)
4724 : quotient = op0;
4725 37015 : else if (d == -1)
4726 0 : quotient = expand_unop (int_mode, neg_optab, op0,
4727 : tquotient, 0);
4728 37015 : else if (size <= HOST_BITS_PER_WIDE_INT
4729 35614 : && abs_d == HOST_WIDE_INT_1U << (size - 1))
4730 : {
4731 : /* This case is not handled correctly below. */
4732 133 : quotient = emit_store_flag (tquotient, EQ, op0, op1,
4733 : int_mode, 1, 1);
4734 133 : if (quotient == 0)
4735 1318 : goto fail1;
4736 : }
4737 36882 : else if (EXACT_POWER_OF_2_OR_ZERO_P (d)
4738 12834 : && (size <= HOST_BITS_PER_WIDE_INT || d >= 0)
4739 2650 : && (rem_flag
4740 2650 : ? smod_pow2_cheap (speed, int_mode)
4741 10184 : : sdiv_pow2_cheap (speed, int_mode))
4742 : /* We assume that cheap metric is true if the
4743 : optab has an expander for this mode. */
4744 50830 : && ((optab_handler ((rem_flag ? smod_optab
4745 : : sdiv_optab),
4746 : int_mode)
4747 : != CODE_FOR_nothing)
4748 624 : || (optab_handler (sdivmod_optab, int_mode)
4749 : != CODE_FOR_nothing)))
4750 : ;
4751 36264 : else if (EXACT_POWER_OF_2_OR_ZERO_P (abs_d))
4752 : {
4753 12733 : if (rem_flag)
4754 : {
4755 2520 : remainder = expand_smod_pow2 (int_mode, op0, d);
4756 2520 : if (remainder)
4757 2520 : return gen_lowpart (mode, remainder);
4758 : }
4759 :
4760 10213 : if (sdiv_pow2_cheap (speed, int_mode)
4761 10213 : && ((optab_handler (sdiv_optab, int_mode)
4762 : != CODE_FOR_nothing)
4763 10 : || (optab_handler (sdivmod_optab, int_mode)
4764 : != CODE_FOR_nothing)))
4765 7 : quotient = expand_divmod (0, TRUNC_DIV_EXPR,
4766 : int_mode, op0,
4767 7 : gen_int_mode (abs_d,
4768 : int_mode),
4769 : NULL_RTX, 0);
4770 : else
4771 10206 : quotient = expand_sdiv_pow2 (int_mode, op0, abs_d);
4772 :
4773 : /* We have computed OP0 / abs(OP1). If OP1 is negative,
4774 : negate the quotient. */
4775 10213 : if (d < 0)
4776 : {
4777 517 : insn = get_last_insn ();
4778 517 : if (insn != last
4779 517 : && abs_d < (HOST_WIDE_INT_1U
4780 : << (HOST_BITS_PER_WIDE_INT - 1)))
4781 517 : set_dst_reg_note (insn, REG_EQUAL,
4782 517 : gen_rtx_DIV (int_mode, op0,
4783 : gen_int_mode
4784 : (abs_d,
4785 : int_mode)),
4786 : quotient);
4787 :
4788 517 : quotient = expand_unop (int_mode, neg_optab,
4789 : quotient, quotient, 0);
4790 : }
4791 : }
4792 23531 : else if (size <= HOST_BITS_PER_WIDE_INT)
4793 : {
4794 22193 : choose_multiplier (abs_d, size, size - 1,
4795 : &ml, &post_shift);
4796 22193 : if (ml < HOST_WIDE_INT_1U << (size - 1))
4797 : {
4798 16779 : rtx t1, t2, t3;
4799 :
4800 17905 : if (post_shift >= BITS_PER_WORD
4801 16779 : || size - 1 >= BITS_PER_WORD)
4802 245 : goto fail1;
4803 :
4804 16534 : extra_cost = (shift_cost (speed, int_mode, post_shift)
4805 16534 : + shift_cost (speed, int_mode, size - 1)
4806 16534 : + add_cost (speed, int_mode));
4807 16534 : t1 = expmed_mult_highpart
4808 16534 : (int_mode, op0, gen_int_mode (ml, int_mode),
4809 : NULL_RTX, 0, max_cost - extra_cost);
4810 16534 : if (t1 == 0)
4811 835 : goto fail1;
4812 15699 : t2 = expand_shift
4813 31398 : (RSHIFT_EXPR, int_mode, t1,
4814 15699 : post_shift, NULL_RTX, 0);
4815 15699 : t3 = expand_shift
4816 15699 : (RSHIFT_EXPR, int_mode, op0,
4817 15699 : size - 1, NULL_RTX, 0);
4818 15699 : if (d < 0)
4819 197 : quotient
4820 197 : = force_operand (gen_rtx_MINUS (int_mode, t3, t2),
4821 : tquotient);
4822 : else
4823 15502 : quotient
4824 15502 : = force_operand (gen_rtx_MINUS (int_mode, t2, t3),
4825 : tquotient);
4826 : }
4827 : else
4828 : {
4829 5414 : rtx t1, t2, t3, t4;
4830 :
4831 5764 : if (post_shift >= BITS_PER_WORD
4832 5409 : || size - 1 >= BITS_PER_WORD)
4833 27 : goto fail1;
4834 :
4835 5387 : ml |= HOST_WIDE_INT_M1U << (size - 1);
4836 5387 : mlr = gen_int_mode (ml, int_mode);
4837 5387 : extra_cost = (shift_cost (speed, int_mode, post_shift)
4838 5387 : + shift_cost (speed, int_mode, size - 1)
4839 5387 : + 2 * add_cost (speed, int_mode));
4840 5387 : t1 = expmed_mult_highpart (int_mode, op0, mlr,
4841 : NULL_RTX, 0,
4842 : max_cost - extra_cost);
4843 5387 : if (t1 == 0)
4844 211 : goto fail1;
4845 5176 : t2 = force_operand (gen_rtx_PLUS (int_mode, t1, op0),
4846 : NULL_RTX);
4847 5176 : t3 = expand_shift
4848 10352 : (RSHIFT_EXPR, int_mode, t2,
4849 5176 : post_shift, NULL_RTX, 0);
4850 5176 : t4 = expand_shift
4851 5176 : (RSHIFT_EXPR, int_mode, op0,
4852 5176 : size - 1, NULL_RTX, 0);
4853 5176 : if (d < 0)
4854 52 : quotient
4855 52 : = force_operand (gen_rtx_MINUS (int_mode, t4, t3),
4856 : tquotient);
4857 : else
4858 5124 : quotient
4859 5124 : = force_operand (gen_rtx_MINUS (int_mode, t3, t4),
4860 : tquotient);
4861 : }
4862 : }
4863 : else /* Too wide mode to use tricky code */
4864 : break;
4865 :
4866 31839 : insn = get_last_insn ();
4867 31839 : if (insn != last)
4868 31221 : set_dst_reg_note (insn, REG_EQUAL,
4869 : gen_rtx_DIV (int_mode, op0, op1),
4870 : quotient);
4871 : }
4872 : break;
4873 : }
4874 99368 : fail1:
4875 101713 : delete_insns_since (last);
4876 101713 : break;
4877 :
4878 1766 : case FLOOR_DIV_EXPR:
4879 1766 : case FLOOR_MOD_EXPR:
4880 : /* We will come here only for signed operations. */
4881 1766 : if (op1_is_constant && HWI_COMPUTABLE_MODE_P (compute_mode))
4882 : {
4883 972 : scalar_int_mode int_mode = as_a <scalar_int_mode> (compute_mode);
4884 972 : int size = GET_MODE_BITSIZE (int_mode);
4885 972 : unsigned HOST_WIDE_INT mh, ml;
4886 972 : int pre_shift, post_shift;
4887 972 : HOST_WIDE_INT d = INTVAL (op1);
4888 :
4889 972 : if (d > 0)
4890 : {
4891 : /* We could just as easily deal with negative constants here,
4892 : but it does not seem worth the trouble for GCC 2.6. */
4893 947 : if (EXACT_POWER_OF_2_OR_ZERO_P (d))
4894 : {
4895 646 : pre_shift = floor_log2 (d);
4896 646 : if (rem_flag)
4897 : {
4898 72 : unsigned HOST_WIDE_INT mask
4899 72 : = (HOST_WIDE_INT_1U << pre_shift) - 1;
4900 72 : remainder = expand_binop
4901 72 : (int_mode, and_optab, op0,
4902 72 : gen_int_mode (mask, int_mode),
4903 : remainder, 0, methods);
4904 72 : if (remainder)
4905 72 : return gen_lowpart (mode, remainder);
4906 : }
4907 574 : quotient = expand_shift
4908 574 : (RSHIFT_EXPR, int_mode, op0,
4909 574 : pre_shift, tquotient, 0);
4910 : }
4911 : else
4912 : {
4913 301 : rtx t1, t2, t3, t4;
4914 :
4915 301 : mh = choose_multiplier (d, size, size - 1,
4916 : &ml, &post_shift);
4917 301 : gcc_assert (!mh);
4918 :
4919 325 : if (post_shift < BITS_PER_WORD
4920 301 : && size - 1 < BITS_PER_WORD)
4921 : {
4922 299 : t1 = expand_shift
4923 299 : (RSHIFT_EXPR, int_mode, op0,
4924 299 : size - 1, NULL_RTX, 0);
4925 299 : t2 = expand_binop (int_mode, xor_optab, op0, t1,
4926 : NULL_RTX, 0, OPTAB_WIDEN);
4927 299 : extra_cost = (shift_cost (speed, int_mode, post_shift)
4928 299 : + shift_cost (speed, int_mode, size - 1)
4929 299 : + 2 * add_cost (speed, int_mode));
4930 299 : t3 = expmed_mult_highpart
4931 299 : (int_mode, t2, gen_int_mode (ml, int_mode),
4932 : NULL_RTX, 1, max_cost - extra_cost);
4933 299 : if (t3 != 0)
4934 : {
4935 268 : t4 = expand_shift
4936 536 : (RSHIFT_EXPR, int_mode, t3,
4937 268 : post_shift, NULL_RTX, 1);
4938 268 : quotient = expand_binop (int_mode, xor_optab,
4939 : t4, t1, tquotient, 0,
4940 : OPTAB_WIDEN);
4941 : }
4942 : }
4943 : }
4944 : }
4945 : else
4946 : {
4947 25 : rtx nsign, t1, t2, t3, t4;
4948 25 : t1 = force_operand (gen_rtx_PLUS (int_mode,
4949 : op0, constm1_rtx), NULL_RTX);
4950 25 : t2 = expand_binop (int_mode, ior_optab, op0, t1, NULL_RTX,
4951 : 0, OPTAB_WIDEN);
4952 50 : nsign = expand_shift (RSHIFT_EXPR, int_mode, t2,
4953 25 : size - 1, NULL_RTX, 0);
4954 25 : t3 = force_operand (gen_rtx_MINUS (int_mode, t1, nsign),
4955 : NULL_RTX);
4956 25 : t4 = expand_divmod (0, TRUNC_DIV_EXPR, int_mode, t3, op1,
4957 : NULL_RTX, 0);
4958 25 : if (t4)
4959 : {
4960 25 : rtx t5;
4961 25 : t5 = expand_unop (int_mode, one_cmpl_optab, nsign,
4962 : NULL_RTX, 0);
4963 25 : quotient = force_operand (gen_rtx_PLUS (int_mode, t4, t5),
4964 : tquotient);
4965 : }
4966 : }
4967 : }
4968 :
4969 900 : if (quotient != 0)
4970 : break;
4971 827 : delete_insns_since (last);
4972 :
4973 : /* Try using an instruction that produces both the quotient and
4974 : remainder, using truncation. We can easily compensate the quotient
4975 : or remainder to get floor rounding, once we have the remainder.
4976 : Notice that we compute also the final remainder value here,
4977 : and return the result right away. */
4978 827 : if (target == 0 || GET_MODE (target) != compute_mode)
4979 125 : target = gen_reg_rtx (compute_mode);
4980 :
4981 827 : if (rem_flag)
4982 : {
4983 329 : remainder
4984 329 : = REG_P (target) ? target : gen_reg_rtx (compute_mode);
4985 329 : quotient = gen_reg_rtx (compute_mode);
4986 : }
4987 : else
4988 : {
4989 498 : quotient
4990 498 : = REG_P (target) ? target : gen_reg_rtx (compute_mode);
4991 498 : remainder = gen_reg_rtx (compute_mode);
4992 : }
4993 :
4994 827 : if (expand_twoval_binop (sdivmod_optab, op0, op1,
4995 : quotient, remainder, 0))
4996 : {
4997 : /* This could be computed with a branch-less sequence.
4998 : Save that for later. */
4999 792 : rtx tem;
5000 792 : rtx_code_label *label = gen_label_rtx ();
5001 792 : do_cmp_and_jump (remainder, const0_rtx, EQ, compute_mode, label);
5002 792 : tem = expand_binop (compute_mode, xor_optab, op0, op1,
5003 : NULL_RTX, 0, OPTAB_WIDEN);
5004 792 : do_cmp_and_jump (tem, const0_rtx, GE, compute_mode, label);
5005 792 : expand_dec (quotient, const1_rtx);
5006 792 : expand_inc (remainder, op1);
5007 792 : emit_label (label);
5008 1281 : return gen_lowpart (mode, rem_flag ? remainder : quotient);
5009 : }
5010 :
5011 : /* No luck with division elimination or divmod. Have to do it
5012 : by conditionally adjusting op0 *and* the result. */
5013 35 : {
5014 35 : rtx_code_label *label1, *label2, *label3, *label4, *label5;
5015 35 : rtx adjusted_op0;
5016 35 : rtx tem;
5017 :
5018 35 : quotient = gen_reg_rtx (compute_mode);
5019 35 : adjusted_op0 = copy_to_mode_reg (compute_mode, op0);
5020 35 : label1 = gen_label_rtx ();
5021 35 : label2 = gen_label_rtx ();
5022 35 : label3 = gen_label_rtx ();
5023 35 : label4 = gen_label_rtx ();
5024 35 : label5 = gen_label_rtx ();
5025 35 : do_cmp_and_jump (op1, const0_rtx, LT, compute_mode, label2);
5026 35 : do_cmp_and_jump (adjusted_op0, const0_rtx, LT, compute_mode, label1);
5027 35 : tem = expand_binop (compute_mode, sdiv_optab, adjusted_op0, op1,
5028 : quotient, 0, methods);
5029 35 : if (tem != quotient)
5030 35 : emit_move_insn (quotient, tem);
5031 35 : emit_jump_insn (targetm.gen_jump (label5));
5032 35 : emit_barrier ();
5033 35 : emit_label (label1);
5034 35 : expand_inc (adjusted_op0, const1_rtx);
5035 35 : emit_jump_insn (targetm.gen_jump (label4));
5036 35 : emit_barrier ();
5037 35 : emit_label (label2);
5038 35 : do_cmp_and_jump (adjusted_op0, const0_rtx, GT, compute_mode, label3);
5039 35 : tem = expand_binop (compute_mode, sdiv_optab, adjusted_op0, op1,
5040 : quotient, 0, methods);
5041 35 : if (tem != quotient)
5042 35 : emit_move_insn (quotient, tem);
5043 35 : emit_jump_insn (targetm.gen_jump (label5));
5044 35 : emit_barrier ();
5045 35 : emit_label (label3);
5046 35 : expand_dec (adjusted_op0, const1_rtx);
5047 35 : emit_label (label4);
5048 35 : tem = expand_binop (compute_mode, sdiv_optab, adjusted_op0, op1,
5049 : quotient, 0, methods);
5050 35 : if (tem != quotient)
5051 35 : emit_move_insn (quotient, tem);
5052 35 : expand_dec (quotient, const1_rtx);
5053 35 : emit_label (label5);
5054 : }
5055 35 : break;
5056 :
5057 383 : case CEIL_DIV_EXPR:
5058 383 : case CEIL_MOD_EXPR:
5059 383 : if (unsignedp)
5060 : {
5061 0 : if (op1_is_constant
5062 0 : && EXACT_POWER_OF_2_OR_ZERO_P (INTVAL (op1))
5063 0 : && (HWI_COMPUTABLE_MODE_P (compute_mode)
5064 0 : || INTVAL (op1) >= 0))
5065 : {
5066 0 : scalar_int_mode int_mode
5067 0 : = as_a <scalar_int_mode> (compute_mode);
5068 0 : rtx t1, t2, t3;
5069 0 : unsigned HOST_WIDE_INT d = INTVAL (op1);
5070 0 : t1 = expand_shift (RSHIFT_EXPR, int_mode, op0,
5071 0 : floor_log2 (d), tquotient, 1);
5072 0 : t2 = expand_binop (int_mode, and_optab, op0,
5073 0 : gen_int_mode (d - 1, int_mode),
5074 : NULL_RTX, 1, methods);
5075 0 : t3 = gen_reg_rtx (int_mode);
5076 0 : t3 = emit_store_flag (t3, NE, t2, const0_rtx, int_mode, 1, 1);
5077 0 : if (t3 == 0)
5078 : {
5079 0 : rtx_code_label *lab;
5080 0 : lab = gen_label_rtx ();
5081 0 : do_cmp_and_jump (t2, const0_rtx, EQ, int_mode, lab);
5082 0 : expand_inc (t1, const1_rtx);
5083 0 : emit_label (lab);
5084 0 : quotient = t1;
5085 : }
5086 : else
5087 0 : quotient = force_operand (gen_rtx_PLUS (int_mode, t1, t3),
5088 : tquotient);
5089 : break;
5090 : }
5091 :
5092 : /* Try using an instruction that produces both the quotient and
5093 : remainder, using truncation. We can easily compensate the
5094 : quotient or remainder to get ceiling rounding, once we have the
5095 : remainder. Notice that we compute also the final remainder
5096 : value here, and return the result right away. */
5097 0 : if (target == 0 || GET_MODE (target) != compute_mode)
5098 0 : target = gen_reg_rtx (compute_mode);
5099 :
5100 0 : if (rem_flag)
5101 : {
5102 0 : remainder = (REG_P (target)
5103 0 : ? target : gen_reg_rtx (compute_mode));
5104 0 : quotient = gen_reg_rtx (compute_mode);
5105 : }
5106 : else
5107 : {
5108 0 : quotient = (REG_P (target)
5109 0 : ? target : gen_reg_rtx (compute_mode));
5110 0 : remainder = gen_reg_rtx (compute_mode);
5111 : }
5112 :
5113 0 : if (expand_twoval_binop (udivmod_optab, op0, op1, quotient,
5114 : remainder, 1))
5115 : {
5116 : /* This could be computed with a branch-less sequence.
5117 : Save that for later. */
5118 0 : rtx_code_label *label = gen_label_rtx ();
5119 0 : do_cmp_and_jump (remainder, const0_rtx, EQ,
5120 : compute_mode, label);
5121 0 : expand_inc (quotient, const1_rtx);
5122 0 : expand_dec (remainder, op1);
5123 0 : emit_label (label);
5124 0 : return gen_lowpart (mode, rem_flag ? remainder : quotient);
5125 : }
5126 :
5127 : /* No luck with division elimination or divmod. Have to do it
5128 : by conditionally adjusting op0 *and* the result. */
5129 0 : {
5130 0 : rtx_code_label *label1, *label2;
5131 0 : rtx adjusted_op0, tem;
5132 :
5133 0 : quotient = gen_reg_rtx (compute_mode);
5134 0 : adjusted_op0 = copy_to_mode_reg (compute_mode, op0);
5135 0 : label1 = gen_label_rtx ();
5136 0 : label2 = gen_label_rtx ();
5137 0 : do_cmp_and_jump (adjusted_op0, const0_rtx, NE,
5138 : compute_mode, label1);
5139 0 : emit_move_insn (quotient, const0_rtx);
5140 0 : emit_jump_insn (targetm.gen_jump (label2));
5141 0 : emit_barrier ();
5142 0 : emit_label (label1);
5143 0 : expand_dec (adjusted_op0, const1_rtx);
5144 0 : tem = expand_binop (compute_mode, udiv_optab, adjusted_op0, op1,
5145 : quotient, 1, methods);
5146 0 : if (tem != quotient)
5147 0 : emit_move_insn (quotient, tem);
5148 0 : expand_inc (quotient, const1_rtx);
5149 0 : emit_label (label2);
5150 : }
5151 : }
5152 : else /* signed */
5153 : {
5154 383 : if (op1_is_constant && EXACT_POWER_OF_2_OR_ZERO_P (INTVAL (op1))
5155 27 : && INTVAL (op1) >= 0)
5156 : {
5157 : /* This is extremely similar to the code for the unsigned case
5158 : above. For 2.7 we should merge these variants, but for
5159 : 2.6.1 I don't want to touch the code for unsigned since that
5160 : get used in C. The signed case will only be used by other
5161 : languages (Ada). */
5162 :
5163 27 : rtx t1, t2, t3;
5164 27 : unsigned HOST_WIDE_INT d = INTVAL (op1);
5165 54 : t1 = expand_shift (RSHIFT_EXPR, compute_mode, op0,
5166 27 : floor_log2 (d), tquotient, 0);
5167 27 : t2 = expand_binop (compute_mode, and_optab, op0,
5168 27 : gen_int_mode (d - 1, compute_mode),
5169 : NULL_RTX, 1, methods);
5170 27 : t3 = gen_reg_rtx (compute_mode);
5171 27 : t3 = emit_store_flag (t3, NE, t2, const0_rtx,
5172 : compute_mode, 1, 1);
5173 27 : if (t3 == 0)
5174 : {
5175 0 : rtx_code_label *lab;
5176 0 : lab = gen_label_rtx ();
5177 0 : do_cmp_and_jump (t2, const0_rtx, EQ, compute_mode, lab);
5178 0 : expand_inc (t1, const1_rtx);
5179 0 : emit_label (lab);
5180 0 : quotient = t1;
5181 : }
5182 : else
5183 27 : quotient = force_operand (gen_rtx_PLUS (compute_mode,
5184 : t1, t3),
5185 : tquotient);
5186 : break;
5187 : }
5188 :
5189 : /* Try using an instruction that produces both the quotient and
5190 : remainder, using truncation. We can easily compensate the
5191 : quotient or remainder to get ceiling rounding, once we have the
5192 : remainder. Notice that we compute also the final remainder
5193 : value here, and return the result right away. */
5194 356 : if (target == 0 || GET_MODE (target) != compute_mode)
5195 15 : target = gen_reg_rtx (compute_mode);
5196 356 : if (rem_flag)
5197 : {
5198 149 : remainder= (REG_P (target)
5199 149 : ? target : gen_reg_rtx (compute_mode));
5200 149 : quotient = gen_reg_rtx (compute_mode);
5201 : }
5202 : else
5203 : {
5204 207 : quotient = (REG_P (target)
5205 207 : ? target : gen_reg_rtx (compute_mode));
5206 207 : remainder = gen_reg_rtx (compute_mode);
5207 : }
5208 :
5209 356 : if (expand_twoval_binop (sdivmod_optab, op0, op1, quotient,
5210 : remainder, 0))
5211 : {
5212 : /* This could be computed with a branch-less sequence.
5213 : Save that for later. */
5214 356 : rtx tem;
5215 356 : rtx_code_label *label = gen_label_rtx ();
5216 356 : do_cmp_and_jump (remainder, const0_rtx, EQ,
5217 : compute_mode, label);
5218 356 : tem = expand_binop (compute_mode, xor_optab, op0, op1,
5219 : NULL_RTX, 0, OPTAB_WIDEN);
5220 356 : do_cmp_and_jump (tem, const0_rtx, LT, compute_mode, label);
5221 356 : expand_inc (quotient, const1_rtx);
5222 356 : expand_dec (remainder, op1);
5223 356 : emit_label (label);
5224 563 : return gen_lowpart (mode, rem_flag ? remainder : quotient);
5225 : }
5226 :
5227 : /* No luck with division elimination or divmod. Have to do it
5228 : by conditionally adjusting op0 *and* the result. */
5229 0 : {
5230 0 : rtx_code_label *label1, *label2, *label3, *label4, *label5;
5231 0 : rtx adjusted_op0;
5232 0 : rtx tem;
5233 :
5234 0 : quotient = gen_reg_rtx (compute_mode);
5235 0 : adjusted_op0 = copy_to_mode_reg (compute_mode, op0);
5236 0 : label1 = gen_label_rtx ();
5237 0 : label2 = gen_label_rtx ();
5238 0 : label3 = gen_label_rtx ();
5239 0 : label4 = gen_label_rtx ();
5240 0 : label5 = gen_label_rtx ();
5241 0 : do_cmp_and_jump (op1, const0_rtx, LT, compute_mode, label2);
5242 0 : do_cmp_and_jump (adjusted_op0, const0_rtx, GT,
5243 : compute_mode, label1);
5244 0 : tem = expand_binop (compute_mode, sdiv_optab, adjusted_op0, op1,
5245 : quotient, 0, methods);
5246 0 : if (tem != quotient)
5247 0 : emit_move_insn (quotient, tem);
5248 0 : emit_jump_insn (targetm.gen_jump (label5));
5249 0 : emit_barrier ();
5250 0 : emit_label (label1);
5251 0 : expand_dec (adjusted_op0, const1_rtx);
5252 0 : emit_jump_insn (targetm.gen_jump (label4));
5253 0 : emit_barrier ();
5254 0 : emit_label (label2);
5255 0 : do_cmp_and_jump (adjusted_op0, const0_rtx, LT,
5256 : compute_mode, label3);
5257 0 : tem = expand_binop (compute_mode, sdiv_optab, adjusted_op0, op1,
5258 : quotient, 0, methods);
5259 0 : if (tem != quotient)
5260 0 : emit_move_insn (quotient, tem);
5261 0 : emit_jump_insn (targetm.gen_jump (label5));
5262 0 : emit_barrier ();
5263 0 : emit_label (label3);
5264 0 : expand_inc (adjusted_op0, const1_rtx);
5265 0 : emit_label (label4);
5266 0 : tem = expand_binop (compute_mode, sdiv_optab, adjusted_op0, op1,
5267 : quotient, 0, methods);
5268 0 : if (tem != quotient)
5269 0 : emit_move_insn (quotient, tem);
5270 0 : expand_inc (quotient, const1_rtx);
5271 0 : emit_label (label5);
5272 : }
5273 : }
5274 : break;
5275 :
5276 51971 : case EXACT_DIV_EXPR:
5277 51971 : if (op1_is_constant && HWI_COMPUTABLE_MODE_P (compute_mode))
5278 : {
5279 51963 : scalar_int_mode int_mode = as_a <scalar_int_mode> (compute_mode);
5280 51963 : int size = GET_MODE_BITSIZE (int_mode);
5281 51963 : HOST_WIDE_INT d = INTVAL (op1);
5282 51963 : unsigned HOST_WIDE_INT ml;
5283 51963 : int pre_shift;
5284 51963 : rtx t1;
5285 :
5286 51963 : pre_shift = ctz_or_zero (d);
5287 51963 : ml = invert_mod2n (d >> pre_shift, size);
5288 51963 : t1 = expand_shift (RSHIFT_EXPR, int_mode, op0,
5289 51963 : pre_shift, NULL_RTX, unsignedp);
5290 51963 : quotient = expand_mult (int_mode, t1, gen_int_mode (ml, int_mode),
5291 : NULL_RTX, 1);
5292 :
5293 51963 : insn = get_last_insn ();
5294 103926 : set_dst_reg_note (insn, REG_EQUAL,
5295 : gen_rtx_fmt_ee (unsignedp ? UDIV : DIV,
5296 : int_mode, op0, op1),
5297 : quotient);
5298 : }
5299 : break;
5300 :
5301 2 : case ROUND_DIV_EXPR:
5302 2 : case ROUND_MOD_EXPR:
5303 2 : if (unsignedp)
5304 : {
5305 0 : scalar_int_mode int_mode = as_a <scalar_int_mode> (compute_mode);
5306 0 : rtx tem;
5307 0 : rtx_code_label *label;
5308 0 : label = gen_label_rtx ();
5309 0 : quotient = gen_reg_rtx (int_mode);
5310 0 : remainder = gen_reg_rtx (int_mode);
5311 0 : if (expand_twoval_binop (udivmod_optab, op0, op1, quotient, remainder, 1) == 0)
5312 : {
5313 0 : rtx tem;
5314 0 : quotient = expand_binop (int_mode, udiv_optab, op0, op1,
5315 : quotient, 1, methods);
5316 0 : tem = expand_mult (int_mode, quotient, op1, NULL_RTX, 1);
5317 0 : remainder = expand_binop (int_mode, sub_optab, op0, tem,
5318 : remainder, 1, methods);
5319 : }
5320 0 : tem = plus_constant (int_mode, op1, -1);
5321 0 : tem = expand_shift (RSHIFT_EXPR, int_mode, tem, 1, NULL_RTX, 1);
5322 0 : do_cmp_and_jump (remainder, tem, LEU, int_mode, label);
5323 0 : expand_inc (quotient, const1_rtx);
5324 0 : expand_dec (remainder, op1);
5325 0 : emit_label (label);
5326 : }
5327 : else
5328 : {
5329 2 : scalar_int_mode int_mode = as_a <scalar_int_mode> (compute_mode);
5330 2 : int size = GET_MODE_BITSIZE (int_mode);
5331 2 : rtx abs_rem, abs_op1, tem, mask;
5332 2 : rtx_code_label *label;
5333 2 : label = gen_label_rtx ();
5334 2 : quotient = gen_reg_rtx (int_mode);
5335 2 : remainder = gen_reg_rtx (int_mode);
5336 2 : if (expand_twoval_binop (sdivmod_optab, op0, op1, quotient, remainder, 0) == 0)
5337 : {
5338 0 : rtx tem;
5339 0 : quotient = expand_binop (int_mode, sdiv_optab, op0, op1,
5340 : quotient, 0, methods);
5341 0 : tem = expand_mult (int_mode, quotient, op1, NULL_RTX, 0);
5342 0 : remainder = expand_binop (int_mode, sub_optab, op0, tem,
5343 : remainder, 0, methods);
5344 : }
5345 2 : abs_rem = expand_abs (int_mode, remainder, NULL_RTX, 1, 0);
5346 2 : abs_op1 = expand_abs (int_mode, op1, NULL_RTX, 1, 0);
5347 2 : tem = expand_shift (LSHIFT_EXPR, int_mode, abs_rem,
5348 : 1, NULL_RTX, 1);
5349 2 : do_cmp_and_jump (tem, abs_op1, LTU, int_mode, label);
5350 2 : tem = expand_binop (int_mode, xor_optab, op0, op1,
5351 : NULL_RTX, 0, OPTAB_WIDEN);
5352 4 : mask = expand_shift (RSHIFT_EXPR, int_mode, tem,
5353 2 : size - 1, NULL_RTX, 0);
5354 2 : tem = expand_binop (int_mode, xor_optab, mask, const1_rtx,
5355 : NULL_RTX, 0, OPTAB_WIDEN);
5356 2 : tem = expand_binop (int_mode, sub_optab, tem, mask,
5357 : NULL_RTX, 0, OPTAB_WIDEN);
5358 2 : expand_inc (quotient, tem);
5359 2 : tem = expand_binop (int_mode, xor_optab, mask, op1,
5360 : NULL_RTX, 0, OPTAB_WIDEN);
5361 2 : tem = expand_binop (int_mode, sub_optab, tem, mask,
5362 : NULL_RTX, 0, OPTAB_WIDEN);
5363 2 : expand_dec (remainder, tem);
5364 2 : emit_label (label);
5365 : }
5366 3 : return gen_lowpart (mode, rem_flag ? remainder : quotient);
5367 :
5368 0 : default:
5369 0 : gcc_unreachable ();
5370 : }
5371 :
5372 243092 : if (quotient == 0)
5373 : {
5374 105675 : if (target && GET_MODE (target) != compute_mode)
5375 63486 : target = 0;
5376 :
5377 105675 : if (rem_flag)
5378 : {
5379 : /* Try to produce the remainder without producing the quotient.
5380 : If we seem to have a divmod pattern that does not require widening,
5381 : don't try widening here. We should really have a WIDEN argument
5382 : to expand_twoval_binop, since what we'd really like to do here is
5383 : 1) try a mod insn in compute_mode
5384 : 2) try a divmod insn in compute_mode
5385 : 3) try a div insn in compute_mode and multiply-subtract to get
5386 : remainder
5387 : 4) try the same things with widening allowed. */
5388 40189 : remainder
5389 41660 : = sign_expand_binop (compute_mode, umod_optab, smod_optab,
5390 : op0, op1, target,
5391 : unsignedp,
5392 40189 : ((optab_handler (optab2, compute_mode)
5393 : != CODE_FOR_nothing)
5394 : ? OPTAB_DIRECT : OPTAB_WIDEN));
5395 40189 : if (remainder == 0)
5396 : {
5397 : /* No luck there. Can we do remainder and divide at once
5398 : without a library call? */
5399 39982 : remainder = gen_reg_rtx (compute_mode);
5400 63089 : if (! expand_twoval_binop ((unsignedp
5401 : ? udivmod_optab
5402 : : sdivmod_optab),
5403 : op0, op1,
5404 : NULL_RTX, remainder, unsignedp))
5405 : remainder = 0;
5406 : }
5407 :
5408 38718 : if (remainder)
5409 38925 : return gen_lowpart (mode, remainder);
5410 : }
5411 :
5412 : /* Produce the quotient. Try a quotient insn, but not a library call.
5413 : If we have a divmod in this mode, use it in preference to widening
5414 : the div (for this test we assume it will not fail). Note that optab2
5415 : is set to the one of the two optabs that the call below will use. */
5416 66750 : quotient
5417 71937 : = sign_expand_binop (compute_mode, udiv_optab, sdiv_optab,
5418 : op0, op1, rem_flag ? NULL_RTX : target,
5419 : unsignedp,
5420 66750 : ((optab_handler (optab2, compute_mode)
5421 : != CODE_FOR_nothing)
5422 : ? OPTAB_DIRECT : OPTAB_WIDEN));
5423 :
5424 66750 : if (quotient == 0)
5425 : {
5426 : /* No luck there. Try a quotient-and-remainder insn,
5427 : keeping the quotient alone. */
5428 66360 : quotient = gen_reg_rtx (compute_mode);
5429 96236 : if (! expand_twoval_binop (unsignedp ? udivmod_optab : sdivmod_optab,
5430 : op0, op1,
5431 : quotient, NULL_RTX, unsignedp))
5432 : {
5433 3534 : quotient = 0;
5434 3534 : if (! rem_flag)
5435 : /* Still no luck. If we are not computing the remainder,
5436 : use a library call for the quotient. */
5437 2292 : quotient = sign_expand_binop (compute_mode,
5438 : udiv_optab, sdiv_optab,
5439 : op0, op1, target,
5440 : unsignedp, methods);
5441 : }
5442 : }
5443 : }
5444 :
5445 204201 : if (rem_flag)
5446 : {
5447 21930 : if (target && GET_MODE (target) != compute_mode)
5448 14601 : target = 0;
5449 :
5450 21930 : if (quotient == 0)
5451 : {
5452 : /* No divide instruction either. Use library for remainder. */
5453 1242 : remainder = sign_expand_binop (compute_mode, umod_optab, smod_optab,
5454 : op0, op1, target,
5455 : unsignedp, methods);
5456 : /* No remainder function. Try a quotient-and-remainder
5457 : function, keeping the remainder. */
5458 1242 : if (!remainder
5459 0 : && (methods == OPTAB_LIB || methods == OPTAB_LIB_WIDEN))
5460 : {
5461 0 : remainder = gen_reg_rtx (compute_mode);
5462 0 : if (!expand_twoval_binop_libfunc
5463 0 : (unsignedp ? udivmod_optab : sdivmod_optab,
5464 : op0, op1,
5465 : NULL_RTX, remainder,
5466 : unsignedp ? UMOD : MOD))
5467 0 : remainder = NULL_RTX;
5468 : }
5469 : }
5470 : else
5471 : {
5472 : /* We divided. Now finish doing X - Y * (X / Y). */
5473 20688 : remainder = expand_mult (compute_mode, quotient, op1,
5474 : NULL_RTX, unsignedp);
5475 20688 : remainder = expand_binop (compute_mode, sub_optab, op0,
5476 : remainder, target, unsignedp,
5477 : methods);
5478 : }
5479 : }
5480 :
5481 205443 : if (methods != OPTAB_LIB_WIDEN
5482 1708 : && (rem_flag ? remainder : quotient) == NULL_RTX)
5483 : return NULL_RTX;
5484 :
5485 388956 : return gen_lowpart (mode, rem_flag ? remainder : quotient);
5486 : }
5487 :
5488 : /* Return a tree node with data type TYPE, describing the value of X.
5489 : Usually this is an VAR_DECL, if there is no obvious better choice.
5490 : X may be an expression, however we only support those expressions
5491 : generated by loop.c. */
5492 :
5493 : tree
5494 684431 : make_tree (tree type, rtx x)
5495 : {
5496 684431 : tree t;
5497 :
5498 684431 : switch (GET_CODE (x))
5499 : {
5500 22763 : case CONST_INT:
5501 22763 : case CONST_WIDE_INT:
5502 22763 : t = wide_int_to_tree (type, rtx_mode_t (x, TYPE_MODE (type)));
5503 22763 : return t;
5504 :
5505 0 : case CONST_POLY_INT:
5506 0 : return wide_int_to_tree (type, const_poly_int_value (x));
5507 :
5508 0 : case CONST_DOUBLE:
5509 0 : STATIC_ASSERT (HOST_BITS_PER_WIDE_INT * 2 <= MAX_BITSIZE_MODE_ANY_INT);
5510 0 : if (TARGET_SUPPORTS_WIDE_INT == 0 && GET_MODE (x) == VOIDmode)
5511 : t = wide_int_to_tree (type,
5512 : wide_int::from_array (&CONST_DOUBLE_LOW (x), 2,
5513 : HOST_BITS_PER_WIDE_INT * 2));
5514 : else
5515 0 : t = build_real (type, *CONST_DOUBLE_REAL_VALUE (x));
5516 :
5517 0 : return t;
5518 :
5519 0 : case CONST_VECTOR:
5520 0 : {
5521 0 : unsigned int npatterns = CONST_VECTOR_NPATTERNS (x);
5522 0 : unsigned int nelts_per_pattern = CONST_VECTOR_NELTS_PER_PATTERN (x);
5523 0 : tree itype = TREE_TYPE (type);
5524 :
5525 : /* Build a tree with vector elements. */
5526 0 : tree_vector_builder elts (type, npatterns, nelts_per_pattern);
5527 0 : unsigned int count = elts.encoded_nelts ();
5528 0 : for (unsigned int i = 0; i < count; ++i)
5529 : {
5530 0 : rtx elt = CONST_VECTOR_ELT (x, i);
5531 0 : elts.quick_push (make_tree (itype, elt));
5532 : }
5533 :
5534 0 : return elts.build ();
5535 0 : }
5536 :
5537 0 : case PLUS:
5538 0 : return fold_build2 (PLUS_EXPR, type, make_tree (type, XEXP (x, 0)),
5539 : make_tree (type, XEXP (x, 1)));
5540 :
5541 0 : case MINUS:
5542 0 : return fold_build2 (MINUS_EXPR, type, make_tree (type, XEXP (x, 0)),
5543 : make_tree (type, XEXP (x, 1)));
5544 :
5545 0 : case NEG:
5546 0 : return fold_build1 (NEGATE_EXPR, type, make_tree (type, XEXP (x, 0)));
5547 :
5548 0 : case MULT:
5549 0 : return fold_build2 (MULT_EXPR, type, make_tree (type, XEXP (x, 0)),
5550 : make_tree (type, XEXP (x, 1)));
5551 :
5552 0 : case ASHIFT:
5553 0 : return fold_build2 (LSHIFT_EXPR, type, make_tree (type, XEXP (x, 0)),
5554 : make_tree (type, XEXP (x, 1)));
5555 :
5556 0 : case LSHIFTRT:
5557 0 : t = unsigned_type_for (type);
5558 0 : return fold_convert (type, build2 (RSHIFT_EXPR, t,
5559 : make_tree (t, XEXP (x, 0)),
5560 : make_tree (type, XEXP (x, 1))));
5561 :
5562 0 : case ASHIFTRT:
5563 0 : t = signed_type_for (type);
5564 0 : return fold_convert (type, build2 (RSHIFT_EXPR, t,
5565 : make_tree (t, XEXP (x, 0)),
5566 : make_tree (type, XEXP (x, 1))));
5567 :
5568 0 : case DIV:
5569 0 : if (TREE_CODE (type) != REAL_TYPE)
5570 0 : t = signed_type_for (type);
5571 : else
5572 : t = type;
5573 :
5574 0 : return fold_convert (type, build2 (TRUNC_DIV_EXPR, t,
5575 : make_tree (t, XEXP (x, 0)),
5576 : make_tree (t, XEXP (x, 1))));
5577 0 : case UDIV:
5578 0 : t = unsigned_type_for (type);
5579 0 : return fold_convert (type, build2 (TRUNC_DIV_EXPR, t,
5580 : make_tree (t, XEXP (x, 0)),
5581 : make_tree (t, XEXP (x, 1))));
5582 :
5583 0 : case SIGN_EXTEND:
5584 0 : case ZERO_EXTEND:
5585 0 : t = lang_hooks.types.type_for_mode (GET_MODE (XEXP (x, 0)),
5586 : GET_CODE (x) == ZERO_EXTEND);
5587 0 : return fold_convert (type, make_tree (t, XEXP (x, 0)));
5588 :
5589 0 : case CONST:
5590 0 : return make_tree (type, XEXP (x, 0));
5591 :
5592 0 : case SYMBOL_REF:
5593 0 : t = SYMBOL_REF_DECL (x);
5594 0 : if (t)
5595 0 : return fold_convert (type, build_fold_addr_expr (t));
5596 : /* fall through. */
5597 :
5598 661668 : default:
5599 661668 : t = build_decl (RTL_LOCATION (x), VAR_DECL, NULL_TREE, type);
5600 :
5601 : /* If TYPE is a POINTER_TYPE, we might need to convert X from
5602 : address mode to pointer mode. */
5603 661668 : if (POINTER_TYPE_P (type))
5604 821908 : x = convert_memory_address_addr_space
5605 410954 : (SCALAR_INT_TYPE_MODE (type), x, TYPE_ADDR_SPACE (TREE_TYPE (type)));
5606 :
5607 : /* Note that we do *not* use SET_DECL_RTL here, because we do not
5608 : want set_decl_rtl to go adjusting REG_ATTRS for this temporary. */
5609 661668 : t->decl_with_rtl.rtl = x;
5610 :
5611 661668 : return t;
5612 : }
5613 : }
5614 :
5615 : /* Compute the logical-and of OP0 and OP1, storing it in TARGET
5616 : and returning TARGET.
5617 :
5618 : If TARGET is 0, a pseudo-register or constant is returned. */
5619 :
5620 : rtx
5621 73171 : expand_and (machine_mode mode, rtx op0, rtx op1, rtx target)
5622 : {
5623 73171 : rtx tem = 0;
5624 :
5625 73171 : if (GET_MODE (op0) == VOIDmode && GET_MODE (op1) == VOIDmode)
5626 80 : tem = simplify_binary_operation (AND, mode, op0, op1);
5627 80 : if (tem == 0)
5628 73091 : tem = expand_binop (mode, and_optab, op0, op1, target, 0, OPTAB_LIB_WIDEN);
5629 :
5630 73171 : if (target == 0)
5631 : target = tem;
5632 44564 : else if (tem != target)
5633 13 : emit_move_insn (target, tem);
5634 73171 : return target;
5635 : }
5636 :
5637 : /* Helper function for emit_store_flag. */
5638 : rtx
5639 756150 : emit_cstore (rtx target, enum insn_code icode, enum rtx_code code,
5640 : machine_mode mode, machine_mode compare_mode,
5641 : int unsignedp, rtx x, rtx y, int normalizep,
5642 : machine_mode target_mode)
5643 : {
5644 756150 : class expand_operand ops[4];
5645 756150 : rtx op0, comparison, subtarget;
5646 756150 : rtx_insn *last;
5647 756150 : scalar_int_mode result_mode = targetm.cstore_mode (icode);
5648 756150 : scalar_int_mode int_target_mode;
5649 :
5650 756150 : last = get_last_insn ();
5651 756150 : x = prepare_operand (icode, x, 2, mode, compare_mode, unsignedp);
5652 756150 : y = prepare_operand (icode, y, 3, mode, compare_mode, unsignedp);
5653 756150 : if (!x || !y)
5654 : {
5655 284 : delete_insns_since (last);
5656 284 : return NULL_RTX;
5657 : }
5658 :
5659 755866 : if (target_mode == VOIDmode)
5660 : int_target_mode = result_mode;
5661 : else
5662 755850 : int_target_mode = as_a <scalar_int_mode> (target_mode);
5663 755866 : if (!target)
5664 67381 : target = gen_reg_rtx (int_target_mode);
5665 :
5666 755866 : comparison = gen_rtx_fmt_ee (code, result_mode, x, y);
5667 :
5668 755866 : create_output_operand (&ops[0], optimize ? NULL_RTX : target, result_mode);
5669 755866 : create_fixed_operand (&ops[1], comparison);
5670 755866 : create_fixed_operand (&ops[2], x);
5671 755866 : create_fixed_operand (&ops[3], y);
5672 755866 : if (!maybe_expand_insn (icode, 4, ops))
5673 : {
5674 151421 : delete_insns_since (last);
5675 151421 : return NULL_RTX;
5676 : }
5677 604445 : subtarget = ops[0].value;
5678 :
5679 : /* If we are converting to a wider mode, first convert to
5680 : INT_TARGET_MODE, then normalize. This produces better combining
5681 : opportunities on machines that have a SIGN_EXTRACT when we are
5682 : testing a single bit. This mostly benefits the 68k.
5683 :
5684 : If STORE_FLAG_VALUE does not have the sign bit set when
5685 : interpreted in MODE, we can do this conversion as unsigned, which
5686 : is usually more efficient. */
5687 604445 : if (GET_MODE_PRECISION (int_target_mode) > GET_MODE_PRECISION (result_mode))
5688 : {
5689 113446 : gcc_assert (GET_MODE_PRECISION (result_mode) != 1
5690 : || STORE_FLAG_VALUE == 1 || STORE_FLAG_VALUE == -1);
5691 :
5692 113446 : bool unsignedp = (STORE_FLAG_VALUE >= 0);
5693 113446 : convert_move (target, subtarget, unsignedp);
5694 :
5695 113446 : op0 = target;
5696 113446 : result_mode = int_target_mode;
5697 : }
5698 : else
5699 : op0 = subtarget;
5700 :
5701 : /* If we want to keep subexpressions around, don't reuse our last
5702 : target. */
5703 604445 : if (optimize)
5704 487190 : subtarget = 0;
5705 :
5706 : /* Now normalize to the proper value in MODE. Sometimes we don't
5707 : have to do anything. */
5708 604445 : if (normalizep == 0 || normalizep == STORE_FLAG_VALUE)
5709 : ;
5710 : /* STORE_FLAG_VALUE might be the most negative number, so write
5711 : the comparison this way to avoid a compiler-time warning. */
5712 353 : else if (- normalizep == STORE_FLAG_VALUE)
5713 353 : op0 = expand_unop (result_mode, neg_optab, op0, subtarget, 0);
5714 :
5715 : /* We don't want to use STORE_FLAG_VALUE < 0 below since this makes
5716 : it hard to use a value of just the sign bit due to ANSI integer
5717 : constant typing rules. */
5718 0 : else if (val_signbit_known_set_p (result_mode, STORE_FLAG_VALUE))
5719 0 : op0 = expand_shift (RSHIFT_EXPR, result_mode, op0,
5720 0 : GET_MODE_BITSIZE (result_mode) - 1, subtarget,
5721 : normalizep == 1);
5722 : else
5723 : {
5724 0 : gcc_assert (STORE_FLAG_VALUE & 1);
5725 :
5726 0 : op0 = expand_and (result_mode, op0, const1_rtx, subtarget);
5727 0 : if (normalizep == -1)
5728 : op0 = expand_unop (result_mode, neg_optab, op0, op0, 0);
5729 : }
5730 :
5731 : /* If we were converting to a smaller mode, do the conversion now. */
5732 604445 : if (int_target_mode != result_mode)
5733 : {
5734 0 : convert_move (target, op0, 0);
5735 0 : return target;
5736 : }
5737 : else
5738 : return op0;
5739 : }
5740 :
5741 :
5742 : /* A subroutine of emit_store_flag only including "tricks" that do not
5743 : need a recursive call. These are kept separate to avoid infinite
5744 : loops. */
5745 :
5746 : static rtx
5747 722377 : emit_store_flag_1 (rtx target, enum rtx_code code, rtx op0, rtx op1,
5748 : machine_mode mode, int unsignedp, int normalizep,
5749 : machine_mode target_mode)
5750 : {
5751 722377 : rtx subtarget;
5752 722377 : enum insn_code icode;
5753 722377 : machine_mode compare_mode;
5754 722377 : enum mode_class mclass;
5755 :
5756 722377 : if (unsignedp)
5757 165387 : code = unsigned_condition (code);
5758 :
5759 : /* If one operand is constant, make it the second one. Only do this
5760 : if the other operand is not constant as well. */
5761 :
5762 722377 : if (swap_commutative_operands_p (op0, op1))
5763 : {
5764 5083 : std::swap (op0, op1);
5765 5083 : code = swap_condition (code);
5766 : }
5767 :
5768 722377 : if (mode == VOIDmode)
5769 41130 : mode = GET_MODE (op0);
5770 :
5771 722377 : if (CONST_SCALAR_INT_P (op1))
5772 330425 : canonicalize_comparison (mode, &code, &op1);
5773 :
5774 : /* For some comparisons with 1 and -1, we can convert this to
5775 : comparisons with zero. This will often produce more opportunities for
5776 : store-flag insns. */
5777 :
5778 722377 : switch (code)
5779 : {
5780 45414 : case LT:
5781 45414 : if (op1 == const1_rtx)
5782 59 : op1 = const0_rtx, code = LE;
5783 : break;
5784 26766 : case LE:
5785 26766 : if (op1 == constm1_rtx)
5786 0 : op1 = const0_rtx, code = LT;
5787 : break;
5788 39624 : case GE:
5789 39624 : if (op1 == const1_rtx)
5790 0 : op1 = const0_rtx, code = GT;
5791 : break;
5792 33153 : case GT:
5793 33153 : if (op1 == constm1_rtx)
5794 136 : op1 = const0_rtx, code = GE;
5795 : break;
5796 4222 : case GEU:
5797 4222 : if (op1 == const1_rtx)
5798 0 : op1 = const0_rtx, code = NE;
5799 : break;
5800 8381 : case LTU:
5801 8381 : if (op1 == const1_rtx)
5802 10 : op1 = const0_rtx, code = EQ;
5803 : break;
5804 : default:
5805 : break;
5806 : }
5807 :
5808 : /* If this is A < 0 or A >= 0, we can do this by taking the ones
5809 : complement of A (for GE) and shifting the sign bit to the low bit. */
5810 722377 : scalar_int_mode int_mode;
5811 200075 : if (op1 == const0_rtx && (code == LT || code == GE)
5812 722377 : && is_int_mode (mode, &int_mode)
5813 722377 : && (normalizep || STORE_FLAG_VALUE == 1
5814 : || val_signbit_p (int_mode, STORE_FLAG_VALUE)))
5815 : {
5816 40409 : scalar_int_mode int_target_mode;
5817 40409 : subtarget = target;
5818 :
5819 40409 : if (!target)
5820 : int_target_mode = int_mode;
5821 : else
5822 : {
5823 : /* If the result is to be wider than OP0, it is best to convert it
5824 : first. If it is to be narrower, it is *incorrect* to convert it
5825 : first. */
5826 40409 : int_target_mode = as_a <scalar_int_mode> (target_mode);
5827 121227 : if (GET_MODE_SIZE (int_target_mode) > GET_MODE_SIZE (int_mode))
5828 : {
5829 461 : op0 = convert_modes (int_target_mode, int_mode, op0, 0);
5830 461 : int_mode = int_target_mode;
5831 : }
5832 : }
5833 :
5834 40409 : if (int_target_mode != int_mode)
5835 26894 : subtarget = 0;
5836 :
5837 40409 : if (code == GE)
5838 20798 : op0 = expand_unop (int_mode, one_cmpl_optab, op0,
5839 : ((STORE_FLAG_VALUE == 1 || normalizep)
5840 : ? 0 : subtarget), 0);
5841 :
5842 40409 : if (STORE_FLAG_VALUE == 1 || normalizep)
5843 : /* If we are supposed to produce a 0/1 value, we want to do
5844 : a logical shift from the sign bit to the low-order bit; for
5845 : a -1/0 value, we do an arithmetic shift. */
5846 80818 : op0 = expand_shift (RSHIFT_EXPR, int_mode, op0,
5847 40409 : GET_MODE_BITSIZE (int_mode) - 1,
5848 : subtarget, normalizep != -1);
5849 :
5850 40409 : if (int_mode != int_target_mode)
5851 26894 : op0 = convert_modes (int_target_mode, int_mode, op0, 0);
5852 :
5853 40409 : return op0;
5854 : }
5855 :
5856 : /* Next try expanding this via the backend's cstore<mode>4. */
5857 681968 : mclass = GET_MODE_CLASS (mode);
5858 693967 : FOR_EACH_WIDER_MODE_FROM (compare_mode, mode)
5859 : {
5860 686682 : machine_mode optab_mode = mclass == MODE_CC ? CCmode : compare_mode;
5861 686682 : icode = optab_handler (cstore_optab, optab_mode);
5862 686682 : if (icode != CODE_FOR_nothing)
5863 : {
5864 674683 : do_pending_stack_adjust ();
5865 674683 : rtx tem = emit_cstore (target, icode, code, mode, compare_mode,
5866 : unsignedp, op0, op1, normalizep, target_mode);
5867 674683 : if (tem)
5868 : return tem;
5869 :
5870 86463 : if (GET_MODE_CLASS (mode) == MODE_FLOAT)
5871 : {
5872 81449 : enum rtx_code scode = swap_condition (code);
5873 :
5874 81449 : tem = emit_cstore (target, icode, scode, mode, compare_mode,
5875 : unsignedp, op1, op0, normalizep, target_mode);
5876 81449 : if (tem)
5877 : return tem;
5878 : }
5879 : break;
5880 : }
5881 : }
5882 :
5883 : /* If we are comparing a double-word integer with zero or -1, we can
5884 : convert the comparison into one involving a single word. */
5885 77541 : if (is_int_mode (mode, &int_mode)
5886 7872 : && GET_MODE_BITSIZE (int_mode) == BITS_PER_WORD * 2
5887 4730 : && (!MEM_P (op0) || ! MEM_VOLATILE_P (op0)))
5888 : {
5889 4730 : rtx tem;
5890 4730 : if ((code == EQ || code == NE)
5891 0 : && (op1 == const0_rtx || op1 == constm1_rtx))
5892 : {
5893 0 : rtx op00, op01;
5894 :
5895 : /* Do a logical OR or AND of the two words and compare the
5896 : result. */
5897 0 : op00 = force_subreg (word_mode, op0, int_mode, 0);
5898 0 : op01 = force_subreg (word_mode, op0, int_mode, UNITS_PER_WORD);
5899 0 : tem = expand_binop (word_mode,
5900 0 : op1 == const0_rtx ? ior_optab : and_optab,
5901 : op00, op01, NULL_RTX, unsignedp,
5902 : OPTAB_DIRECT);
5903 :
5904 0 : if (tem != 0)
5905 0 : tem = emit_store_flag (NULL_RTX, code, tem, op1, word_mode,
5906 : unsignedp, normalizep);
5907 : }
5908 4730 : else if ((code == LT || code == GE) && op1 == const0_rtx)
5909 : {
5910 0 : rtx op0h;
5911 :
5912 : /* If testing the sign bit, can just test on high word. */
5913 0 : op0h = force_highpart_subreg (word_mode, op0, int_mode);
5914 0 : tem = emit_store_flag (NULL_RTX, code, op0h, op1, word_mode,
5915 : unsignedp, normalizep);
5916 0 : }
5917 : else
5918 : tem = NULL_RTX;
5919 :
5920 0 : if (tem)
5921 : {
5922 0 : if (target_mode == VOIDmode || GET_MODE (tem) == target_mode)
5923 : return tem;
5924 0 : if (!target)
5925 0 : target = gen_reg_rtx (target_mode);
5926 :
5927 0 : convert_move (target, tem,
5928 0 : !val_signbit_known_set_p (word_mode,
5929 : (normalizep ? normalizep
5930 : : STORE_FLAG_VALUE)));
5931 0 : return target;
5932 : }
5933 : }
5934 :
5935 : return 0;
5936 : }
5937 :
5938 : /* Subroutine of emit_store_flag that handles cases in which the operands
5939 : are scalar integers. SUBTARGET is the target to use for temporary
5940 : operations and TRUEVAL is the value to store when the condition is
5941 : true. All other arguments are as for emit_store_flag. */
5942 :
5943 : rtx
5944 2507 : emit_store_flag_int (rtx target, rtx subtarget, enum rtx_code code, rtx op0,
5945 : rtx op1, scalar_int_mode mode, int unsignedp,
5946 : int normalizep, rtx trueval)
5947 : {
5948 2507 : machine_mode target_mode = target ? GET_MODE (target) : VOIDmode;
5949 2507 : rtx_insn *last = get_last_insn ();
5950 :
5951 : /* If this is an equality comparison of integers, we can try to exclusive-or
5952 : (or subtract) the two operands and use a recursive call to try the
5953 : comparison with zero. Don't do any of these cases if branches are
5954 : very cheap. */
5955 :
5956 2507 : if ((code == EQ || code == NE) && op1 != const0_rtx)
5957 : {
5958 0 : rtx tem = expand_binop (mode, xor_optab, op0, op1, subtarget, 1,
5959 : OPTAB_WIDEN);
5960 :
5961 0 : if (tem == 0)
5962 0 : tem = expand_binop (mode, sub_optab, op0, op1, subtarget, 1,
5963 : OPTAB_WIDEN);
5964 0 : if (tem != 0)
5965 0 : tem = emit_store_flag (target, code, tem, const0_rtx,
5966 : mode, unsignedp, normalizep);
5967 0 : if (tem != 0)
5968 : return tem;
5969 :
5970 0 : delete_insns_since (last);
5971 : }
5972 :
5973 : /* For integer comparisons, try the reverse comparison. However, for
5974 : small X and if we'd have anyway to extend, implementing "X != 0"
5975 : as "-(int)X >> 31" is still cheaper than inverting "(int)X == 0". */
5976 2507 : rtx_code rcode = reverse_condition (code);
5977 2507 : if (can_compare_p (rcode, mode, ccp_store_flag)
5978 2507 : && ! (optab_handler (cstore_optab, mode) == CODE_FOR_nothing
5979 0 : && code == NE
5980 0 : && GET_MODE_SIZE (mode) < UNITS_PER_WORD
5981 0 : && op1 == const0_rtx))
5982 : {
5983 2507 : int want_add = ((STORE_FLAG_VALUE == 1 && normalizep == -1)
5984 : || (STORE_FLAG_VALUE == -1 && normalizep == 1));
5985 :
5986 : /* Again, for the reverse comparison, use either an addition or a XOR. */
5987 2507 : if (want_add
5988 2507 : && rtx_cost (GEN_INT (normalizep), mode, PLUS, 1,
5989 0 : optimize_insn_for_speed_p ()) == 0)
5990 : {
5991 0 : rtx tem = emit_store_flag_1 (subtarget, rcode, op0, op1, mode, 0,
5992 : STORE_FLAG_VALUE, target_mode);
5993 0 : if (tem != 0)
5994 0 : tem = expand_binop (target_mode, add_optab, tem,
5995 0 : gen_int_mode (normalizep, target_mode),
5996 : target, 0, OPTAB_WIDEN);
5997 0 : if (tem != 0)
5998 : return tem;
5999 : }
6000 2507 : else if (!want_add
6001 5014 : && rtx_cost (trueval, mode, XOR, 1,
6002 2507 : optimize_insn_for_speed_p ()) == 0)
6003 : {
6004 2507 : rtx tem = emit_store_flag_1 (subtarget, rcode, op0, op1, mode, 0,
6005 : normalizep, target_mode);
6006 2507 : if (tem != 0)
6007 0 : tem = expand_binop (target_mode, xor_optab, tem, trueval, target,
6008 0 : INTVAL (trueval) >= 0, OPTAB_WIDEN);
6009 0 : if (tem != 0)
6010 : return tem;
6011 : }
6012 :
6013 2507 : delete_insns_since (last);
6014 : }
6015 :
6016 : /* Some other cases we can do are EQ, NE, LE, and GT comparisons with
6017 : the constant zero. Reject all other comparisons at this point. Only
6018 : do LE and GT if branches are expensive since they are expensive on
6019 : 2-operand machines. */
6020 :
6021 2507 : if (op1 != const0_rtx
6022 2507 : || (code != EQ && code != NE
6023 91 : && (BRANCH_COST (optimize_insn_for_speed_p (),
6024 91 : false) <= 1 || (code != LE && code != GT))))
6025 2274 : return 0;
6026 :
6027 : /* Try to put the result of the comparison in the sign bit. Assume we can't
6028 : do the necessary operation below. */
6029 :
6030 233 : rtx tem = 0;
6031 :
6032 : /* To see if A <= 0, compute (A | (A - 1)). A <= 0 iff that result has
6033 : the sign bit set. */
6034 :
6035 233 : if (code == LE)
6036 : {
6037 : /* This is destructive, so SUBTARGET can't be OP0. */
6038 59 : if (rtx_equal_p (subtarget, op0))
6039 0 : subtarget = 0;
6040 :
6041 59 : tem = expand_binop (mode, sub_optab, op0, const1_rtx, subtarget, 0,
6042 : OPTAB_WIDEN);
6043 59 : if (tem)
6044 59 : tem = expand_binop (mode, ior_optab, op0, tem, subtarget, 0,
6045 : OPTAB_WIDEN);
6046 : }
6047 :
6048 : /* To see if A > 0, compute (((signed) A) << BITS) - A, where BITS is the
6049 : number of bits in the mode of OP0, minus one. */
6050 :
6051 233 : if (code == GT)
6052 : {
6053 32 : if (rtx_equal_p (subtarget, op0))
6054 0 : subtarget = 0;
6055 :
6056 32 : tem = maybe_expand_shift (RSHIFT_EXPR, mode, op0,
6057 32 : GET_MODE_BITSIZE (mode) - 1,
6058 : subtarget, 0);
6059 32 : if (tem)
6060 32 : tem = expand_binop (mode, sub_optab, tem, op0, subtarget, 0,
6061 : OPTAB_WIDEN);
6062 : }
6063 :
6064 233 : if (code == EQ || code == NE)
6065 : {
6066 : /* For EQ or NE, one way to do the comparison is to apply an operation
6067 : that converts the operand into a positive number if it is nonzero
6068 : or zero if it was originally zero. Then, for EQ, we subtract 1 and
6069 : for NE we negate. This puts the result in the sign bit. Then we
6070 : normalize with a shift, if needed.
6071 :
6072 : Two operations that can do the above actions are ABS and FFS, so try
6073 : them. If that doesn't work, and MODE is smaller than a full word,
6074 : we can use zero-extension to the wider mode (an unsigned conversion)
6075 : as the operation. */
6076 :
6077 : /* Note that ABS doesn't yield a positive number for INT_MIN, but
6078 : that is compensated by the subsequent overflow when subtracting
6079 : one / negating. */
6080 :
6081 142 : if (optab_handler (abs_optab, mode) != CODE_FOR_nothing)
6082 142 : tem = expand_unop (mode, abs_optab, op0, subtarget, 1);
6083 0 : else if (optab_handler (ffs_optab, mode) != CODE_FOR_nothing)
6084 0 : tem = expand_unop (mode, ffs_optab, op0, subtarget, 1);
6085 0 : else if (GET_MODE_SIZE (mode) < UNITS_PER_WORD)
6086 : {
6087 0 : tem = convert_modes (word_mode, mode, op0, 1);
6088 0 : mode = word_mode;
6089 : }
6090 :
6091 142 : if (tem != 0)
6092 : {
6093 142 : if (code == EQ)
6094 0 : tem = expand_binop (mode, sub_optab, tem, const1_rtx, subtarget,
6095 : 0, OPTAB_WIDEN);
6096 : else
6097 142 : tem = expand_unop (mode, neg_optab, tem, subtarget, 0);
6098 : }
6099 :
6100 : /* If we couldn't do it that way, for NE we can "or" the two's complement
6101 : of the value with itself. For EQ, we take the one's complement of
6102 : that "or", which is an extra insn, so we only handle EQ if branches
6103 : are expensive. */
6104 :
6105 142 : if (tem == 0
6106 142 : && (code == NE
6107 0 : || BRANCH_COST (optimize_insn_for_speed_p (),
6108 : false) > 1))
6109 : {
6110 0 : if (rtx_equal_p (subtarget, op0))
6111 0 : subtarget = 0;
6112 :
6113 0 : tem = expand_unop (mode, neg_optab, op0, subtarget, 0);
6114 0 : tem = expand_binop (mode, ior_optab, tem, op0, subtarget, 0,
6115 : OPTAB_WIDEN);
6116 :
6117 0 : if (tem && code == EQ)
6118 0 : tem = expand_unop (mode, one_cmpl_optab, tem, subtarget, 0);
6119 : }
6120 : }
6121 :
6122 233 : if (tem && normalizep)
6123 233 : tem = maybe_expand_shift (RSHIFT_EXPR, mode, tem,
6124 233 : GET_MODE_BITSIZE (mode) - 1,
6125 : subtarget, normalizep == 1);
6126 :
6127 233 : if (tem)
6128 : {
6129 233 : if (!target)
6130 : ;
6131 233 : else if (GET_MODE (tem) != target_mode)
6132 : {
6133 91 : convert_move (target, tem, 0);
6134 91 : tem = target;
6135 : }
6136 142 : else if (!subtarget)
6137 : {
6138 73 : emit_move_insn (target, tem);
6139 73 : tem = target;
6140 : }
6141 : }
6142 : else
6143 0 : delete_insns_since (last);
6144 :
6145 : return tem;
6146 : }
6147 :
6148 : /* Emit a store-flags instruction for comparison CODE on OP0 and OP1
6149 : and storing in TARGET. Normally return TARGET.
6150 : Return 0 if that cannot be done.
6151 :
6152 : MODE is the mode to use for OP0 and OP1 should they be CONST_INTs. If
6153 : it is VOIDmode, they cannot both be CONST_INT.
6154 :
6155 : UNSIGNEDP is for the case where we have to widen the operands
6156 : to perform the operation. It says to use zero-extension.
6157 :
6158 : NORMALIZEP is 1 if we should convert the result to be either zero
6159 : or one. Normalize is -1 if we should convert the result to be
6160 : either zero or -1. If NORMALIZEP is zero, the result will be left
6161 : "raw" out of the scc insn. */
6162 :
6163 : rtx
6164 654409 : emit_store_flag (rtx target, enum rtx_code code, rtx op0, rtx op1,
6165 : machine_mode mode, int unsignedp, int normalizep)
6166 : {
6167 654409 : machine_mode target_mode = target ? GET_MODE (target) : VOIDmode;
6168 654409 : enum rtx_code rcode;
6169 654409 : rtx subtarget;
6170 654409 : rtx tem, trueval;
6171 654409 : rtx_insn *last;
6172 :
6173 : /* If we compare constants, we shouldn't use a store-flag operation,
6174 : but a constant load. We can get there via the vanilla route that
6175 : usually generates a compare-branch sequence, but will in this case
6176 : fold the comparison to a constant, and thus elide the branch. */
6177 654409 : if (CONSTANT_P (op0) && CONSTANT_P (op1))
6178 : return NULL_RTX;
6179 :
6180 654092 : tem = emit_store_flag_1 (target, code, op0, op1, mode, unsignedp, normalizep,
6181 : target_mode);
6182 654092 : if (tem)
6183 : return tem;
6184 :
6185 : /* If we reached here, we can't do this with a scc insn, however there
6186 : are some comparisons that can be done in other ways. Don't do any
6187 : of these cases if branches are very cheap. */
6188 74498 : if (BRANCH_COST (optimize_insn_for_speed_p (), false) == 0)
6189 : return 0;
6190 :
6191 : /* See what we need to return. We can only return a 1, -1, or the
6192 : sign bit. */
6193 :
6194 74498 : if (normalizep == 0)
6195 : {
6196 0 : if (STORE_FLAG_VALUE == 1 || STORE_FLAG_VALUE == -1)
6197 0 : normalizep = STORE_FLAG_VALUE;
6198 :
6199 : else if (val_signbit_p (mode, STORE_FLAG_VALUE))
6200 : ;
6201 : else
6202 : return 0;
6203 : }
6204 :
6205 74498 : last = get_last_insn ();
6206 :
6207 : /* If optimizing, use different pseudo registers for each insn, instead
6208 : of reusing the same pseudo. This leads to better CSE, but slows
6209 : down the compiler, since there are more pseudos. */
6210 74429 : subtarget = (!optimize
6211 74498 : && (target_mode == mode)) ? target : NULL_RTX;
6212 74498 : trueval = GEN_INT (normalizep ? normalizep : STORE_FLAG_VALUE);
6213 :
6214 : /* For floating-point comparisons, try the reverse comparison or try
6215 : changing the "orderedness" of the comparison. */
6216 74498 : if (GET_MODE_CLASS (mode) == MODE_FLOAT)
6217 : {
6218 68469 : enum rtx_code first_code;
6219 68469 : bool and_them;
6220 :
6221 68469 : rcode = reverse_condition_maybe_unordered (code);
6222 68469 : if (can_compare_p (rcode, mode, ccp_store_flag)
6223 68469 : && (code == ORDERED || code == UNORDERED
6224 0 : || (! HONOR_NANS (mode) && (code == LTGT || code == UNEQ))
6225 0 : || (! HONOR_SNANS (mode) && (code == EQ || code == NE))))
6226 : {
6227 0 : int want_add = ((STORE_FLAG_VALUE == 1 && normalizep == -1)
6228 : || (STORE_FLAG_VALUE == -1 && normalizep == 1));
6229 :
6230 : /* For the reverse comparison, use either an addition or a XOR. */
6231 0 : if (want_add
6232 0 : && rtx_cost (GEN_INT (normalizep), mode, PLUS, 1,
6233 0 : optimize_insn_for_speed_p ()) == 0)
6234 : {
6235 0 : tem = emit_store_flag_1 (subtarget, rcode, op0, op1, mode, 0,
6236 : STORE_FLAG_VALUE, target_mode);
6237 0 : if (tem)
6238 0 : return expand_binop (target_mode, add_optab, tem,
6239 0 : gen_int_mode (normalizep, target_mode),
6240 : target, 0, OPTAB_WIDEN);
6241 : }
6242 0 : else if (!want_add
6243 0 : && rtx_cost (trueval, mode, XOR, 1,
6244 0 : optimize_insn_for_speed_p ()) == 0)
6245 : {
6246 0 : tem = emit_store_flag_1 (subtarget, rcode, op0, op1, mode, 0,
6247 : normalizep, target_mode);
6248 0 : if (tem)
6249 0 : return expand_binop (target_mode, xor_optab, tem, trueval,
6250 0 : target, INTVAL (trueval) >= 0,
6251 0 : OPTAB_WIDEN);
6252 : }
6253 : }
6254 :
6255 68469 : delete_insns_since (last);
6256 :
6257 : /* Cannot split ORDERED and UNORDERED, only try the above trick. */
6258 68469 : if (code == ORDERED || code == UNORDERED)
6259 : return 0;
6260 :
6261 68343 : and_them = split_comparison (code, mode, &first_code, &code);
6262 :
6263 : /* If there are no NaNs, the first comparison should always fall through.
6264 : Effectively change the comparison to the other one. */
6265 68343 : if (!HONOR_NANS (mode))
6266 : {
6267 404 : gcc_assert (first_code == (and_them ? ORDERED : UNORDERED));
6268 243 : return emit_store_flag_1 (target, code, op0, op1, mode, 0, normalizep,
6269 243 : target_mode);
6270 : }
6271 :
6272 68100 : if (!HAVE_conditional_move)
6273 : return 0;
6274 :
6275 : /* Do not turn a trapping comparison into a non-trapping one. */
6276 68100 : if ((code != EQ && code != NE && code != UNEQ && code != LTGT)
6277 2565 : && flag_trapping_math)
6278 : return 0;
6279 :
6280 : /* Try using a setcc instruction for ORDERED/UNORDERED, followed by a
6281 : conditional move. */
6282 65535 : tem = emit_store_flag_1 (subtarget, first_code, op0, op1, mode, 0,
6283 : normalizep, target_mode);
6284 65535 : if (tem == 0)
6285 : return 0;
6286 :
6287 65000 : if (and_them)
6288 2687 : tem = emit_conditional_move (target, { code, op0, op1, mode },
6289 2687 : tem, const0_rtx, GET_MODE (tem), 0);
6290 : else
6291 62313 : tem = emit_conditional_move (target, { code, op0, op1, mode },
6292 62313 : trueval, tem, GET_MODE (tem), 0);
6293 :
6294 65000 : if (tem == 0)
6295 666 : delete_insns_since (last);
6296 65000 : return tem;
6297 : }
6298 :
6299 : /* The remaining tricks only apply to integer comparisons. */
6300 :
6301 6029 : scalar_int_mode int_mode;
6302 6029 : if (is_int_mode (mode, &int_mode))
6303 2507 : return emit_store_flag_int (target, subtarget, code, op0, op1, int_mode,
6304 2507 : unsignedp, normalizep, trueval);
6305 :
6306 : return 0;
6307 : }
6308 :
6309 : /* Like emit_store_flag, but always succeeds. */
6310 :
6311 : rtx
6312 603757 : emit_store_flag_force (rtx target, enum rtx_code code, rtx op0, rtx op1,
6313 : machine_mode mode, int unsignedp, int normalizep)
6314 : {
6315 603757 : rtx tem;
6316 603757 : rtx_code_label *label;
6317 603757 : rtx trueval, falseval;
6318 :
6319 : /* First see if emit_store_flag can do the job. */
6320 603757 : tem = emit_store_flag (target, code, op0, op1, mode, unsignedp, normalizep);
6321 603757 : if (tem != 0)
6322 : return tem;
6323 :
6324 : /* If one operand is constant, make it the second one. Only do this
6325 : if the other operand is not constant as well. */
6326 10006 : if (swap_commutative_operands_p (op0, op1))
6327 : {
6328 8 : std::swap (op0, op1);
6329 8 : code = swap_condition (code);
6330 : }
6331 :
6332 10006 : if (mode == VOIDmode)
6333 0 : mode = GET_MODE (op0);
6334 :
6335 10006 : if (!target)
6336 0 : target = gen_reg_rtx (word_mode);
6337 :
6338 : /* If this failed, we have to do this with set/compare/jump/set code.
6339 : For foo != 0, if foo is in OP0, just replace it with 1 if nonzero. */
6340 10006 : trueval = normalizep ? GEN_INT (normalizep) : const1_rtx;
6341 10006 : if (code == NE
6342 1753 : && GET_MODE_CLASS (mode) == MODE_INT
6343 41 : && REG_P (target)
6344 41 : && op0 == target
6345 0 : && op1 == const0_rtx)
6346 : {
6347 0 : label = gen_label_rtx ();
6348 0 : do_compare_rtx_and_jump (target, const0_rtx, EQ, unsignedp, mode,
6349 : NULL_RTX, NULL, label,
6350 : profile_probability::uninitialized ());
6351 0 : emit_move_insn (target, trueval);
6352 0 : emit_label (label);
6353 0 : return target;
6354 : }
6355 :
6356 10006 : if (!REG_P (target)
6357 10006 : || reg_mentioned_p (target, op0) || reg_mentioned_p (target, op1))
6358 5 : target = gen_reg_rtx (GET_MODE (target));
6359 :
6360 : /* Jump in the right direction if the target cannot implement CODE
6361 : but can jump on its reverse condition. */
6362 10006 : falseval = const0_rtx;
6363 10006 : if (! can_compare_p (code, mode, ccp_jump)
6364 10006 : && (! FLOAT_MODE_P (mode)
6365 7375 : || code == ORDERED || code == UNORDERED
6366 7204 : || (! HONOR_NANS (mode) && (code == LTGT || code == UNEQ))
6367 7204 : || (! HONOR_SNANS (mode) && (code == EQ || code == NE))))
6368 : {
6369 2724 : enum rtx_code rcode;
6370 2724 : if (FLOAT_MODE_P (mode))
6371 2724 : rcode = reverse_condition_maybe_unordered (code);
6372 : else
6373 0 : rcode = reverse_condition (code);
6374 :
6375 : /* Canonicalize to UNORDERED for the libcall. */
6376 2724 : if (can_compare_p (rcode, mode, ccp_jump)
6377 2724 : || (code == ORDERED && ! can_compare_p (ORDERED, mode, ccp_jump)))
6378 : {
6379 147 : falseval = trueval;
6380 147 : trueval = const0_rtx;
6381 147 : code = rcode;
6382 : }
6383 : }
6384 :
6385 10006 : emit_move_insn (target, trueval);
6386 10006 : label = gen_label_rtx ();
6387 10006 : do_compare_rtx_and_jump (op0, op1, code, unsignedp, mode, NULL_RTX, NULL,
6388 : label, profile_probability::uninitialized ());
6389 :
6390 10006 : emit_move_insn (target, falseval);
6391 10006 : emit_label (label);
6392 :
6393 10006 : return target;
6394 : }
6395 :
6396 : /* Expand a vector (left) rotate of MODE of X by an immediate AMT as a vector
6397 : permute operation. Emit code to put the result in DST if successful and
6398 : return it. Otherwise return NULL. This is intended to implement vector
6399 : rotates by byte amounts using vector permutes when the target does not offer
6400 : native vector rotate operations. */
6401 : rtx
6402 0 : expand_rotate_as_vec_perm (machine_mode mode, rtx dst, rtx x, rtx amt)
6403 : {
6404 0 : rtx amt_unwrap = unwrap_const_vec_duplicate (amt);
6405 : /* For now handle only rotate by the same integer constant in all lanes.
6406 : In principle rotates by any constant vector are representable through
6407 : permutes as long as the individual rotate amounts are multiples of
6408 : BITS_PER_UNIT. */
6409 0 : if (!CONST_INT_P (amt_unwrap))
6410 : return NULL_RTX;
6411 :
6412 0 : int rotamnt = INTVAL (amt_unwrap);
6413 0 : if (rotamnt % BITS_PER_UNIT != 0)
6414 : return NULL_RTX;
6415 0 : machine_mode qimode;
6416 0 : if (!qimode_for_vec_perm (mode).exists (&qimode))
6417 0 : return NULL_RTX;
6418 :
6419 0 : vec_perm_builder builder;
6420 0 : unsigned nunits = GET_MODE_SIZE (GET_MODE_INNER (mode));
6421 0 : poly_uint64 total_units = GET_MODE_SIZE (mode);
6422 0 : builder.new_vector (total_units, nunits, 3);
6423 0 : unsigned rot_bytes = rotamnt / BITS_PER_UNIT;
6424 0 : unsigned rot_to_perm = BYTES_BIG_ENDIAN ? rot_bytes : nunits - rot_bytes;
6425 0 : for (unsigned j = 0; j < 3 * nunits; j += nunits)
6426 0 : for (unsigned i = 0; i < nunits; i++)
6427 0 : builder.quick_push ((rot_to_perm + i) % nunits + j);
6428 :
6429 0 : rtx perm_src = lowpart_subreg (qimode, x, mode);
6430 0 : rtx perm_dst = lowpart_subreg (qimode, dst, mode);
6431 0 : rtx res
6432 0 : = expand_vec_perm_const (qimode, perm_src, perm_src, builder,
6433 : qimode, perm_dst);
6434 0 : if (!res)
6435 : return NULL_RTX;
6436 0 : if (!rtx_equal_p (res, perm_dst))
6437 0 : emit_move_insn (dst, lowpart_subreg (mode, res, qimode));
6438 : return dst;
6439 0 : }
6440 :
6441 : /* Helper function for canonicalize_cmp_for_target. Swap between inclusive
6442 : and exclusive ranges in order to create an equivalent comparison. See
6443 : canonicalize_cmp_for_target for the possible cases. */
6444 :
6445 : static enum rtx_code
6446 47 : equivalent_cmp_code (enum rtx_code code)
6447 : {
6448 47 : switch (code)
6449 : {
6450 : case GT:
6451 : return GE;
6452 0 : case GE:
6453 0 : return GT;
6454 0 : case LT:
6455 0 : return LE;
6456 0 : case LE:
6457 0 : return LT;
6458 2 : case GTU:
6459 2 : return GEU;
6460 0 : case GEU:
6461 0 : return GTU;
6462 1 : case LTU:
6463 1 : return LEU;
6464 2 : case LEU:
6465 2 : return LTU;
6466 :
6467 0 : default:
6468 0 : return code;
6469 : }
6470 : }
6471 :
6472 : /* Choose the more appropriate immediate in scalar integer comparisons. The
6473 : purpose of this is to end up with an immediate which can be loaded into a
6474 : register in fewer moves, if possible.
6475 :
6476 : For each integer comparison there exists an equivalent choice:
6477 : i) a > b or a >= b + 1
6478 : ii) a <= b or a < b + 1
6479 : iii) a >= b or a > b - 1
6480 : iv) a < b or a <= b - 1
6481 :
6482 : MODE is the mode of the first operand.
6483 : CODE points to the comparison code.
6484 : IMM points to the rtx containing the immediate. *IMM must satisfy
6485 : CONST_SCALAR_INT_P on entry and continues to satisfy CONST_SCALAR_INT_P
6486 : on exit. */
6487 :
6488 : void
6489 4738008 : canonicalize_comparison (machine_mode mode, enum rtx_code *code, rtx *imm)
6490 : {
6491 4738008 : if (!SCALAR_INT_MODE_P (mode))
6492 3908343 : return;
6493 :
6494 4734218 : int to_add = 0;
6495 4734218 : enum signop sgn = unsigned_condition_p (*code) ? UNSIGNED : SIGNED;
6496 :
6497 : /* Extract the immediate value from the rtx. */
6498 4734218 : wide_int imm_val = rtx_mode_t (*imm, mode);
6499 :
6500 4734218 : if (*code == GT || *code == GTU || *code == LE || *code == LEU)
6501 : to_add = 1;
6502 : else if (*code == GE || *code == GEU || *code == LT || *code == LTU)
6503 : to_add = -1;
6504 : else
6505 : return;
6506 :
6507 : /* Check for overflow/underflow in the case of signed values and
6508 : wrapping around in the case of unsigned values. If any occur
6509 : cancel the optimization. */
6510 829809 : wi::overflow_type overflow = wi::OVF_NONE;
6511 829809 : wide_int imm_modif;
6512 :
6513 829809 : if (to_add == 1)
6514 597869 : imm_modif = wi::add (imm_val, 1, sgn, &overflow);
6515 : else
6516 231940 : imm_modif = wi::sub (imm_val, 1, sgn, &overflow);
6517 :
6518 829809 : if (overflow)
6519 144 : return;
6520 :
6521 829665 : rtx new_imm = immed_wide_int_const (imm_modif, mode);
6522 :
6523 829665 : int old_cost = rtx_cost (*imm, mode, COMPARE, 0, true);
6524 829665 : int new_cost = rtx_cost (new_imm, mode, COMPARE, 0, true);
6525 :
6526 829665 : if (dump_file && (dump_flags & TDF_DETAILS))
6527 : {
6528 7 : fprintf (dump_file, ";; cmp: %s, old cst: ",
6529 7 : GET_RTX_NAME (*code));
6530 7 : print_rtl (dump_file, *imm);
6531 7 : fprintf (dump_file, " new cst: ");
6532 7 : print_rtl (dump_file, new_imm);
6533 7 : fprintf (dump_file, "\n");
6534 7 : fprintf (dump_file, ";; old cst cost: %d, new cst cost: %d\n",
6535 : old_cost, new_cost);
6536 : }
6537 :
6538 : /* Update the immediate and the code. */
6539 829665 : if (old_cost > new_cost)
6540 : {
6541 47 : *code = equivalent_cmp_code (*code);
6542 47 : *imm = new_imm;
6543 : }
6544 4734362 : }
6545 :
6546 :
6547 :
6548 : /* Perform possibly multi-word comparison and conditional jump to LABEL
6549 : if ARG1 OP ARG2 true where ARG1 and ARG2 are of mode MODE. This is
6550 : now a thin wrapper around do_compare_rtx_and_jump. */
6551 :
6552 : static void
6553 2407 : do_cmp_and_jump (rtx arg1, rtx arg2, enum rtx_code op, machine_mode mode,
6554 : rtx_code_label *label)
6555 : {
6556 2407 : int unsignedp = (op == LTU || op == LEU || op == GTU || op == GEU);
6557 2407 : do_compare_rtx_and_jump (arg1, arg2, op, unsignedp, mode, NULL_RTX,
6558 : NULL, label, profile_probability::uninitialized ());
6559 2407 : }
|
