Line data Source code
1 : /* Medium-level subroutines: convert bit-field store and extract
2 : and shifts, multiplies and divides to rtl instructions.
3 : Copyright (C) 1987-2026 Free Software Foundation, Inc.
4 :
5 : This file is part of GCC.
6 :
7 : GCC is free software; you can redistribute it and/or modify it under
8 : the terms of the GNU General Public License as published by the Free
9 : Software Foundation; either version 3, or (at your option) any later
10 : version.
11 :
12 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 : for more details.
16 :
17 : You should have received a copy of the GNU General Public License
18 : along with GCC; see the file COPYING3. If not see
19 : <http://www.gnu.org/licenses/>. */
20 :
21 : /* Work around tree-optimization/91825. */
22 : #pragma GCC diagnostic warning "-Wmaybe-uninitialized"
23 :
24 : #include "config.h"
25 : #include "system.h"
26 : #include "coretypes.h"
27 : #include "backend.h"
28 : #include "target.h"
29 : #include "rtl.h"
30 : #include "tree.h"
31 : #include "predict.h"
32 : #include "memmodel.h"
33 : #include "tm_p.h"
34 : #include "optabs.h"
35 : #include "expmed.h"
36 : #include "regs.h"
37 : #include "emit-rtl.h"
38 : #include "diagnostic-core.h"
39 : #include "fold-const.h"
40 : #include "stor-layout.h"
41 : #include "dojump.h"
42 : #include "explow.h"
43 : #include "expr.h"
44 : #include "langhooks.h"
45 : #include "tree-vector-builder.h"
46 : #include "recog.h"
47 :
48 : struct target_expmed default_target_expmed;
49 : #if SWITCHABLE_TARGET
50 : struct target_expmed *this_target_expmed = &default_target_expmed;
51 : #endif
52 :
53 : static bool store_integral_bit_field (rtx, opt_scalar_int_mode,
54 : unsigned HOST_WIDE_INT,
55 : unsigned HOST_WIDE_INT,
56 : poly_uint64, poly_uint64,
57 : machine_mode, rtx, bool, bool);
58 : /* The constant divisor of the last division expanded, so that a modulo by
59 : the same constant can reuse the quotient. Reset for each function by
60 : prepare_function_start: carrying it across functions would make a
61 : function's code depend on what was expanded before it in the translation
62 : unit. */
63 : HOST_WIDE_INT last_div_const = 0;
64 :
65 : static void store_fixed_bit_field (rtx, opt_scalar_int_mode,
66 : unsigned HOST_WIDE_INT,
67 : unsigned HOST_WIDE_INT,
68 : poly_uint64, poly_uint64,
69 : rtx, scalar_int_mode, bool);
70 : static void store_fixed_bit_field_1 (rtx, scalar_int_mode,
71 : unsigned HOST_WIDE_INT,
72 : unsigned HOST_WIDE_INT,
73 : rtx, scalar_int_mode, bool);
74 : static void store_split_bit_field (rtx, opt_scalar_int_mode,
75 : unsigned HOST_WIDE_INT,
76 : unsigned HOST_WIDE_INT,
77 : poly_uint64, poly_uint64,
78 : rtx, scalar_int_mode, bool);
79 : static rtx extract_integral_bit_field (rtx, opt_scalar_int_mode,
80 : unsigned HOST_WIDE_INT,
81 : unsigned HOST_WIDE_INT, int, rtx,
82 : machine_mode, machine_mode, bool, bool);
83 : static rtx extract_fixed_bit_field (machine_mode, rtx, opt_scalar_int_mode,
84 : unsigned HOST_WIDE_INT,
85 : unsigned HOST_WIDE_INT, rtx, int, bool);
86 : static rtx extract_fixed_bit_field_1 (machine_mode, rtx, scalar_int_mode,
87 : unsigned HOST_WIDE_INT,
88 : unsigned HOST_WIDE_INT, rtx, int, bool);
89 : static rtx lshift_value (machine_mode, unsigned HOST_WIDE_INT, int);
90 : static rtx extract_split_bit_field (rtx, opt_scalar_int_mode,
91 : unsigned HOST_WIDE_INT,
92 : unsigned HOST_WIDE_INT, int, bool);
93 : static void do_cmp_and_jump (rtx, rtx, enum rtx_code, machine_mode, rtx_code_label *);
94 : static rtx expand_smod_pow2 (scalar_int_mode, rtx, HOST_WIDE_INT);
95 : static rtx expand_sdiv_pow2 (scalar_int_mode, rtx, HOST_WIDE_INT);
96 :
97 : /* Return a constant integer mask value of mode MODE with BITSIZE ones
98 : followed by BITPOS zeros, or the complement of that if COMPLEMENT.
99 : The mask is truncated if necessary to the width of mode MODE. The
100 : mask is zero-extended if BITSIZE+BITPOS is too small for MODE. */
101 :
102 : static inline rtx
103 215633 : mask_rtx (scalar_int_mode mode, int bitpos, int bitsize, bool complement)
104 : {
105 215633 : return immed_wide_int_const
106 215633 : (wi::shifted_mask (bitpos, bitsize, complement,
107 215633 : GET_MODE_PRECISION (mode)), mode);
108 : }
109 :
110 : /* Test whether a value is zero of a power of two. */
111 : #define EXACT_POWER_OF_2_OR_ZERO_P(x) \
112 : (((x) & ((x) - HOST_WIDE_INT_1U)) == 0)
113 :
114 : struct init_expmed_rtl
115 : {
116 : rtx reg;
117 : rtx plus;
118 : rtx neg;
119 : rtx mult;
120 : rtx sdiv;
121 : rtx udiv;
122 : rtx sdiv_32;
123 : rtx smod_32;
124 : rtx wide_mult;
125 : rtx wide_lshr;
126 : rtx wide_trunc;
127 : rtx shift;
128 : rtx shift_mult;
129 : rtx shift_add;
130 : rtx shift_sub0;
131 : rtx shift_sub1;
132 : rtx zext;
133 : rtx trunc;
134 :
135 : rtx pow2[MAX_BITS_PER_WORD];
136 : rtx cint[MAX_BITS_PER_WORD];
137 : };
138 :
139 : static void
140 30814000 : init_expmed_one_conv (struct init_expmed_rtl *all, scalar_int_mode to_mode,
141 : scalar_int_mode from_mode, bool speed)
142 : {
143 30814000 : int to_size, from_size;
144 30814000 : rtx which;
145 :
146 30814000 : to_size = GET_MODE_PRECISION (to_mode);
147 30814000 : from_size = GET_MODE_PRECISION (from_mode);
148 :
149 : /* Most partial integers have a precision less than the "full"
150 : integer it requires for storage. In case one doesn't, for
151 : comparison purposes here, reduce the bit size by one in that
152 : case. */
153 30814000 : if (GET_MODE_CLASS (to_mode) == MODE_PARTIAL_INT
154 30814000 : && pow2p_hwi (to_size))
155 6162800 : to_size --;
156 30814000 : if (GET_MODE_CLASS (from_mode) == MODE_PARTIAL_INT
157 30814000 : && pow2p_hwi (from_size))
158 0 : from_size --;
159 :
160 : /* Assume cost of zero-extend and sign-extend is the same. */
161 30814000 : which = (to_size < from_size ? all->trunc : all->zext);
162 :
163 30814000 : PUT_MODE (all->reg, from_mode);
164 30814000 : set_convert_cost (to_mode, from_mode, speed,
165 : set_src_cost (which, to_mode, speed));
166 : /* Restore all->reg's mode. */
167 30814000 : PUT_MODE (all->reg, to_mode);
168 30814000 : }
169 :
170 : static void
171 18048200 : init_expmed_one_mode (struct init_expmed_rtl *all,
172 : machine_mode mode, int speed)
173 : {
174 18048200 : int m, n, mode_bitsize;
175 18048200 : machine_mode mode_from;
176 :
177 18048200 : mode_bitsize = GET_MODE_UNIT_BITSIZE (mode);
178 :
179 18048200 : PUT_MODE (all->reg, mode);
180 18048200 : PUT_MODE (all->plus, mode);
181 18048200 : PUT_MODE (all->neg, mode);
182 18048200 : PUT_MODE (all->mult, mode);
183 18048200 : PUT_MODE (all->sdiv, mode);
184 18048200 : PUT_MODE (all->udiv, mode);
185 18048200 : PUT_MODE (all->sdiv_32, mode);
186 18048200 : PUT_MODE (all->smod_32, mode);
187 18048200 : PUT_MODE (all->wide_trunc, mode);
188 18048200 : PUT_MODE (all->shift, mode);
189 18048200 : PUT_MODE (all->shift_mult, mode);
190 18048200 : PUT_MODE (all->shift_add, mode);
191 18048200 : PUT_MODE (all->shift_sub0, mode);
192 18048200 : PUT_MODE (all->shift_sub1, mode);
193 18048200 : PUT_MODE (all->zext, mode);
194 18048200 : PUT_MODE (all->trunc, mode);
195 :
196 18048200 : set_add_cost (speed, mode, set_src_cost (all->plus, mode, speed));
197 18048200 : set_neg_cost (speed, mode, set_src_cost (all->neg, mode, speed));
198 18048200 : set_mul_cost (speed, mode, set_src_cost (all->mult, mode, speed));
199 18048200 : set_sdiv_cost (speed, mode, set_src_cost (all->sdiv, mode, speed));
200 18048200 : set_udiv_cost (speed, mode, set_src_cost (all->udiv, mode, speed));
201 :
202 18048200 : set_sdiv_pow2_cheap (speed, mode, (set_src_cost (all->sdiv_32, mode, speed)
203 18048200 : <= 2 * add_cost (speed, mode)));
204 18048200 : set_smod_pow2_cheap (speed, mode, (set_src_cost (all->smod_32, mode, speed)
205 18048200 : <= 4 * add_cost (speed, mode)));
206 :
207 18048200 : set_shift_cost (speed, mode, 0, 0);
208 18048200 : {
209 18048200 : int cost = add_cost (speed, mode);
210 18048200 : set_shiftadd_cost (speed, mode, 0, cost);
211 18048200 : set_shiftsub0_cost (speed, mode, 0, cost);
212 18048200 : set_shiftsub1_cost (speed, mode, 0, cost);
213 : }
214 :
215 18048200 : n = MIN (MAX_BITS_PER_WORD, mode_bitsize);
216 605715200 : for (m = 1; m < n; m++)
217 : {
218 587667000 : XEXP (all->shift, 1) = all->cint[m];
219 587667000 : XEXP (all->shift_mult, 1) = all->pow2[m];
220 :
221 587667000 : set_shift_cost (speed, mode, m, set_src_cost (all->shift, mode, speed));
222 587667000 : set_shiftadd_cost (speed, mode, m, set_src_cost (all->shift_add, mode,
223 : speed));
224 587667000 : set_shiftsub0_cost (speed, mode, m, set_src_cost (all->shift_sub0, mode,
225 : speed));
226 587667000 : set_shiftsub1_cost (speed, mode, m, set_src_cost (all->shift_sub1, mode,
227 : speed));
228 : }
229 :
230 18048200 : scalar_int_mode int_mode_to;
231 18048200 : if (is_a <scalar_int_mode> (mode, &int_mode_to))
232 : {
233 35216000 : for (mode_from = MIN_MODE_INT; mode_from <= MAX_MODE_INT;
234 30814000 : mode_from = (machine_mode)(mode_from + 1))
235 30814000 : init_expmed_one_conv (all, int_mode_to,
236 : as_a <scalar_int_mode> (mode_from), speed);
237 :
238 4402000 : scalar_int_mode wider_mode;
239 4402000 : if (GET_MODE_CLASS (int_mode_to) == MODE_INT
240 4402000 : && GET_MODE_WIDER_MODE (int_mode_to).exists (&wider_mode))
241 : {
242 2641200 : PUT_MODE (all->reg, mode);
243 2641200 : PUT_MODE (all->zext, wider_mode);
244 2641200 : PUT_MODE (all->wide_mult, wider_mode);
245 2641200 : PUT_MODE (all->wide_lshr, wider_mode);
246 2641200 : XEXP (all->wide_lshr, 1)
247 2641200 : = gen_int_shift_amount (wider_mode, mode_bitsize);
248 :
249 2641200 : set_mul_widen_cost (speed, wider_mode,
250 : set_src_cost (all->wide_mult, wider_mode, speed));
251 2641200 : set_mul_highpart_cost (speed, int_mode_to,
252 : set_src_cost (all->wide_trunc,
253 : int_mode_to, speed));
254 : }
255 : }
256 18048200 : }
257 :
258 : void
259 220100 : init_expmed (void)
260 : {
261 220100 : struct init_expmed_rtl all;
262 220100 : machine_mode mode = QImode;
263 220100 : int m, speed;
264 :
265 220100 : memset (&all, 0, sizeof all);
266 14086400 : for (m = 1; m < MAX_BITS_PER_WORD; m++)
267 : {
268 13866300 : all.pow2[m] = GEN_INT (HOST_WIDE_INT_1 << m);
269 13866300 : all.cint[m] = GEN_INT (m);
270 : }
271 :
272 : /* Avoid using hard regs in ways which may be unsupported. */
273 220100 : all.reg = gen_raw_REG (mode, LAST_VIRTUAL_REGISTER + 1);
274 220100 : all.plus = gen_rtx_PLUS (mode, all.reg, all.reg);
275 220100 : all.neg = gen_rtx_NEG (mode, all.reg);
276 220100 : all.mult = gen_rtx_MULT (mode, all.reg, all.reg);
277 220100 : all.sdiv = gen_rtx_DIV (mode, all.reg, all.reg);
278 220100 : all.udiv = gen_rtx_UDIV (mode, all.reg, all.reg);
279 220100 : all.sdiv_32 = gen_rtx_DIV (mode, all.reg, all.pow2[5]);
280 220100 : all.smod_32 = gen_rtx_MOD (mode, all.reg, all.pow2[5]);
281 220100 : all.zext = gen_rtx_ZERO_EXTEND (mode, all.reg);
282 220100 : all.wide_mult = gen_rtx_MULT (mode, all.zext, all.zext);
283 220100 : all.wide_lshr = gen_rtx_LSHIFTRT (mode, all.wide_mult, all.reg);
284 220100 : all.wide_trunc = gen_rtx_TRUNCATE (mode, all.wide_lshr);
285 220100 : all.shift = gen_rtx_ASHIFT (mode, all.reg, all.reg);
286 220100 : all.shift_mult = gen_rtx_MULT (mode, all.reg, all.reg);
287 220100 : all.shift_add = gen_rtx_PLUS (mode, all.shift_mult, all.reg);
288 220100 : all.shift_sub0 = gen_rtx_MINUS (mode, all.shift_mult, all.reg);
289 220100 : all.shift_sub1 = gen_rtx_MINUS (mode, all.reg, all.shift_mult);
290 220100 : all.trunc = gen_rtx_TRUNCATE (mode, all.reg);
291 :
292 660300 : for (speed = 0; speed < 2; speed++)
293 : {
294 440200 : crtl->maybe_hot_insn_p = speed;
295 440200 : set_zero_cost (speed, set_src_cost (const0_rtx, QImode, speed));
296 :
297 3521600 : for (mode = MIN_MODE_INT; mode <= MAX_MODE_INT;
298 3081400 : mode = (machine_mode)(mode + 1))
299 3081400 : init_expmed_one_mode (&all, mode, speed);
300 :
301 : if (MIN_MODE_PARTIAL_INT != VOIDmode)
302 1760800 : for (mode = MIN_MODE_PARTIAL_INT; mode <= MAX_MODE_PARTIAL_INT;
303 1320600 : mode = (machine_mode)(mode + 1))
304 1320600 : init_expmed_one_mode (&all, mode, speed);
305 :
306 : if (MIN_MODE_VECTOR_INT != VOIDmode)
307 14086400 : for (mode = MIN_MODE_VECTOR_INT; mode <= MAX_MODE_VECTOR_INT;
308 13646200 : mode = (machine_mode)(mode + 1))
309 13646200 : init_expmed_one_mode (&all, mode, speed);
310 : }
311 :
312 220100 : if (alg_hash_used_p ())
313 : {
314 1057 : struct alg_hash_entry *p = alg_hash_entry_ptr (0);
315 1057 : memset (p, 0, sizeof (*p) * NUM_ALG_HASH_ENTRIES);
316 : }
317 : else
318 219043 : set_alg_hash_used_p (true);
319 220100 : default_rtl_profile ();
320 :
321 220100 : ggc_free (all.trunc);
322 220100 : ggc_free (all.shift_sub1);
323 220100 : ggc_free (all.shift_sub0);
324 220100 : ggc_free (all.shift_add);
325 220100 : ggc_free (all.shift_mult);
326 220100 : ggc_free (all.shift);
327 220100 : ggc_free (all.wide_trunc);
328 220100 : ggc_free (all.wide_lshr);
329 220100 : ggc_free (all.wide_mult);
330 220100 : ggc_free (all.zext);
331 220100 : ggc_free (all.smod_32);
332 220100 : ggc_free (all.sdiv_32);
333 220100 : ggc_free (all.udiv);
334 220100 : ggc_free (all.sdiv);
335 220100 : ggc_free (all.mult);
336 220100 : ggc_free (all.neg);
337 220100 : ggc_free (all.plus);
338 220100 : ggc_free (all.reg);
339 220100 : }
340 :
341 : /* Return an rtx representing minus the value of X.
342 : MODE is the intended mode of the result,
343 : useful if X is a CONST_INT. */
344 :
345 : rtx
346 1038324 : negate_rtx (machine_mode mode, rtx x)
347 : {
348 1038324 : rtx result = simplify_unary_operation (NEG, mode, x, mode);
349 :
350 1038324 : if (result == 0)
351 1358 : result = expand_unop (mode, neg_optab, x, NULL_RTX, 0);
352 :
353 1038324 : return result;
354 : }
355 :
356 : /* Whether reverse storage order is supported on the target. */
357 : static int reverse_storage_order_supported = -1;
358 :
359 : /* Check whether reverse storage order is supported on the target. */
360 :
361 : static void
362 293 : check_reverse_storage_order_support (void)
363 : {
364 293 : if (BYTES_BIG_ENDIAN != WORDS_BIG_ENDIAN)
365 : {
366 : reverse_storage_order_supported = 0;
367 : sorry ("reverse scalar storage order");
368 : }
369 : else
370 293 : reverse_storage_order_supported = 1;
371 0 : }
372 :
373 : /* Whether reverse FP storage order is supported on the target. */
374 : static int reverse_float_storage_order_supported = -1;
375 :
376 : /* Check whether reverse FP storage order is supported on the target. */
377 :
378 : static void
379 55 : check_reverse_float_storage_order_support (void)
380 : {
381 55 : if (FLOAT_WORDS_BIG_ENDIAN != WORDS_BIG_ENDIAN)
382 : {
383 : reverse_float_storage_order_supported = 0;
384 : sorry ("reverse floating-point scalar storage order");
385 : }
386 : else
387 55 : reverse_float_storage_order_supported = 1;
388 0 : }
389 :
390 : /* Return an rtx representing value of X with reverse storage order.
391 : MODE is the intended mode of the result,
392 : useful if X is a CONST_INT. */
393 :
394 : rtx
395 3321 : flip_storage_order (machine_mode mode, rtx x)
396 : {
397 3321 : scalar_int_mode int_mode;
398 3321 : rtx result;
399 :
400 3321 : if (mode == QImode)
401 : return x;
402 :
403 2441 : if (COMPLEX_MODE_P (mode))
404 : {
405 44 : rtx real = read_complex_part (x, false);
406 44 : rtx imag = read_complex_part (x, true);
407 :
408 88 : real = flip_storage_order (GET_MODE_INNER (mode), real);
409 88 : imag = flip_storage_order (GET_MODE_INNER (mode), imag);
410 :
411 44 : return gen_rtx_CONCAT (mode, real, imag);
412 : }
413 :
414 2397 : if (UNLIKELY (reverse_storage_order_supported < 0))
415 293 : check_reverse_storage_order_support ();
416 :
417 2397 : if (!is_a <scalar_int_mode> (mode, &int_mode))
418 : {
419 243 : if (FLOAT_MODE_P (mode)
420 243 : && UNLIKELY (reverse_float_storage_order_supported < 0))
421 55 : check_reverse_float_storage_order_support ();
422 :
423 243 : if (!int_mode_for_size (GET_MODE_PRECISION (mode), 0).exists (&int_mode)
424 243 : || !targetm.scalar_mode_supported_p (int_mode))
425 : {
426 0 : sorry ("reverse storage order for %smode", GET_MODE_NAME (mode));
427 0 : return x;
428 : }
429 243 : x = gen_lowpart (int_mode, x);
430 : }
431 :
432 2397 : result = simplify_unary_operation (BSWAP, int_mode, x, int_mode);
433 2397 : if (result == 0)
434 1073 : result = expand_unop (int_mode, bswap_optab, x, NULL_RTX, 1);
435 :
436 2397 : if (int_mode != mode)
437 243 : result = gen_lowpart (mode, result);
438 :
439 : return result;
440 : }
441 :
442 : /* If MODE is set, adjust bitfield memory MEM so that it points to the
443 : first unit of mode MODE that contains a bitfield of size BITSIZE at
444 : bit position BITNUM. If MODE is not set, return a BLKmode reference
445 : to every byte in the bitfield. Set *NEW_BITNUM to the bit position
446 : of the field within the new memory. */
447 :
448 : static rtx
449 424416 : narrow_bit_field_mem (rtx mem, opt_scalar_int_mode mode,
450 : unsigned HOST_WIDE_INT bitsize,
451 : unsigned HOST_WIDE_INT bitnum,
452 : unsigned HOST_WIDE_INT *new_bitnum)
453 : {
454 424416 : scalar_int_mode imode;
455 424416 : if (mode.exists (&imode))
456 : {
457 424416 : unsigned int unit = GET_MODE_BITSIZE (imode);
458 424416 : *new_bitnum = bitnum % unit;
459 424416 : HOST_WIDE_INT offset = (bitnum - *new_bitnum) / BITS_PER_UNIT;
460 424416 : return adjust_bitfield_address (mem, imode, offset);
461 : }
462 : else
463 : {
464 0 : *new_bitnum = bitnum % BITS_PER_UNIT;
465 0 : HOST_WIDE_INT offset = bitnum / BITS_PER_UNIT;
466 0 : HOST_WIDE_INT size = ((*new_bitnum + bitsize + BITS_PER_UNIT - 1)
467 0 : / BITS_PER_UNIT);
468 0 : return adjust_bitfield_address_size (mem, BLKmode, offset, size);
469 : }
470 : }
471 :
472 : /* The caller wants to perform insertion or extraction PATTERN on a
473 : bitfield of size BITSIZE at BITNUM bits into memory operand OP0.
474 : BITREGION_START and BITREGION_END are as for store_bit_field
475 : and FIELDMODE is the natural mode of the field.
476 :
477 : Search for a mode that is compatible with the memory access
478 : restrictions and (where applicable) with a register insertion or
479 : extraction. Return the new memory on success, storing the adjusted
480 : bit position in *NEW_BITNUM. Return null otherwise. */
481 :
482 : static rtx
483 205036 : adjust_bit_field_mem_for_reg (enum extraction_pattern pattern,
484 : rtx op0, HOST_WIDE_INT bitsize,
485 : HOST_WIDE_INT bitnum,
486 : poly_uint64 bitregion_start,
487 : poly_uint64 bitregion_end,
488 : machine_mode fieldmode,
489 : unsigned HOST_WIDE_INT *new_bitnum)
490 : {
491 410072 : bit_field_mode_iterator iter (bitsize, bitnum, bitregion_start,
492 410072 : bitregion_end, MEM_ALIGN (op0),
493 410072 : MEM_VOLATILE_P (op0));
494 205036 : scalar_int_mode best_mode;
495 205036 : if (iter.next_mode (&best_mode))
496 : {
497 : /* We can use a memory in BEST_MODE. See whether this is true for
498 : any wider modes. All other things being equal, we prefer to
499 : use the widest mode possible because it tends to expose more
500 : CSE opportunities. */
501 198533 : if (!iter.prefer_smaller_modes ())
502 : {
503 : /* Limit the search to the mode required by the corresponding
504 : register insertion or extraction instruction, if any. */
505 317 : scalar_int_mode limit_mode = word_mode;
506 317 : extraction_insn insn;
507 634 : if (get_best_reg_extraction_insn (&insn, pattern,
508 317 : GET_MODE_BITSIZE (best_mode),
509 : fieldmode))
510 317 : limit_mode = insn.field_mode;
511 :
512 317 : scalar_int_mode wider_mode;
513 317 : while (iter.next_mode (&wider_mode)
514 1052 : && GET_MODE_SIZE (wider_mode) <= GET_MODE_SIZE (limit_mode))
515 111 : best_mode = wider_mode;
516 : }
517 198533 : return narrow_bit_field_mem (op0, best_mode, bitsize, bitnum,
518 : new_bitnum);
519 : }
520 : return NULL_RTX;
521 : }
522 :
523 : /* Return true if a bitfield of size BITSIZE at bit number BITNUM within
524 : a structure of mode STRUCT_MODE represents a lowpart subreg. The subreg
525 : offset is then BITNUM / BITS_PER_UNIT. */
526 :
527 : static bool
528 830359 : lowpart_bit_field_p (poly_uint64 bitnum, poly_uint64 bitsize,
529 : machine_mode struct_mode)
530 : {
531 830359 : poly_uint64 regsize = REGMODE_NATURAL_SIZE (struct_mode);
532 830359 : if (BYTES_BIG_ENDIAN)
533 : return (multiple_p (bitnum, BITS_PER_UNIT)
534 : && (known_eq (bitnum + bitsize, GET_MODE_BITSIZE (struct_mode))
535 : || multiple_p (bitnum + bitsize,
536 : regsize * BITS_PER_UNIT)));
537 : else
538 830359 : return multiple_p (bitnum, regsize * BITS_PER_UNIT);
539 : }
540 :
541 : /* Return true if -fstrict-volatile-bitfields applies to an access of OP0
542 : containing BITSIZE bits starting at BITNUM, with field mode FIELDMODE.
543 : Return false if the access would touch memory outside the range
544 : BITREGION_START to BITREGION_END for conformance to the C++ memory
545 : model. */
546 :
547 : static bool
548 1597954 : strict_volatile_bitfield_p (rtx op0, unsigned HOST_WIDE_INT bitsize,
549 : unsigned HOST_WIDE_INT bitnum,
550 : scalar_int_mode fieldmode,
551 : poly_uint64 bitregion_start,
552 : poly_uint64 bitregion_end)
553 : {
554 1597954 : unsigned HOST_WIDE_INT modesize = GET_MODE_BITSIZE (fieldmode);
555 :
556 : /* -fstrict-volatile-bitfields must be enabled and we must have a
557 : volatile MEM. */
558 1597954 : if (!MEM_P (op0)
559 165649 : || !MEM_VOLATILE_P (op0)
560 1598166 : || flag_strict_volatile_bitfields <= 0)
561 : return false;
562 :
563 : /* The bit size must not be larger than the field mode, and
564 : the field mode must not be larger than a word. */
565 14 : if (bitsize > modesize || modesize > BITS_PER_WORD)
566 : return false;
567 :
568 : /* Check for cases of unaligned fields that must be split. */
569 14 : if (bitnum % modesize + bitsize > modesize)
570 : return false;
571 :
572 : /* The memory must be sufficiently aligned for a MODESIZE access.
573 : This condition guarantees, that the memory access will not
574 : touch anything after the end of the structure. */
575 11 : if (MEM_ALIGN (op0) < modesize)
576 : return false;
577 :
578 : /* Check for cases where the C++ memory model applies. */
579 11 : if (maybe_ne (bitregion_end, 0U)
580 11 : && (maybe_lt (bitnum - bitnum % modesize, bitregion_start)
581 4 : || maybe_gt (bitnum - bitnum % modesize + modesize - 1,
582 : bitregion_end)))
583 0 : return false;
584 :
585 : return true;
586 : }
587 :
588 : /* Return true if OP is a memory and if a bitfield of size BITSIZE at
589 : bit number BITNUM can be treated as a simple value of mode MODE.
590 : Store the byte offset in *BYTENUM if so. */
591 :
592 : static bool
593 555877 : simple_mem_bitfield_p (rtx op0, poly_uint64 bitsize, poly_uint64 bitnum,
594 : machine_mode mode, poly_uint64 *bytenum)
595 : {
596 555877 : return (MEM_P (op0)
597 259901 : && multiple_p (bitnum, BITS_PER_UNIT, bytenum)
598 208104 : && known_eq (bitsize, GET_MODE_BITSIZE (mode))
599 607851 : && (!targetm.slow_unaligned_access (mode, MEM_ALIGN (op0))
600 0 : || (multiple_p (bitnum, GET_MODE_ALIGNMENT (mode))
601 0 : && MEM_ALIGN (op0) >= GET_MODE_ALIGNMENT (mode))));
602 : }
603 :
604 : /* Try to use instruction INSV to store VALUE into a field of OP0.
605 : If OP0_MODE is defined, it is the mode of OP0, otherwise OP0 is a
606 : BLKmode MEM. VALUE_MODE is the mode of VALUE. BITSIZE and BITNUM
607 : are as for store_bit_field. */
608 :
609 : static bool
610 109227 : store_bit_field_using_insv (const extraction_insn *insv, rtx op0,
611 : opt_scalar_int_mode op0_mode,
612 : unsigned HOST_WIDE_INT bitsize,
613 : unsigned HOST_WIDE_INT bitnum,
614 : rtx value, scalar_int_mode value_mode)
615 : {
616 109227 : class expand_operand ops[4];
617 109227 : rtx value1;
618 109227 : rtx xop0 = op0;
619 109227 : rtx_insn *last = get_last_insn ();
620 109227 : bool copy_back = false;
621 :
622 109227 : scalar_int_mode op_mode = insv->field_mode;
623 109227 : unsigned int unit = GET_MODE_BITSIZE (op_mode);
624 109227 : if (bitsize == 0 || bitsize > unit)
625 : return false;
626 :
627 109214 : if (MEM_P (xop0))
628 : /* Get a reference to the first byte of the field. */
629 0 : xop0 = narrow_bit_field_mem (xop0, insv->struct_mode, bitsize, bitnum,
630 : &bitnum);
631 : else
632 : {
633 : /* Convert from counting within OP0 to counting in OP_MODE. */
634 109214 : if (BYTES_BIG_ENDIAN)
635 : bitnum += unit - GET_MODE_BITSIZE (op0_mode.require ());
636 :
637 : /* If xop0 is a register, we need it in OP_MODE
638 : to make it acceptable to the format of insv. */
639 109214 : if (GET_CODE (xop0) == SUBREG)
640 : {
641 : /* If such a SUBREG can't be created, give up. */
642 35298 : if (!validate_subreg (op_mode, GET_MODE (SUBREG_REG (xop0)),
643 35298 : SUBREG_REG (xop0), SUBREG_BYTE (xop0)))
644 : return false;
645 : /* We can't just change the mode, because this might clobber op0,
646 : and we will need the original value of op0 if insv fails. */
647 35298 : xop0 = gen_rtx_SUBREG (op_mode, SUBREG_REG (xop0),
648 35298 : SUBREG_BYTE (xop0));
649 : }
650 109214 : if (REG_P (xop0) && GET_MODE (xop0) != op_mode)
651 26955 : xop0 = gen_lowpart_SUBREG (op_mode, xop0);
652 : }
653 :
654 : /* If the destination is a paradoxical subreg such that we need a
655 : truncate to the inner mode, perform the insertion on a temporary and
656 : truncate the result to the original destination. Note that we can't
657 : just truncate the paradoxical subreg as (truncate:N (subreg:W (reg:N
658 : X) 0)) is (reg:N X). */
659 109214 : if (GET_CODE (xop0) == SUBREG
660 62253 : && REG_P (SUBREG_REG (xop0))
661 171467 : && !TRULY_NOOP_TRUNCATION_MODES_P (GET_MODE (SUBREG_REG (xop0)),
662 : op_mode))
663 : {
664 0 : rtx tem = gen_reg_rtx (op_mode);
665 0 : emit_move_insn (tem, xop0);
666 0 : xop0 = tem;
667 0 : copy_back = true;
668 : }
669 :
670 : /* There are similar overflow check at the start of store_bit_field_1,
671 : but that only check the situation where the field lies completely
672 : outside the register, while there do have situation where the field
673 : lies partially in the register, we need to adjust bitsize for this
674 : partial overflow situation. Without this fix, pr48335-2.c on big-endian
675 : will broken on those arch support bit insert instruction, like arm, aarch64
676 : etc. */
677 109214 : if (bitsize + bitnum > unit && bitnum < unit)
678 : {
679 2 : warning (OPT_Wextra, "write of %wu-bit data outside the bound of "
680 : "destination object, data truncated into %wu-bit",
681 : bitsize, unit - bitnum);
682 2 : bitsize = unit - bitnum;
683 : }
684 :
685 : /* If BITS_BIG_ENDIAN is zero on a BYTES_BIG_ENDIAN machine, we count
686 : "backwards" from the size of the unit we are inserting into.
687 : Otherwise, we count bits from the most significant on a
688 : BYTES/BITS_BIG_ENDIAN machine. */
689 :
690 109214 : if (BITS_BIG_ENDIAN != BYTES_BIG_ENDIAN)
691 : bitnum = unit - bitsize - bitnum;
692 :
693 : /* Convert VALUE to op_mode (which insv insn wants) in VALUE1. */
694 109214 : value1 = value;
695 109214 : if (value_mode != op_mode)
696 : {
697 140588 : if (GET_MODE_BITSIZE (value_mode) >= bitsize)
698 : {
699 70294 : rtx tmp;
700 : /* Optimization: Don't bother really extending VALUE
701 : if it has all the bits we will actually use. However,
702 : if we must narrow it, be sure we do it correctly. */
703 :
704 210882 : if (GET_MODE_SIZE (value_mode) < GET_MODE_SIZE (op_mode))
705 : {
706 32939 : tmp = simplify_subreg (op_mode, value1, value_mode, 0);
707 32939 : if (! tmp)
708 32354 : tmp = simplify_gen_subreg (op_mode,
709 : force_reg (value_mode, value1),
710 : value_mode, 0);
711 : }
712 : else
713 : {
714 37355 : if (targetm.mode_rep_extended (op_mode, value_mode) != UNKNOWN)
715 0 : tmp = simplify_gen_unary (TRUNCATE, op_mode,
716 : value1, value_mode);
717 : else
718 : {
719 37355 : tmp = gen_lowpart_if_possible (op_mode, value1);
720 37355 : if (! tmp)
721 0 : tmp = gen_lowpart (op_mode, force_reg (value_mode, value1));
722 : }
723 : }
724 : value1 = tmp;
725 : }
726 0 : else if (CONST_INT_P (value))
727 0 : value1 = gen_int_mode (INTVAL (value), op_mode);
728 : else
729 : /* Parse phase is supposed to make VALUE's data type
730 : match that of the component reference, which is a type
731 : at least as wide as the field; so VALUE should have
732 : a mode that corresponds to that type. */
733 0 : gcc_assert (CONSTANT_P (value));
734 : }
735 :
736 109214 : create_fixed_operand (&ops[0], xop0);
737 109214 : create_integer_operand (&ops[1], bitsize);
738 109214 : create_integer_operand (&ops[2], bitnum);
739 109214 : create_input_operand (&ops[3], value1, op_mode);
740 109214 : if (maybe_expand_insn (insv->icode, 4, ops))
741 : {
742 2098 : if (copy_back)
743 0 : convert_move (op0, xop0, true);
744 : return true;
745 : }
746 107116 : delete_insns_since (last);
747 107116 : return false;
748 : }
749 :
750 : /* A subroutine of store_bit_field, with the same arguments. Return true
751 : if the operation could be implemented.
752 :
753 : If FALLBACK_P is true, fall back to store_fixed_bit_field if we have
754 : no other way of implementing the operation. If FALLBACK_P is false,
755 : return false instead.
756 :
757 : if UNDEFINED_P is true then STR_RTX is undefined and may be set using
758 : a subreg instead. */
759 :
760 : static bool
761 901255 : store_bit_field_1 (rtx str_rtx, poly_uint64 bitsize, poly_uint64 bitnum,
762 : poly_uint64 bitregion_start, poly_uint64 bitregion_end,
763 : machine_mode fieldmode,
764 : rtx value, bool reverse, bool fallback_p, bool undefined_p)
765 : {
766 901255 : rtx op0 = str_rtx;
767 :
768 901261 : while (GET_CODE (op0) == SUBREG)
769 : {
770 6 : bitnum += subreg_memory_offset (op0) * BITS_PER_UNIT;
771 6 : op0 = SUBREG_REG (op0);
772 : }
773 :
774 : /* No action is needed if the target is a register and if the field
775 : lies completely outside that register. This can occur if the source
776 : code contains an out-of-bounds access to a small array. */
777 1727314 : if (REG_P (op0) && known_ge (bitnum, GET_MODE_BITSIZE (GET_MODE (op0))))
778 : return true;
779 :
780 : /* Use vec_set patterns for inserting parts of vectors whenever
781 : available. */
782 901252 : machine_mode outermode = GET_MODE (op0);
783 901252 : scalar_mode innermode = GET_MODE_INNER (outermode);
784 901252 : poly_uint64 pos;
785 899002 : if (VECTOR_MODE_P (outermode)
786 2627 : && !MEM_P (op0)
787 2615 : && optab_handler (vec_set_optab, outermode) != CODE_FOR_nothing
788 1420 : && fieldmode == innermode
789 1159 : && known_eq (bitsize, GET_MODE_PRECISION (innermode))
790 902411 : && multiple_p (bitnum, GET_MODE_PRECISION (innermode), &pos))
791 : {
792 1159 : class expand_operand ops[3];
793 1159 : enum insn_code icode = optab_handler (vec_set_optab, outermode);
794 :
795 1159 : create_fixed_operand (&ops[0], op0);
796 1159 : create_input_operand (&ops[1], value, innermode);
797 1159 : create_integer_operand (&ops[2], pos);
798 1159 : if (maybe_expand_insn (icode, 3, ops))
799 1159 : return true;
800 : }
801 :
802 : /* If the target is a register, overwriting the entire object, or storing
803 : a full-word or multi-word field can be done with just a SUBREG. */
804 900093 : if (!MEM_P (op0)
805 1724990 : && known_eq (bitsize, GET_MODE_BITSIZE (fieldmode)))
806 : {
807 : /* Use the subreg machinery either to narrow OP0 to the required
808 : words or to cope with mode punning between equal-sized modes.
809 : In the latter case, use subreg on the rhs side, not lhs. */
810 757902 : rtx sub;
811 757902 : poly_uint64 bytenum;
812 757902 : poly_uint64 regsize = REGMODE_NATURAL_SIZE (GET_MODE (op0));
813 757902 : if (known_eq (bitnum, 0U)
814 1140977 : && known_eq (bitsize, GET_MODE_BITSIZE (GET_MODE (op0))))
815 : {
816 61575 : sub = force_subreg (GET_MODE (op0), value, fieldmode, 0);
817 61575 : if (sub)
818 : {
819 61575 : if (reverse)
820 1 : sub = flip_storage_order (GET_MODE (op0), sub);
821 61575 : emit_move_insn (op0, sub);
822 61575 : return true;
823 : }
824 : }
825 885941 : else if (multiple_p (bitnum, BITS_PER_UNIT, &bytenum)
826 696320 : && (undefined_p
827 692666 : || (multiple_p (bitnum, regsize * BITS_PER_UNIT)
828 681293 : && multiple_p (bitsize, regsize * BITS_PER_UNIT)))
829 1297830 : && known_ge (GET_MODE_BITSIZE (GET_MODE (op0)), bitsize))
830 : {
831 648899 : sub = simplify_gen_subreg (fieldmode, op0, GET_MODE (op0), bytenum);
832 648899 : if (sub)
833 : {
834 648897 : if (reverse)
835 0 : value = flip_storage_order (fieldmode, value);
836 648897 : emit_move_insn (sub, value);
837 648897 : return true;
838 : }
839 : }
840 : }
841 :
842 : /* If the target is memory, storing any naturally aligned field can be
843 : done with a simple store. For targets that support fast unaligned
844 : memory, any naturally sized, unit aligned field can be done directly. */
845 189621 : poly_uint64 bytenum;
846 189621 : if (simple_mem_bitfield_p (op0, bitsize, bitnum, fieldmode, &bytenum))
847 : {
848 7027 : op0 = adjust_bitfield_address (op0, fieldmode, bytenum);
849 7027 : if (reverse)
850 0 : value = flip_storage_order (fieldmode, value);
851 7027 : emit_move_insn (op0, value);
852 7027 : return true;
853 : }
854 :
855 : /* It's possible we'll need to handle other cases here for
856 : polynomial bitnum and bitsize. */
857 :
858 : /* From here on we need to be looking at a fixed-size insertion. */
859 182594 : unsigned HOST_WIDE_INT ibitsize = bitsize.to_constant ();
860 182594 : unsigned HOST_WIDE_INT ibitnum = bitnum.to_constant ();
861 :
862 : /* Make sure we are playing with integral modes. Pun with subregs
863 : if we aren't. This must come after the entire register case above,
864 : since that case is valid for any mode. The following cases are only
865 : valid for integral modes. */
866 182594 : opt_scalar_int_mode op0_mode = int_mode_for_mode (GET_MODE (op0));
867 182594 : scalar_int_mode imode;
868 182594 : bool need_stack_p = false;
869 182594 : if (!op0_mode.exists (&imode) || imode != GET_MODE (op0))
870 : {
871 19963 : if (MEM_P (op0))
872 15284 : op0 = adjust_bitfield_address_size (op0, op0_mode.else_blk (),
873 : 0, MEM_SIZE (op0));
874 4679 : else if (!op0_mode.exists ()
875 4679 : || maybe_lt
876 4819 : ((unsigned) UNITS_PER_WORD,
877 4679 : (poly_uint64) REGMODE_NATURAL_SIZE (GET_MODE (op0))))
878 : {
879 0 : if (ibitnum == 0
880 0 : && known_eq (ibitsize, GET_MODE_BITSIZE (GET_MODE (op0)))
881 0 : && MEM_P (value)
882 0 : && !reverse)
883 : {
884 0 : value = adjust_address (value, GET_MODE (op0), 0);
885 0 : emit_move_insn (op0, value);
886 0 : return true;
887 : }
888 0 : if (!fallback_p)
889 : return false;
890 : need_stack_p = true;
891 : }
892 : else
893 4679 : op0 = gen_lowpart (op0_mode.require (), op0);
894 : }
895 162631 : else if (!MEM_P (op0)
896 273726 : && maybe_lt ((unsigned) UNITS_PER_WORD,
897 109746 : (poly_uint64) REGMODE_NATURAL_SIZE (GET_MODE (op0))))
898 : need_stack_p = true;
899 :
900 : /* With or without punning we might be faced with a mode that we cannot
901 : split into words. If so, spill OP0 to the stack and recurse.
902 : This happens at most once. */
903 19963 : if (need_stack_p)
904 : {
905 0 : rtx temp = assign_stack_temp (GET_MODE (op0),
906 0 : GET_MODE_SIZE (GET_MODE (op0)));
907 0 : emit_move_insn (temp, op0);
908 0 : store_bit_field_1 (temp, bitsize, bitnum, 0, 0, fieldmode, value,
909 : reverse, fallback_p, undefined_p);
910 0 : emit_move_insn (op0, temp);
911 0 : return true;
912 : }
913 :
914 182594 : return store_integral_bit_field (op0, op0_mode, ibitsize, ibitnum,
915 : bitregion_start, bitregion_end,
916 182594 : fieldmode, value, reverse, fallback_p);
917 : }
918 :
919 : /* Subroutine of store_bit_field_1, with the same arguments, except
920 : that BITSIZE and BITNUM are constant. Handle cases specific to
921 : integral modes. If OP0_MODE is defined, it is the mode of OP0,
922 : otherwise OP0 is a BLKmode MEM. */
923 :
924 : static bool
925 182594 : store_integral_bit_field (rtx op0, opt_scalar_int_mode op0_mode,
926 : unsigned HOST_WIDE_INT bitsize,
927 : unsigned HOST_WIDE_INT bitnum,
928 : poly_uint64 bitregion_start,
929 : poly_uint64 bitregion_end,
930 : machine_mode fieldmode,
931 : rtx value, bool reverse, bool fallback_p)
932 : {
933 : /* Storing an lsb-aligned field in a register
934 : can be done with a movstrict instruction. */
935 :
936 182594 : if (!MEM_P (op0)
937 114425 : && !reverse
938 375241 : && lowpart_bit_field_p (bitnum, bitsize, op0_mode.require ())
939 82626 : && known_eq (bitsize, GET_MODE_BITSIZE (fieldmode))
940 218644 : && optab_handler (movstrict_optab, fieldmode) != CODE_FOR_nothing)
941 : {
942 4377 : class expand_operand ops[2];
943 4377 : enum insn_code icode = optab_handler (movstrict_optab, fieldmode);
944 4377 : rtx arg0 = op0;
945 4377 : unsigned HOST_WIDE_INT subreg_off;
946 :
947 4377 : if (GET_CODE (arg0) == SUBREG)
948 : {
949 : /* Else we've got some float mode source being extracted into
950 : a different float mode destination -- this combination of
951 : subregs results in Severe Tire Damage. */
952 469 : gcc_assert (GET_MODE (SUBREG_REG (arg0)) == fieldmode
953 : || GET_MODE_CLASS (fieldmode) == MODE_INT
954 : || GET_MODE_CLASS (fieldmode) == MODE_PARTIAL_INT);
955 : arg0 = SUBREG_REG (arg0);
956 : }
957 :
958 4377 : subreg_off = bitnum / BITS_PER_UNIT;
959 4411 : if (validate_subreg (fieldmode, GET_MODE (arg0), arg0, subreg_off)
960 : /* STRICT_LOW_PART must have a non-paradoxical subreg as
961 : operand. */
962 4377 : && !paradoxical_subreg_p (fieldmode, GET_MODE (arg0)))
963 : {
964 4343 : arg0 = gen_rtx_SUBREG (fieldmode, arg0, subreg_off);
965 :
966 4343 : create_fixed_operand (&ops[0], arg0);
967 : /* Shrink the source operand to FIELDMODE. */
968 4343 : create_convert_operand_to (&ops[1], value, fieldmode, false);
969 4343 : if (maybe_expand_insn (icode, 2, ops))
970 4342 : return true;
971 : }
972 : }
973 :
974 : /* Handle fields bigger than a word. */
975 :
976 179853 : if (bitsize > BITS_PER_WORD)
977 : {
978 : /* Here we transfer the words of the field
979 : in the order least significant first.
980 : This is because the most significant word is the one which may
981 : be less than full.
982 : However, only do that if the value is not BLKmode. */
983 :
984 916 : const bool backwards = WORDS_BIG_ENDIAN && fieldmode != BLKmode;
985 916 : const int nwords = (bitsize + (BITS_PER_WORD - 1)) / BITS_PER_WORD;
986 916 : rtx_insn *last;
987 :
988 : /* This is the mode we must force value to, so that there will be enough
989 : subwords to extract. Note that fieldmode will often (always?) be
990 : VOIDmode, because that is what store_field uses to indicate that this
991 : is a bit field, but passing VOIDmode to operand_subword_force
992 : is not allowed.
993 :
994 : The mode must be fixed-size, since insertions into variable-sized
995 : objects are meant to be handled before calling this function. */
996 916 : fixed_size_mode value_mode = as_a <fixed_size_mode> (GET_MODE (value));
997 916 : if (value_mode == VOIDmode)
998 24 : value_mode
999 24 : = smallest_int_mode_for_size (nwords * BITS_PER_WORD).require ();
1000 :
1001 916 : last = get_last_insn ();
1002 2743 : for (int i = 0; i < nwords; i++)
1003 : {
1004 : /* Number of bits to be stored in this iteration, i.e. BITS_PER_WORD
1005 : except maybe for the last iteration. */
1006 3664 : const unsigned HOST_WIDE_INT new_bitsize
1007 1868 : = MIN (BITS_PER_WORD, bitsize - i * BITS_PER_WORD);
1008 : /* Bit offset from the starting bit number in the target. */
1009 3676 : const unsigned int bit_offset
1010 : = backwards ^ reverse
1011 1832 : ? MAX ((int) bitsize - (i + 1) * BITS_PER_WORD, 0)
1012 : : i * BITS_PER_WORD;
1013 :
1014 : /* No further action is needed if the target is a register and if
1015 : this field lies completely outside that register. */
1016 2104 : if (REG_P (op0) && known_ge (bitnum + bit_offset,
1017 : GET_MODE_BITSIZE (GET_MODE (op0))))
1018 : {
1019 5 : if (backwards ^ reverse)
1020 0 : continue;
1021 : /* For forward operation we are finished. */
1022 182594 : return true;
1023 : }
1024 :
1025 : /* Starting word number in the value. */
1026 1827 : const unsigned int wordnum
1027 : = backwards
1028 1827 : ? GET_MODE_SIZE (value_mode) / UNITS_PER_WORD - (i + 1)
1029 : : i;
1030 : /* The chunk of the value in word_mode. We use bit-field extraction
1031 : in BLKmode to handle unaligned memory references and to shift the
1032 : last chunk right on big-endian machines if need be. */
1033 1827 : rtx value_word
1034 : = (fieldmode == BLKmode
1035 1578 : || maybe_lt ((unsigned) UNITS_PER_WORD,
1036 1578 : (poly_uint64) REGMODE_NATURAL_SIZE (value_mode)))
1037 1863 : ? extract_bit_field (value, new_bitsize, wordnum * BITS_PER_WORD,
1038 : 1, NULL_RTX, word_mode, word_mode, false,
1039 : NULL)
1040 1827 : : operand_subword_force (value, wordnum, value_mode);
1041 :
1042 1827 : if (!store_bit_field_1 (op0, new_bitsize,
1043 1827 : bitnum + bit_offset,
1044 : bitregion_start, bitregion_end,
1045 : word_mode,
1046 : value_word, reverse, fallback_p, false))
1047 : {
1048 0 : delete_insns_since (last);
1049 0 : return false;
1050 : }
1051 : }
1052 : return true;
1053 : }
1054 :
1055 : /* If VALUE has a floating-point or complex mode, access it as an
1056 : integer of the corresponding size. This can occur on a machine
1057 : with 64 bit registers that uses SFmode for float. It can also
1058 : occur for unaligned float or complex fields. */
1059 177336 : rtx orig_value = value;
1060 177336 : scalar_int_mode value_mode;
1061 177336 : if (GET_MODE (value) == VOIDmode)
1062 : /* By this point we've dealt with values that are bigger than a word,
1063 : so word_mode is a conservatively correct choice. */
1064 105629 : value_mode = word_mode;
1065 71707 : else if (!is_a <scalar_int_mode> (GET_MODE (value), &value_mode))
1066 : {
1067 1130 : value_mode = int_mode_for_mode (GET_MODE (value)).require ();
1068 1130 : value = gen_reg_rtx (value_mode);
1069 1130 : emit_move_insn (gen_lowpart (GET_MODE (orig_value), value), orig_value);
1070 : }
1071 :
1072 : /* If OP0 is a multi-word register, narrow it to the affected word.
1073 : If the region spans two words, defer to store_split_bit_field.
1074 : Don't do this if op0 is a single hard register wider than word
1075 : such as a float or vector register. */
1076 177336 : if (!MEM_P (op0)
1077 220801 : && GET_MODE_SIZE (op0_mode.require ()) > UNITS_PER_WORD
1078 212685 : && (!REG_P (op0)
1079 35328 : || !HARD_REGISTER_P (op0)
1080 177336 : || hard_regno_nregs (REGNO (op0), op0_mode.require ()) != 1))
1081 : {
1082 35422 : if (bitnum % BITS_PER_WORD + bitsize > BITS_PER_WORD)
1083 : {
1084 698 : if (!fallback_p)
1085 : return false;
1086 :
1087 71 : store_split_bit_field (op0, op0_mode, bitsize, bitnum,
1088 : bitregion_start, bitregion_end,
1089 : value, value_mode, reverse);
1090 71 : return true;
1091 : }
1092 34649 : rtx new_op0
1093 34649 : = simplify_gen_subreg (word_mode, op0, op0_mode.require (),
1094 34724 : bitnum / BITS_PER_WORD * UNITS_PER_WORD);
1095 34649 : if (!new_op0)
1096 : {
1097 : /* No valid word-mode SUBREG of op0 at this offset. Defer to
1098 : store_split_bit_field, which addresses op0 a word at a time. */
1099 0 : if (!fallback_p)
1100 : return false;
1101 0 : store_split_bit_field (op0, op0_mode, bitsize, bitnum,
1102 : bitregion_start, bitregion_end,
1103 : value, value_mode, reverse);
1104 0 : return true;
1105 : }
1106 34649 : op0 = new_op0;
1107 34649 : op0_mode = word_mode;
1108 34724 : bitnum %= BITS_PER_WORD;
1109 : }
1110 :
1111 : /* From here on we can assume that the field to be stored in fits
1112 : within a word. If the destination is a register, it too fits
1113 : in a word. */
1114 :
1115 176638 : extraction_insn insv;
1116 176638 : if (!MEM_P (op0)
1117 109246 : && !reverse
1118 109229 : && get_best_reg_extraction_insn (&insv, EP_insv,
1119 218458 : GET_MODE_BITSIZE (op0_mode.require ()),
1120 : fieldmode)
1121 285865 : && store_bit_field_using_insv (&insv, op0, op0_mode,
1122 : bitsize, bitnum, value, value_mode))
1123 2098 : return true;
1124 :
1125 : /* If OP0 is a memory, try copying it to a register and seeing if a
1126 : cheap register alternative is available. */
1127 174540 : if (MEM_P (op0) && !reverse)
1128 : {
1129 66992 : if (get_best_mem_extraction_insn (&insv, EP_insv, bitsize, bitnum,
1130 : fieldmode)
1131 66992 : && store_bit_field_using_insv (&insv, op0, op0_mode,
1132 : bitsize, bitnum, value, value_mode))
1133 0 : return true;
1134 :
1135 66992 : rtx_insn *last = get_last_insn ();
1136 :
1137 : /* Try loading part of OP0 into a register, inserting the bitfield
1138 : into that, and then copying the result back to OP0. */
1139 66992 : unsigned HOST_WIDE_INT bitpos;
1140 66992 : rtx xop0 = adjust_bit_field_mem_for_reg (EP_insv, op0, bitsize, bitnum,
1141 : bitregion_start, bitregion_end,
1142 : fieldmode, &bitpos);
1143 66992 : if (xop0)
1144 : {
1145 63155 : rtx tempreg = copy_to_reg (xop0);
1146 63155 : if (store_bit_field_1 (tempreg, bitsize, bitpos,
1147 : bitregion_start, bitregion_end,
1148 : fieldmode, orig_value, reverse, false, false))
1149 : {
1150 0 : emit_move_insn (xop0, tempreg);
1151 0 : return true;
1152 : }
1153 63155 : delete_insns_since (last);
1154 : }
1155 : }
1156 :
1157 174540 : if (!fallback_p)
1158 : return false;
1159 :
1160 112012 : store_fixed_bit_field (op0, op0_mode, bitsize, bitnum, bitregion_start,
1161 : bitregion_end, value, value_mode, reverse);
1162 112012 : return true;
1163 : }
1164 :
1165 : /* Generate code to store value from rtx VALUE
1166 : into a bit-field within structure STR_RTX
1167 : containing BITSIZE bits starting at bit BITNUM.
1168 :
1169 : BITREGION_START is bitpos of the first bitfield in this region.
1170 : BITREGION_END is the bitpos of the ending bitfield in this region.
1171 : These two fields are 0, if the C++ memory model does not apply,
1172 : or we are not interested in keeping track of bitfield regions.
1173 :
1174 : FIELDMODE is the machine-mode of the FIELD_DECL node for this field.
1175 :
1176 : If REVERSE is true, the store is to be done in reverse order.
1177 :
1178 : If UNDEFINED_P is true then STR_RTX is currently undefined. */
1179 :
1180 : void
1181 836273 : store_bit_field (rtx str_rtx, poly_uint64 bitsize, poly_uint64 bitnum,
1182 : poly_uint64 bitregion_start, poly_uint64 bitregion_end,
1183 : machine_mode fieldmode,
1184 : rtx value, bool reverse, bool undefined_p)
1185 : {
1186 : /* Handle -fstrict-volatile-bitfields in the cases where it applies. */
1187 836273 : unsigned HOST_WIDE_INT ibitsize = 0, ibitnum = 0;
1188 836273 : scalar_int_mode int_mode;
1189 836273 : if (bitsize.is_constant (&ibitsize)
1190 836273 : && bitnum.is_constant (&ibitnum)
1191 1576644 : && is_a <scalar_int_mode> (fieldmode, &int_mode)
1192 740375 : && strict_volatile_bitfield_p (str_rtx, ibitsize, ibitnum, int_mode,
1193 : bitregion_start, bitregion_end))
1194 : {
1195 : /* Storing of a full word can be done with a simple store.
1196 : We know here that the field can be accessed with one single
1197 : instruction. For targets that support unaligned memory,
1198 : an unaligned access may be necessary. */
1199 8 : if (ibitsize == GET_MODE_BITSIZE (int_mode))
1200 : {
1201 0 : str_rtx = adjust_bitfield_address (str_rtx, int_mode,
1202 : ibitnum / BITS_PER_UNIT);
1203 0 : if (reverse)
1204 0 : value = flip_storage_order (int_mode, value);
1205 0 : gcc_assert (ibitnum % BITS_PER_UNIT == 0);
1206 0 : emit_move_insn (str_rtx, value);
1207 : }
1208 : else
1209 : {
1210 4 : rtx temp;
1211 :
1212 4 : str_rtx = narrow_bit_field_mem (str_rtx, int_mode, ibitsize,
1213 : ibitnum, &ibitnum);
1214 8 : gcc_assert (ibitnum + ibitsize <= GET_MODE_BITSIZE (int_mode));
1215 4 : temp = copy_to_reg (str_rtx);
1216 4 : if (!store_bit_field_1 (temp, ibitsize, ibitnum, 0, 0,
1217 : int_mode, value, reverse, true, undefined_p))
1218 0 : gcc_unreachable ();
1219 :
1220 4 : emit_move_insn (str_rtx, temp);
1221 : }
1222 :
1223 4 : return;
1224 : }
1225 :
1226 : /* Under the C++0x memory model, we must not touch bits outside the
1227 : bit region. Adjust the address to start at the beginning of the
1228 : bit region. */
1229 836269 : if (MEM_P (str_rtx) && maybe_ne (bitregion_start, 0U))
1230 : {
1231 51065 : scalar_int_mode best_mode;
1232 51065 : machine_mode addr_mode = VOIDmode;
1233 :
1234 51065 : poly_uint64 offset = exact_div (bitregion_start, BITS_PER_UNIT);
1235 51065 : bitnum -= bitregion_start;
1236 51065 : poly_int64 size = bits_to_bytes_round_up (bitnum + bitsize);
1237 51065 : bitregion_end -= bitregion_start;
1238 51065 : bitregion_start = 0;
1239 51065 : if (bitsize.is_constant (&ibitsize)
1240 51065 : && bitnum.is_constant (&ibitnum)
1241 51065 : && get_best_mode (ibitsize, ibitnum,
1242 : bitregion_start, bitregion_end,
1243 51065 : MEM_ALIGN (str_rtx), INT_MAX,
1244 51065 : MEM_VOLATILE_P (str_rtx), &best_mode))
1245 47651 : addr_mode = best_mode;
1246 51065 : str_rtx = adjust_bitfield_address_size (str_rtx, addr_mode,
1247 : offset, size);
1248 : }
1249 :
1250 836269 : if (!store_bit_field_1 (str_rtx, bitsize, bitnum,
1251 : bitregion_start, bitregion_end,
1252 : fieldmode, value, reverse, true, undefined_p))
1253 0 : gcc_unreachable ();
1254 : }
1255 :
1256 : /* Use shifts and boolean operations to store VALUE into a bit field of
1257 : width BITSIZE in OP0, starting at bit BITNUM. If OP0_MODE is defined,
1258 : it is the mode of OP0, otherwise OP0 is a BLKmode MEM. VALUE_MODE is
1259 : the mode of VALUE.
1260 :
1261 : If REVERSE is true, the store is to be done in reverse order. */
1262 :
1263 : static void
1264 129511 : store_fixed_bit_field (rtx op0, opt_scalar_int_mode op0_mode,
1265 : unsigned HOST_WIDE_INT bitsize,
1266 : unsigned HOST_WIDE_INT bitnum,
1267 : poly_uint64 bitregion_start, poly_uint64 bitregion_end,
1268 : rtx value, scalar_int_mode value_mode, bool reverse)
1269 : {
1270 : /* There is a case not handled here:
1271 : a structure with a known alignment of just a halfword
1272 : and a field split across two aligned halfwords within the structure.
1273 : Or likewise a structure with a known alignment of just a byte
1274 : and a field split across two bytes.
1275 : Such cases are not supposed to be able to occur. */
1276 :
1277 129511 : scalar_int_mode best_mode;
1278 129511 : if (MEM_P (op0))
1279 : {
1280 84749 : unsigned int max_bitsize = BITS_PER_WORD;
1281 84749 : scalar_int_mode imode;
1282 144640 : if (op0_mode.exists (&imode) && GET_MODE_BITSIZE (imode) < max_bitsize)
1283 90894 : max_bitsize = GET_MODE_BITSIZE (imode);
1284 :
1285 84749 : if (!get_best_mode (bitsize, bitnum, bitregion_start, bitregion_end,
1286 84749 : MEM_ALIGN (op0), max_bitsize, MEM_VOLATILE_P (op0),
1287 : &best_mode))
1288 : {
1289 : /* The only way this should occur is if the field spans word
1290 : boundaries. */
1291 6569 : store_split_bit_field (op0, op0_mode, bitsize, bitnum,
1292 : bitregion_start, bitregion_end,
1293 : value, value_mode, reverse);
1294 6569 : return;
1295 : }
1296 :
1297 78180 : op0 = narrow_bit_field_mem (op0, best_mode, bitsize, bitnum, &bitnum);
1298 : }
1299 : else
1300 44762 : best_mode = op0_mode.require ();
1301 :
1302 122942 : store_fixed_bit_field_1 (op0, best_mode, bitsize, bitnum,
1303 : value, value_mode, reverse);
1304 : }
1305 :
1306 : /* Helper function for store_fixed_bit_field, stores
1307 : the bit field always using MODE, which is the mode of OP0. The other
1308 : arguments are as for store_fixed_bit_field. */
1309 :
1310 : static void
1311 122942 : store_fixed_bit_field_1 (rtx op0, scalar_int_mode mode,
1312 : unsigned HOST_WIDE_INT bitsize,
1313 : unsigned HOST_WIDE_INT bitnum,
1314 : rtx value, scalar_int_mode value_mode, bool reverse)
1315 : {
1316 122942 : rtx temp;
1317 122942 : int all_zero = 0;
1318 122942 : int all_one = 0;
1319 :
1320 : /* Note that bitsize + bitnum can be greater than GET_MODE_BITSIZE (mode)
1321 : for invalid input, such as f5 from gcc.dg/pr48335-2.c. */
1322 :
1323 122942 : if (reverse ? !BYTES_BIG_ENDIAN : BYTES_BIG_ENDIAN)
1324 : /* BITNUM is the distance between our msb
1325 : and that of the containing datum.
1326 : Convert it to the distance from the lsb. */
1327 1058 : bitnum = GET_MODE_BITSIZE (mode) - bitsize - bitnum;
1328 :
1329 : /* Now BITNUM is always the distance between our lsb
1330 : and that of OP0. */
1331 :
1332 : /* Shift VALUE left by BITNUM bits. If VALUE is not constant,
1333 : we must first convert its mode to MODE. */
1334 :
1335 122942 : if (CONST_INT_P (value))
1336 : {
1337 73173 : unsigned HOST_WIDE_INT v = UINTVAL (value);
1338 :
1339 73173 : if (bitsize < HOST_BITS_PER_WIDE_INT)
1340 73156 : v &= (HOST_WIDE_INT_1U << bitsize) - 1;
1341 :
1342 73173 : if (v == 0)
1343 : all_zero = 1;
1344 59503 : else if ((bitsize < HOST_BITS_PER_WIDE_INT
1345 59495 : && v == (HOST_WIDE_INT_1U << bitsize) - 1)
1346 50962 : || (bitsize == HOST_BITS_PER_WIDE_INT
1347 50962 : && v == HOST_WIDE_INT_M1U))
1348 8541 : all_one = 1;
1349 :
1350 73173 : value = lshift_value (mode, v, bitnum);
1351 : }
1352 : else
1353 : {
1354 49769 : int must_and = (GET_MODE_BITSIZE (value_mode) != bitsize
1355 77220 : && bitnum + bitsize != GET_MODE_BITSIZE (mode));
1356 :
1357 49769 : if (value_mode != mode)
1358 27235 : value = convert_to_mode (mode, value, 1);
1359 :
1360 49769 : if (must_and)
1361 20646 : value = expand_binop (mode, and_optab, value,
1362 : mask_rtx (mode, 0, bitsize, 0),
1363 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
1364 49769 : if (bitnum > 0)
1365 14128 : value = expand_shift (LSHIFT_EXPR, mode, value,
1366 14128 : bitnum, NULL_RTX, 1);
1367 : }
1368 :
1369 122942 : if (reverse)
1370 529 : value = flip_storage_order (mode, value);
1371 :
1372 : /* A field filling the whole of a MEM has no surrounding bits to preserve,
1373 : so store it directly: the read of a volatile OP0 cannot be removed later
1374 : and would be a spurious access with side effects (PR71048). */
1375 182522 : if (MEM_P (op0) && bitnum == 0 && bitsize == GET_MODE_BITSIZE (mode))
1376 : {
1377 13639 : op0 = copy_rtx (op0);
1378 13639 : emit_move_insn (op0, value);
1379 13639 : return;
1380 : }
1381 :
1382 : /* Now clear the chosen bits in OP0,
1383 : except that if VALUE is -1 we need not bother. */
1384 : /* We keep the intermediates in registers to allow CSE to combine
1385 : consecutive bitfield assignments. */
1386 :
1387 109303 : temp = force_reg (mode, op0);
1388 :
1389 109303 : if (! all_one)
1390 : {
1391 101453 : rtx mask = mask_rtx (mode, bitnum, bitsize, 1);
1392 101453 : if (reverse)
1393 517 : mask = flip_storage_order (mode, mask);
1394 101453 : temp = expand_binop (mode, and_optab, temp, mask,
1395 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
1396 101453 : temp = force_reg (mode, temp);
1397 : }
1398 :
1399 : /* Now logical-or VALUE into OP0, unless it is zero. */
1400 :
1401 109303 : if (! all_zero)
1402 : {
1403 : /* Use aop_optab if the field has been cleared. */
1404 186984 : temp = expand_binop (mode, all_one ? ior_optab : aop_optab,
1405 : temp, value, NULL_RTX, 1, OPTAB_LIB_WIDEN);
1406 97417 : temp = force_reg (mode, temp);
1407 : }
1408 :
1409 109303 : if (op0 != temp)
1410 : {
1411 109303 : op0 = copy_rtx (op0);
1412 109303 : emit_move_insn (op0, temp);
1413 : }
1414 : }
1415 :
1416 : /* Store a bit field that is split across multiple accessible memory objects.
1417 :
1418 : OP0 is the REG, SUBREG or MEM rtx for the first of the objects.
1419 : BITSIZE is the field width; BITPOS the position of its first bit
1420 : (within the word).
1421 : VALUE is the value to store, which has mode VALUE_MODE.
1422 : If OP0_MODE is defined, it is the mode of OP0, otherwise OP0 is
1423 : a BLKmode MEM.
1424 :
1425 : If REVERSE is true, the store is to be done in reverse order.
1426 :
1427 : This does not yet handle fields wider than BITS_PER_WORD. */
1428 :
1429 : static void
1430 6640 : store_split_bit_field (rtx op0, opt_scalar_int_mode op0_mode,
1431 : unsigned HOST_WIDE_INT bitsize,
1432 : unsigned HOST_WIDE_INT bitpos,
1433 : poly_uint64 bitregion_start, poly_uint64 bitregion_end,
1434 : rtx value, scalar_int_mode value_mode, bool reverse)
1435 : {
1436 6640 : unsigned int unit, total_bits, bitsdone = 0;
1437 :
1438 : /* Make sure UNIT isn't larger than BITS_PER_WORD, we can only handle that
1439 : much at a time. */
1440 6640 : if (REG_P (op0) || GET_CODE (op0) == SUBREG)
1441 71 : unit = BITS_PER_WORD;
1442 : else
1443 6569 : unit = MIN (MEM_ALIGN (op0), BITS_PER_WORD);
1444 :
1445 : /* If OP0 is a memory with a mode, then UNIT must not be larger than
1446 : OP0's mode as well. Otherwise, store_fixed_bit_field will call us
1447 : again, and we will mutually recurse forever. */
1448 6640 : if (MEM_P (op0) && op0_mode.exists ())
1449 5111 : unit = MIN (unit, GET_MODE_BITSIZE (op0_mode.require ()));
1450 :
1451 : /* If VALUE is a constant other than a CONST_INT, get it into a register in
1452 : WORD_MODE. If we can do this using gen_lowpart_common, do so. Note
1453 : that VALUE might be a floating-point constant. */
1454 6640 : if (CONSTANT_P (value) && !CONST_INT_P (value))
1455 : {
1456 0 : rtx word = gen_lowpart_common (word_mode, value);
1457 :
1458 0 : if (word && (value != word))
1459 : value = word;
1460 : else
1461 0 : value = gen_lowpart_common (word_mode, force_reg (value_mode, value));
1462 0 : value_mode = word_mode;
1463 : }
1464 :
1465 6640 : total_bits = GET_MODE_BITSIZE (value_mode);
1466 :
1467 30941 : while (bitsdone < bitsize)
1468 : {
1469 24301 : unsigned HOST_WIDE_INT thissize;
1470 24301 : unsigned HOST_WIDE_INT thispos;
1471 24301 : unsigned HOST_WIDE_INT offset;
1472 24301 : rtx part;
1473 :
1474 24301 : offset = (bitpos + bitsdone) / unit;
1475 24301 : thispos = (bitpos + bitsdone) % unit;
1476 :
1477 : /* When region of bytes we can touch is restricted, decrease
1478 : UNIT close to the end of the region as needed. If op0 is a REG
1479 : or SUBREG of REG, don't do this, as there can't be data races
1480 : on a register and we can expand shorter code in some cases. */
1481 31103 : if (maybe_ne (bitregion_end, 0U)
1482 24301 : && unit > BITS_PER_UNIT
1483 14217 : && maybe_gt (bitpos + bitsdone - thispos + unit, bitregion_end + 1)
1484 6866 : && !REG_P (op0)
1485 31103 : && (GET_CODE (op0) != SUBREG || !REG_P (SUBREG_REG (op0))))
1486 : {
1487 6802 : unit = unit / 2;
1488 6802 : continue;
1489 : }
1490 :
1491 : /* THISSIZE must not overrun a word boundary. Otherwise,
1492 : store_fixed_bit_field will call us again, and we will mutually
1493 : recurse forever. */
1494 17499 : thissize = MIN (bitsize - bitsdone, BITS_PER_WORD);
1495 17499 : thissize = MIN (thissize, unit - thispos);
1496 :
1497 17499 : if (reverse ? !BYTES_BIG_ENDIAN : BYTES_BIG_ENDIAN)
1498 : {
1499 : /* Fetch successively less significant portions. */
1500 214 : if (CONST_INT_P (value))
1501 108 : part = GEN_INT (((unsigned HOST_WIDE_INT) (INTVAL (value))
1502 : >> (bitsize - bitsdone - thissize))
1503 : & ((HOST_WIDE_INT_1 << thissize) - 1));
1504 : /* Likewise, but the source is little-endian. */
1505 106 : else if (reverse)
1506 106 : part = extract_fixed_bit_field (word_mode, value, value_mode,
1507 : thissize,
1508 : bitsize - bitsdone - thissize,
1509 : NULL_RTX, 1, false);
1510 : else
1511 : /* The args are chosen so that the last part includes the
1512 : lsb. Give extract_bit_field the value it needs (with
1513 : endianness compensation) to fetch the piece we want. */
1514 : part = extract_fixed_bit_field (word_mode, value, value_mode,
1515 : thissize,
1516 : total_bits - bitsize + bitsdone,
1517 : NULL_RTX, 1, false);
1518 : }
1519 : else
1520 : {
1521 : /* Fetch successively more significant portions. */
1522 17285 : if (CONST_INT_P (value))
1523 12867 : part = GEN_INT (((unsigned HOST_WIDE_INT) (INTVAL (value))
1524 : >> bitsdone)
1525 : & ((HOST_WIDE_INT_1 << thissize) - 1));
1526 : /* Likewise, but the source is big-endian. */
1527 4418 : else if (reverse)
1528 : part = extract_fixed_bit_field (word_mode, value, value_mode,
1529 : thissize,
1530 : total_bits - bitsdone - thissize,
1531 : NULL_RTX, 1, false);
1532 : else
1533 4418 : part = extract_fixed_bit_field (word_mode, value, value_mode,
1534 : thissize, bitsdone, NULL_RTX,
1535 : 1, false);
1536 : }
1537 :
1538 : /* If OP0 is a register, then handle OFFSET here. */
1539 17499 : rtx op0_piece = op0;
1540 17499 : opt_scalar_int_mode op0_piece_mode = op0_mode;
1541 17499 : if (SUBREG_P (op0) || REG_P (op0))
1542 : {
1543 142 : scalar_int_mode imode;
1544 142 : if (op0_mode.exists (&imode)
1545 142 : && GET_MODE_SIZE (imode) < UNITS_PER_WORD)
1546 : {
1547 0 : if (offset)
1548 0 : op0_piece = const0_rtx;
1549 : }
1550 : else
1551 : {
1552 142 : op0_piece = operand_subword_force (op0,
1553 142 : offset * unit / BITS_PER_WORD,
1554 142 : GET_MODE (op0));
1555 142 : op0_piece_mode = word_mode;
1556 : }
1557 142 : offset &= BITS_PER_WORD / unit - 1;
1558 : }
1559 :
1560 : /* OFFSET is in UNITs, and UNIT is in bits. If WORD is const0_rtx,
1561 : it is just an out-of-bounds access. Ignore it. */
1562 17499 : if (op0_piece != const0_rtx)
1563 17499 : store_fixed_bit_field (op0_piece, op0_piece_mode, thissize,
1564 17499 : offset * unit + thispos, bitregion_start,
1565 : bitregion_end, part, word_mode, reverse);
1566 17499 : bitsdone += thissize;
1567 : }
1568 6640 : }
1569 :
1570 : /* A subroutine of extract_bit_field_1 that converts return value X
1571 : to either MODE or TMODE. MODE, TMODE and UNSIGNEDP are arguments
1572 : to extract_bit_field. */
1573 :
1574 : static rtx
1575 905997 : convert_extracted_bit_field (rtx x, machine_mode mode,
1576 : machine_mode tmode, bool unsignedp)
1577 : {
1578 905997 : if (GET_MODE (x) == tmode || GET_MODE (x) == mode)
1579 : return x;
1580 :
1581 : /* If the x mode is not a scalar integral, first convert to the
1582 : integer mode of that size and then access it as a floating-point
1583 : value via a SUBREG. */
1584 21970 : if (!SCALAR_INT_MODE_P (tmode))
1585 : {
1586 11912 : scalar_int_mode int_mode = int_mode_for_mode (tmode).require ();
1587 11912 : x = convert_to_mode (int_mode, x, unsignedp);
1588 11912 : x = force_reg (int_mode, x);
1589 11912 : return gen_lowpart (tmode, x);
1590 : }
1591 :
1592 10058 : return convert_to_mode (tmode, x, unsignedp);
1593 : }
1594 :
1595 : /* Try to use an ext(z)v pattern to extract a field from OP0.
1596 : Return the extracted value on success, otherwise return null.
1597 : EXTV describes the extraction instruction to use. If OP0_MODE
1598 : is defined, it is the mode of OP0, otherwise OP0 is a BLKmode MEM.
1599 : The other arguments are as for extract_bit_field. */
1600 :
1601 : static rtx
1602 166574 : extract_bit_field_using_extv (const extraction_insn *extv, rtx op0,
1603 : opt_scalar_int_mode op0_mode,
1604 : unsigned HOST_WIDE_INT bitsize,
1605 : unsigned HOST_WIDE_INT bitnum,
1606 : int unsignedp, rtx target,
1607 : machine_mode mode, machine_mode tmode)
1608 : {
1609 166574 : class expand_operand ops[4];
1610 166574 : rtx spec_target = target;
1611 166574 : rtx spec_target_subreg = 0;
1612 166574 : scalar_int_mode ext_mode = extv->field_mode;
1613 166574 : unsigned unit = GET_MODE_BITSIZE (ext_mode);
1614 :
1615 166574 : if (bitsize == 0 || unit < bitsize)
1616 : return NULL_RTX;
1617 :
1618 166574 : if (MEM_P (op0))
1619 : /* Get a reference to the first byte of the field. */
1620 0 : op0 = narrow_bit_field_mem (op0, extv->struct_mode, bitsize, bitnum,
1621 : &bitnum);
1622 : else
1623 : {
1624 : /* Convert from counting within OP0 to counting in EXT_MODE. */
1625 166574 : if (BYTES_BIG_ENDIAN)
1626 : bitnum += unit - GET_MODE_BITSIZE (op0_mode.require ());
1627 :
1628 : /* If op0 is a register, we need it in EXT_MODE to make it
1629 : acceptable to the format of ext(z)v. */
1630 166574 : if (GET_CODE (op0) == SUBREG && op0_mode.require () != ext_mode)
1631 0 : return NULL_RTX;
1632 166574 : if (REG_P (op0) && op0_mode.require () != ext_mode)
1633 50190 : op0 = gen_lowpart_SUBREG (ext_mode, op0);
1634 : }
1635 :
1636 : /* If BITS_BIG_ENDIAN is zero on a BYTES_BIG_ENDIAN machine, we count
1637 : "backwards" from the size of the unit we are extracting from.
1638 : Otherwise, we count bits from the most significant on a
1639 : BYTES/BITS_BIG_ENDIAN machine. */
1640 :
1641 166574 : if (BITS_BIG_ENDIAN != BYTES_BIG_ENDIAN)
1642 : bitnum = unit - bitsize - bitnum;
1643 :
1644 166574 : if (target == 0)
1645 46704 : target = spec_target = gen_reg_rtx (tmode);
1646 :
1647 166574 : if (GET_MODE (target) != ext_mode)
1648 : {
1649 99286 : rtx temp;
1650 : /* Don't use LHS paradoxical subreg if explicit truncation is needed
1651 : between the mode of the extraction (word_mode) and the target
1652 : mode. Instead, create a temporary and use convert_move to set
1653 : the target. */
1654 99286 : if (REG_P (target)
1655 98166 : && TRULY_NOOP_TRUNCATION_MODES_P (GET_MODE (target), ext_mode)
1656 197452 : && (temp = gen_lowpart_if_possible (ext_mode, target)))
1657 : {
1658 97656 : target = temp;
1659 97656 : if (partial_subreg_p (GET_MODE (spec_target), ext_mode))
1660 96798 : spec_target_subreg = target;
1661 : }
1662 : else
1663 1630 : target = gen_reg_rtx (ext_mode);
1664 : }
1665 :
1666 166574 : create_output_operand (&ops[0], target, ext_mode);
1667 166574 : create_fixed_operand (&ops[1], op0);
1668 166574 : create_integer_operand (&ops[2], bitsize);
1669 166574 : create_integer_operand (&ops[3], bitnum);
1670 166574 : if (maybe_expand_insn (extv->icode, 4, ops))
1671 : {
1672 1742 : target = ops[0].value;
1673 1742 : if (target == spec_target)
1674 : return target;
1675 1742 : if (target == spec_target_subreg)
1676 : return spec_target;
1677 58 : return convert_extracted_bit_field (target, mode, tmode, unsignedp);
1678 : }
1679 : return NULL_RTX;
1680 : }
1681 :
1682 : /* See whether it would be valid to extract the part of OP0 with
1683 : mode OP0_MODE described by BITNUM and BITSIZE into a value of
1684 : mode MODE using a subreg operation.
1685 : Return the subreg if so, otherwise return null. */
1686 :
1687 : static rtx
1688 859904 : extract_bit_field_as_subreg (machine_mode mode, rtx op0,
1689 : machine_mode op0_mode,
1690 : poly_uint64 bitsize, poly_uint64 bitnum)
1691 : {
1692 859904 : poly_uint64 bytenum;
1693 859904 : if (multiple_p (bitnum, BITS_PER_UNIT, &bytenum)
1694 821341 : && known_eq (bitsize, GET_MODE_BITSIZE (mode))
1695 859904 : && lowpart_bit_field_p (bitnum, bitsize, op0_mode)
1696 1681245 : && TRULY_NOOP_TRUNCATION_MODES_P (mode, op0_mode))
1697 684691 : return force_subreg (mode, op0, op0_mode, bytenum);
1698 : return NULL_RTX;
1699 : }
1700 :
1701 : /* A subroutine of extract_bit_field, with the same arguments.
1702 : If UNSIGNEDP is -1, the result need not be sign or zero extended.
1703 : If FALLBACK_P is true, fall back to extract_fixed_bit_field
1704 : if we can find no other means of implementing the operation.
1705 : if FALLBACK_P is false, return NULL instead. */
1706 :
1707 : static rtx
1708 1164834 : extract_bit_field_1 (rtx str_rtx, poly_uint64 bitsize, poly_uint64 bitnum,
1709 : int unsignedp, rtx target, machine_mode mode,
1710 : machine_mode tmode, bool reverse, bool fallback_p,
1711 : rtx *alt_rtl)
1712 : {
1713 1164834 : rtx op0 = str_rtx;
1714 1164834 : machine_mode mode1;
1715 :
1716 1164834 : if (tmode == VOIDmode)
1717 0 : tmode = mode;
1718 :
1719 1175576 : while (GET_CODE (op0) == SUBREG)
1720 : {
1721 10742 : bitnum += SUBREG_BYTE (op0) * BITS_PER_UNIT;
1722 10742 : op0 = SUBREG_REG (op0);
1723 : }
1724 :
1725 : /* If we have an out-of-bounds access to a register, just return an
1726 : uninitialized register of the required mode. This can occur if the
1727 : source code contains an out-of-bounds access to a small array. */
1728 2144994 : if (REG_P (op0) && known_ge (bitnum, GET_MODE_BITSIZE (GET_MODE (op0))))
1729 0 : return gen_reg_rtx (tmode);
1730 :
1731 1164834 : if (REG_P (op0)
1732 980160 : && mode == GET_MODE (op0)
1733 150509 : && known_eq (bitnum, 0U)
1734 1403802 : && known_eq (bitsize, GET_MODE_BITSIZE (GET_MODE (op0))))
1735 : {
1736 19155 : if (reverse)
1737 0 : op0 = flip_storage_order (mode, op0);
1738 : /* We're trying to extract a full register from itself. */
1739 : return op0;
1740 : }
1741 :
1742 : /* First try to check for vector from vector extractions. */
1743 1062606 : if (VECTOR_MODE_P (GET_MODE (op0))
1744 104483 : && !MEM_P (op0)
1745 103071 : && VECTOR_MODE_P (tmode)
1746 13222 : && known_eq (bitsize, GET_MODE_PRECISION (tmode))
1747 2317802 : && maybe_gt (GET_MODE_SIZE (GET_MODE (op0)), GET_MODE_SIZE (tmode)))
1748 : {
1749 13222 : machine_mode new_mode = GET_MODE (op0);
1750 39666 : if (GET_MODE_INNER (new_mode) != GET_MODE_INNER (tmode))
1751 : {
1752 214 : scalar_mode inner_mode = GET_MODE_INNER (tmode);
1753 214 : poly_uint64 nunits;
1754 428 : if (!multiple_p (GET_MODE_BITSIZE (GET_MODE (op0)),
1755 214 : GET_MODE_UNIT_BITSIZE (tmode), &nunits)
1756 428 : || !related_vector_mode (tmode, inner_mode,
1757 214 : nunits).exists (&new_mode)
1758 412 : || maybe_ne (GET_MODE_SIZE (new_mode),
1759 610 : GET_MODE_SIZE (GET_MODE (op0))))
1760 16 : new_mode = VOIDmode;
1761 : }
1762 13222 : poly_uint64 pos;
1763 13222 : if (new_mode != VOIDmode
1764 13206 : && (convert_optab_handler (vec_extract_optab, new_mode, tmode)
1765 : != CODE_FOR_nothing)
1766 26428 : && multiple_p (bitnum, GET_MODE_BITSIZE (tmode), &pos))
1767 : {
1768 9009 : class expand_operand ops[3];
1769 9009 : machine_mode outermode = new_mode;
1770 9009 : machine_mode innermode = tmode;
1771 9009 : enum insn_code icode
1772 9009 : = convert_optab_handler (vec_extract_optab, outermode, innermode);
1773 :
1774 9009 : if (new_mode != GET_MODE (op0))
1775 33 : op0 = gen_lowpart (new_mode, op0);
1776 9009 : create_output_operand (&ops[0], target, innermode);
1777 9009 : ops[0].target = 1;
1778 9009 : create_input_operand (&ops[1], op0, outermode);
1779 9009 : create_integer_operand (&ops[2], pos);
1780 9009 : if (maybe_expand_insn (icode, 3, ops))
1781 : {
1782 9009 : if (alt_rtl && ops[0].target)
1783 207 : *alt_rtl = target;
1784 9009 : target = ops[0].value;
1785 9009 : if (GET_MODE (target) != mode)
1786 9009 : return gen_lowpart (tmode, target);
1787 : return target;
1788 : }
1789 : }
1790 : }
1791 :
1792 : /* See if we can get a better vector mode before extracting. */
1793 1055207 : if (VECTOR_MODE_P (GET_MODE (op0))
1794 95474 : && !MEM_P (op0)
1795 1324794 : && GET_MODE_INNER (GET_MODE (op0)) != tmode)
1796 : {
1797 9889 : machine_mode new_mode;
1798 :
1799 9889 : if (GET_MODE_CLASS (tmode) == MODE_FLOAT)
1800 557 : new_mode = MIN_MODE_VECTOR_FLOAT;
1801 : else if (GET_MODE_CLASS (tmode) == MODE_FRACT)
1802 0 : new_mode = MIN_MODE_VECTOR_FRACT;
1803 : else if (GET_MODE_CLASS (tmode) == MODE_UFRACT)
1804 0 : new_mode = MIN_MODE_VECTOR_UFRACT;
1805 : else if (GET_MODE_CLASS (tmode) == MODE_ACCUM)
1806 0 : new_mode = MIN_MODE_VECTOR_ACCUM;
1807 : else if (GET_MODE_CLASS (tmode) == MODE_UACCUM)
1808 0 : new_mode = MIN_MODE_VECTOR_UACCUM;
1809 : else
1810 9332 : new_mode = MIN_MODE_VECTOR_INT;
1811 :
1812 156407 : FOR_EACH_MODE_FROM (new_mode, new_mode)
1813 312098 : if (known_eq (GET_MODE_SIZE (new_mode), GET_MODE_SIZE (GET_MODE (op0)))
1814 65654 : && known_eq (GET_MODE_UNIT_SIZE (new_mode), GET_MODE_SIZE (tmode))
1815 175827 : && known_eq (bitsize, GET_MODE_UNIT_PRECISION (new_mode))
1816 19778 : && multiple_p (bitnum, GET_MODE_UNIT_PRECISION (new_mode))
1817 9852 : && targetm.vector_mode_supported_p (new_mode)
1818 165883 : && targetm.modes_tieable_p (GET_MODE (op0), new_mode))
1819 : break;
1820 9889 : if (new_mode != VOIDmode)
1821 9531 : op0 = gen_lowpart (new_mode, op0);
1822 : }
1823 :
1824 : /* Use vec_extract patterns for extracting parts of vectors whenever
1825 : available. If that fails, see whether the current modes and bitregion
1826 : give a natural subreg. */
1827 1136670 : machine_mode outermode = GET_MODE (op0);
1828 1136670 : if (VECTOR_MODE_P (outermode) && !MEM_P (op0))
1829 : {
1830 94062 : scalar_mode innermode = GET_MODE_INNER (outermode);
1831 :
1832 94062 : enum insn_code icode
1833 94062 : = convert_optab_handler (vec_extract_optab, outermode, innermode);
1834 :
1835 94062 : poly_uint64 pos;
1836 94062 : if (icode != CODE_FOR_nothing
1837 94065 : && known_eq (bitsize, GET_MODE_PRECISION (innermode))
1838 186142 : && multiple_p (bitnum, GET_MODE_PRECISION (innermode), &pos))
1839 : {
1840 92077 : class expand_operand ops[3];
1841 :
1842 92077 : create_output_operand (&ops[0], target,
1843 92077 : insn_data[icode].operand[0].mode);
1844 92077 : ops[0].target = 1;
1845 92077 : create_input_operand (&ops[1], op0, outermode);
1846 92077 : create_integer_operand (&ops[2], pos);
1847 92077 : if (maybe_expand_insn (icode, 3, ops))
1848 : {
1849 92077 : if (alt_rtl && ops[0].target)
1850 22417 : *alt_rtl = target;
1851 92077 : target = ops[0].value;
1852 92077 : if (GET_MODE (target) != mode)
1853 92077 : return gen_lowpart (tmode, target);
1854 : return target;
1855 : }
1856 : }
1857 : /* Using subregs is useful if we're extracting one register vector
1858 : from a multi-register vector. extract_bit_field_as_subreg checks
1859 : for valid bitsize and bitnum, so we don't need to do that here. */
1860 1985 : if (VECTOR_MODE_P (mode))
1861 : {
1862 47 : rtx sub = extract_bit_field_as_subreg (mode, op0, outermode,
1863 : bitsize, bitnum);
1864 47 : if (sub)
1865 : return sub;
1866 : }
1867 : }
1868 :
1869 : /* Make sure we are playing with integral modes. Pun with subregs
1870 : if we aren't. */
1871 1044576 : opt_scalar_int_mode op0_mode = int_mode_for_mode (GET_MODE (op0));
1872 1044576 : scalar_int_mode imode;
1873 1044576 : bool need_stack_p = false;
1874 1044576 : if (!op0_mode.exists (&imode) || imode != GET_MODE (op0))
1875 : {
1876 198158 : if (MEM_P (op0))
1877 159358 : op0 = adjust_bitfield_address_size (op0, op0_mode.else_blk (),
1878 : 0, MEM_SIZE (op0));
1879 38800 : else if (op0_mode.exists (&imode)
1880 39793 : && known_ge
1881 : ((unsigned) UNITS_PER_WORD,
1882 : (poly_uint64) REGMODE_NATURAL_SIZE (GET_MODE (op0))))
1883 : {
1884 38769 : op0 = gen_lowpart (imode, op0);
1885 :
1886 : /* If we got a SUBREG, force it into a register since we
1887 : aren't going to be able to do another SUBREG on it. */
1888 38769 : if (GET_CODE (op0) == SUBREG)
1889 38492 : op0 = force_reg (imode, op0);
1890 : }
1891 : else
1892 : need_stack_p = true;
1893 : }
1894 846418 : else if (!MEM_P (op0)
1895 1746679 : && maybe_lt ((unsigned) UNITS_PER_WORD,
1896 821102 : (poly_uint64) REGMODE_NATURAL_SIZE (GET_MODE (op0))))
1897 0 : need_stack_p = true;
1898 :
1899 1044299 : if (need_stack_p)
1900 : {
1901 62 : poly_int64 size = GET_MODE_SIZE (GET_MODE (op0));
1902 31 : rtx mem = assign_stack_temp (GET_MODE (op0), size);
1903 31 : emit_move_insn (mem, op0);
1904 31 : op0 = adjust_bitfield_address_size (mem, BLKmode, 0, size);
1905 : }
1906 :
1907 : /* ??? We currently assume TARGET is at least as big as BITSIZE.
1908 : If that's wrong, the solution is to test for it and set TARGET to 0
1909 : if needed. */
1910 :
1911 : /* Get the mode of the field to use for atomic access or subreg
1912 : conversion. */
1913 1044576 : if (!SCALAR_INT_MODE_P (tmode)
1914 1044576 : || !mode_for_size (bitsize, GET_MODE_CLASS (tmode), 0).exists (&mode1))
1915 311736 : mode1 = mode;
1916 1044576 : gcc_assert (mode1 != BLKmode);
1917 :
1918 : /* Extraction of a full MODE1 value can be done with a subreg as long
1919 : as the least significant bit of the value is the least significant
1920 : bit of either OP0 or a word of OP0. */
1921 1044576 : if (!MEM_P (op0) && !reverse && op0_mode.exists (&imode))
1922 : {
1923 859857 : rtx sub = extract_bit_field_as_subreg (mode1, op0, imode,
1924 : bitsize, bitnum);
1925 859857 : if (sub)
1926 678320 : return convert_extracted_bit_field (sub, mode, tmode, unsignedp);
1927 : }
1928 :
1929 : /* Extraction of a full MODE1 value can be done with a load as long as
1930 : the field is on a byte boundary and is sufficiently aligned. */
1931 366256 : poly_uint64 bytenum;
1932 366256 : if (simple_mem_bitfield_p (op0, bitsize, bitnum, mode1, &bytenum))
1933 : {
1934 44930 : op0 = adjust_bitfield_address (op0, mode1, bytenum);
1935 44930 : if (reverse)
1936 52 : op0 = flip_storage_order (mode1, op0);
1937 44930 : return convert_extracted_bit_field (op0, mode, tmode, unsignedp);
1938 : }
1939 :
1940 : /* If we have a memory source and a non-constant bit offset, restrict
1941 : the memory to the referenced bytes. This is a worst-case fallback
1942 : but is useful for things like vector booleans. */
1943 321326 : if (MEM_P (op0) && !bitnum.is_constant ())
1944 : {
1945 : bytenum = bits_to_bytes_round_down (bitnum);
1946 : bitnum = num_trailing_bits (bitnum);
1947 : poly_uint64 bytesize = bits_to_bytes_round_up (bitnum + bitsize);
1948 : op0 = adjust_bitfield_address_size (op0, BLKmode, bytenum, bytesize);
1949 : op0_mode = opt_scalar_int_mode ();
1950 : }
1951 :
1952 : /* It's possible we'll need to handle other cases here for
1953 : polynomial bitnum and bitsize. */
1954 :
1955 : /* From here on we need to be looking at a fixed-size insertion. */
1956 321326 : return extract_integral_bit_field (op0, op0_mode, bitsize.to_constant (),
1957 : bitnum.to_constant (), unsignedp,
1958 321326 : target, mode, tmode, reverse, fallback_p);
1959 : }
1960 :
1961 : /* Subroutine of extract_bit_field_1, with the same arguments, except
1962 : that BITSIZE and BITNUM are constant. Handle cases specific to
1963 : integral modes. If OP0_MODE is defined, it is the mode of OP0,
1964 : otherwise OP0 is a BLKmode MEM. */
1965 :
1966 : static rtx
1967 321326 : extract_integral_bit_field (rtx op0, opt_scalar_int_mode op0_mode,
1968 : unsigned HOST_WIDE_INT bitsize,
1969 : unsigned HOST_WIDE_INT bitnum, int unsignedp,
1970 : rtx target, machine_mode mode, machine_mode tmode,
1971 : bool reverse, bool fallback_p)
1972 : {
1973 : /* Handle fields bigger than a word. */
1974 :
1975 324606 : if (bitsize > BITS_PER_WORD)
1976 : {
1977 : /* Here we transfer the words of the field
1978 : in the order least significant first.
1979 : This is because the most significant word is the one which may
1980 : be less than full. */
1981 :
1982 1517 : const bool backwards = WORDS_BIG_ENDIAN;
1983 1517 : unsigned int nwords = (bitsize + (BITS_PER_WORD - 1)) / BITS_PER_WORD;
1984 1517 : unsigned int i;
1985 1517 : rtx_insn *last;
1986 :
1987 1517 : if (target == 0 || !REG_P (target) || !valid_multiword_target_p (target))
1988 1493 : target = gen_reg_rtx (mode);
1989 :
1990 : /* In case we're about to clobber a base register or something
1991 : (see gcc.c-torture/execute/20040625-1.c). */
1992 1517 : if (reg_mentioned_p (target, op0))
1993 0 : target = gen_reg_rtx (mode);
1994 :
1995 : /* Indicate for flow that the entire target reg is being set. */
1996 1517 : emit_clobber (target);
1997 :
1998 : /* The mode must be fixed-size, since extract_bit_field_1 handles
1999 : extractions from variable-sized objects before calling this
2000 : function. */
2001 1517 : unsigned int target_size
2002 1517 : = GET_MODE_SIZE (GET_MODE (target)).to_constant ();
2003 1517 : last = get_last_insn ();
2004 4551 : for (i = 0; i < nwords; i++)
2005 : {
2006 : /* If I is 0, use the low-order word in both field and target;
2007 : if I is 1, use the next to lowest word; and so on. */
2008 : /* Word number in TARGET to use. */
2009 3034 : unsigned int wordnum
2010 : = (backwards ? target_size / UNITS_PER_WORD - i - 1 : i);
2011 : /* Offset from start of field in OP0. */
2012 6068 : unsigned int bit_offset = (backwards ^ reverse
2013 3034 : ? MAX ((int) bitsize - ((int) i + 1)
2014 : * BITS_PER_WORD,
2015 : 0)
2016 3102 : : (int) i * BITS_PER_WORD);
2017 3034 : rtx target_part = operand_subword (target, wordnum, 1, VOIDmode);
2018 3034 : rtx result_part
2019 3210 : = extract_bit_field_1 (op0, MIN (BITS_PER_WORD,
2020 : bitsize - i * BITS_PER_WORD),
2021 3034 : bitnum + bit_offset,
2022 : (unsignedp ? 1 : -1), target_part,
2023 : mode, word_mode, reverse, fallback_p, NULL);
2024 :
2025 3034 : if (!result_part)
2026 : {
2027 0 : delete_insns_since (last);
2028 0 : return NULL;
2029 : }
2030 :
2031 3034 : if (!target_part)
2032 0 : store_bit_field (target, BITS_PER_WORD, i * BITS_PER_WORD,
2033 : 0, 0, word_mode, result_part, false, i == 0);
2034 3034 : else if (result_part != target_part)
2035 2918 : emit_move_insn (target_part, result_part);
2036 : }
2037 :
2038 1517 : if (unsignedp)
2039 : {
2040 : /* Unless we've filled TARGET, the upper regs in a multi-reg value
2041 : need to be zero'd out. */
2042 1531 : if (target_size > nwords * UNITS_PER_WORD)
2043 : {
2044 0 : unsigned int i, total_words;
2045 :
2046 0 : total_words = target_size / UNITS_PER_WORD;
2047 0 : for (i = nwords; i < total_words; i++)
2048 0 : emit_move_insn
2049 0 : (operand_subword (target,
2050 0 : backwards ? total_words - i - 1 : i,
2051 : 1, VOIDmode),
2052 : const0_rtx);
2053 : }
2054 : return target;
2055 : }
2056 :
2057 : /* Signed bit field: sign-extend with two arithmetic shifts. */
2058 40 : target = expand_shift (LSHIFT_EXPR, mode, target,
2059 20 : GET_MODE_BITSIZE (mode) - bitsize, NULL_RTX, 0);
2060 40 : return expand_shift (RSHIFT_EXPR, mode, target,
2061 20 : GET_MODE_BITSIZE (mode) - bitsize, NULL_RTX, 0);
2062 : }
2063 :
2064 : /* If OP0 is a multi-word register, narrow it to the affected word.
2065 : If the region spans two words, defer to extract_split_bit_field. */
2066 503897 : if (!MEM_P (op0) && GET_MODE_SIZE (op0_mode.require ()) > UNITS_PER_WORD)
2067 : {
2068 3909 : if (bitnum % BITS_PER_WORD + bitsize > BITS_PER_WORD)
2069 : {
2070 1146 : if (!fallback_p)
2071 : return NULL_RTX;
2072 64 : target = extract_split_bit_field (op0, op0_mode, bitsize, bitnum,
2073 : unsignedp, reverse);
2074 64 : return convert_extracted_bit_field (target, mode, tmode, unsignedp);
2075 : }
2076 : /* If OP0 is a hard register, copy it to a pseudo before calling
2077 : force_subreg. */
2078 2763 : if (REG_P (op0) && HARD_REGISTER_P (op0))
2079 1 : op0 = copy_to_reg (op0);
2080 2763 : op0 = force_subreg (word_mode, op0, op0_mode.require (),
2081 3227 : bitnum / BITS_PER_WORD * UNITS_PER_WORD);
2082 2763 : op0_mode = word_mode;
2083 2995 : bitnum %= BITS_PER_WORD;
2084 : }
2085 :
2086 : /* From here on we know the desired field is smaller than a word.
2087 : If OP0 is a register, it too fits within a word. */
2088 318663 : enum extraction_pattern pattern = unsignedp ? EP_extzv : EP_extv;
2089 318663 : extraction_insn extv;
2090 318663 : if (!MEM_P (op0)
2091 180396 : && !reverse
2092 : /* ??? We could limit the structure size to the part of OP0 that
2093 : contains the field, with appropriate checks for endianness
2094 : and TARGET_TRULY_NOOP_TRUNCATION. */
2095 499045 : && get_best_reg_extraction_insn (&extv, pattern,
2096 512853 : GET_MODE_BITSIZE (op0_mode.require ()),
2097 : tmode))
2098 : {
2099 166574 : rtx result = extract_bit_field_using_extv (&extv, op0, op0_mode,
2100 : bitsize, bitnum,
2101 : unsignedp, target, mode,
2102 : tmode);
2103 166574 : if (result)
2104 : return result;
2105 : }
2106 :
2107 : /* If OP0 is a memory, try copying it to a register and seeing if a
2108 : cheap register alternative is available. */
2109 316921 : if (MEM_P (op0) & !reverse)
2110 : {
2111 138044 : if (get_best_mem_extraction_insn (&extv, pattern, bitsize, bitnum,
2112 : tmode))
2113 : {
2114 0 : rtx result = extract_bit_field_using_extv (&extv, op0, op0_mode,
2115 : bitsize, bitnum,
2116 : unsignedp, target, mode,
2117 : tmode);
2118 0 : if (result)
2119 0 : return result;
2120 : }
2121 :
2122 138044 : rtx_insn *last = get_last_insn ();
2123 :
2124 : /* Try loading part of OP0 into a register and extracting the
2125 : bitfield from that. */
2126 138044 : unsigned HOST_WIDE_INT bitpos;
2127 138044 : rtx xop0 = adjust_bit_field_mem_for_reg (pattern, op0, bitsize, bitnum,
2128 : 0, 0, tmode, &bitpos);
2129 138044 : if (xop0)
2130 : {
2131 135378 : xop0 = copy_to_reg (xop0);
2132 135378 : rtx result = extract_bit_field_1 (xop0, bitsize, bitpos,
2133 : unsignedp, target,
2134 : mode, tmode, reverse, false, NULL);
2135 135378 : if (result)
2136 : return result;
2137 135378 : delete_insns_since (last);
2138 : }
2139 : }
2140 :
2141 316921 : if (!fallback_p)
2142 : return NULL;
2143 :
2144 : /* Find a correspondingly-sized integer field, so we can apply
2145 : shifts and masks to it. */
2146 182625 : scalar_int_mode int_mode;
2147 182625 : if (!int_mode_for_mode (tmode).exists (&int_mode))
2148 : /* If this fails, we should probably push op0 out to memory and then
2149 : do a load. */
2150 0 : int_mode = int_mode_for_mode (mode).require ();
2151 :
2152 182625 : target = extract_fixed_bit_field (int_mode, op0, op0_mode, bitsize,
2153 : bitnum, target, unsignedp, reverse);
2154 :
2155 : /* Complex values must be reversed piecewise, so we need to undo the global
2156 : reversal, convert to the complex mode and reverse again. */
2157 182625 : if (reverse && COMPLEX_MODE_P (tmode))
2158 : {
2159 0 : target = flip_storage_order (int_mode, target);
2160 0 : target = convert_extracted_bit_field (target, mode, tmode, unsignedp);
2161 0 : target = flip_storage_order (tmode, target);
2162 : }
2163 : else
2164 182625 : target = convert_extracted_bit_field (target, mode, tmode, unsignedp);
2165 :
2166 : return target;
2167 : }
2168 :
2169 : /* Generate code to extract a byte-field from STR_RTX
2170 : containing BITSIZE bits, starting at BITNUM,
2171 : and put it in TARGET if possible (if TARGET is nonzero).
2172 : Regardless of TARGET, we return the rtx for where the value is placed.
2173 :
2174 : STR_RTX is the structure containing the byte (a REG or MEM).
2175 : UNSIGNEDP is nonzero if this is an unsigned bit field.
2176 : MODE is the natural mode of the field value once extracted.
2177 : TMODE is the mode the caller would like the value to have;
2178 : but the value may be returned with type MODE instead.
2179 :
2180 : If REVERSE is true, the extraction is to be done in reverse order.
2181 :
2182 : If a TARGET is specified and we can store in it at no extra cost,
2183 : we do so, and return TARGET.
2184 : Otherwise, we return a REG of mode TMODE or MODE, with TMODE preferred
2185 : if they are equally easy.
2186 :
2187 : If the result can be stored at TARGET, and ALT_RTL is non-NULL,
2188 : then *ALT_RTL is set to TARGET (before legitimziation). */
2189 :
2190 : rtx
2191 1026422 : extract_bit_field (rtx str_rtx, poly_uint64 bitsize, poly_uint64 bitnum,
2192 : int unsignedp, rtx target, machine_mode mode,
2193 : machine_mode tmode, bool reverse, rtx *alt_rtl)
2194 : {
2195 1026422 : machine_mode mode1;
2196 :
2197 : /* Handle -fstrict-volatile-bitfields in the cases where it applies. */
2198 2052844 : if (maybe_ne (GET_MODE_BITSIZE (GET_MODE (str_rtx)), 0))
2199 : mode1 = GET_MODE (str_rtx);
2200 260116 : else if (target && maybe_ne (GET_MODE_BITSIZE (GET_MODE (target)), 0))
2201 : mode1 = GET_MODE (target);
2202 : else
2203 : mode1 = tmode;
2204 :
2205 1026422 : unsigned HOST_WIDE_INT ibitsize, ibitnum;
2206 1026422 : scalar_int_mode int_mode;
2207 1026422 : if (bitsize.is_constant (&ibitsize)
2208 1026422 : && bitnum.is_constant (&ibitnum)
2209 1883994 : && is_a <scalar_int_mode> (mode1, &int_mode)
2210 857579 : && strict_volatile_bitfield_p (str_rtx, ibitsize, ibitnum,
2211 : int_mode, 0, 0))
2212 : {
2213 : /* Extraction of a full INT_MODE value can be done with a simple load.
2214 : We know here that the field can be accessed with one single
2215 : instruction. For targets that support unaligned memory,
2216 : an unaligned access may be necessary. */
2217 14 : if (ibitsize == GET_MODE_BITSIZE (int_mode))
2218 : {
2219 0 : rtx result = adjust_bitfield_address (str_rtx, int_mode,
2220 : ibitnum / BITS_PER_UNIT);
2221 0 : if (reverse)
2222 0 : result = flip_storage_order (int_mode, result);
2223 0 : gcc_assert (ibitnum % BITS_PER_UNIT == 0);
2224 0 : return convert_extracted_bit_field (result, mode, tmode, unsignedp);
2225 : }
2226 :
2227 7 : str_rtx = narrow_bit_field_mem (str_rtx, int_mode, ibitsize, ibitnum,
2228 : &ibitnum);
2229 14 : gcc_assert (ibitnum + ibitsize <= GET_MODE_BITSIZE (int_mode));
2230 7 : str_rtx = copy_to_reg (str_rtx);
2231 7 : return extract_bit_field_1 (str_rtx, ibitsize, ibitnum, unsignedp,
2232 : target, mode, tmode, reverse, true, alt_rtl);
2233 : }
2234 :
2235 1026415 : return extract_bit_field_1 (str_rtx, bitsize, bitnum, unsignedp,
2236 1026415 : target, mode, tmode, reverse, true, alt_rtl);
2237 : }
2238 :
2239 : /* Use shifts and boolean operations to extract a field of BITSIZE bits
2240 : from bit BITNUM of OP0. If OP0_MODE is defined, it is the mode of OP0,
2241 : otherwise OP0 is a BLKmode MEM.
2242 :
2243 : UNSIGNEDP is nonzero for an unsigned bit field (don't sign-extend value).
2244 : If REVERSE is true, the extraction is to be done in reverse order.
2245 :
2246 : If TARGET is nonzero, attempts to store the value there
2247 : and return TARGET, but this is not guaranteed.
2248 : If TARGET is not used, create a pseudo-reg of mode TMODE for the value. */
2249 :
2250 : static rtx
2251 200980 : extract_fixed_bit_field (machine_mode tmode, rtx op0,
2252 : opt_scalar_int_mode op0_mode,
2253 : unsigned HOST_WIDE_INT bitsize,
2254 : unsigned HOST_WIDE_INT bitnum, rtx target,
2255 : int unsignedp, bool reverse)
2256 : {
2257 200980 : scalar_int_mode mode;
2258 200980 : if (MEM_P (op0))
2259 : {
2260 151994 : if (!get_best_mode (bitsize, bitnum, 0, 0, MEM_ALIGN (op0),
2261 151994 : BITS_PER_WORD, MEM_VOLATILE_P (op0), &mode))
2262 : /* The only way this should occur is if the field spans word
2263 : boundaries. */
2264 4302 : return extract_split_bit_field (op0, op0_mode, bitsize, bitnum,
2265 4302 : unsignedp, reverse);
2266 :
2267 147692 : op0 = narrow_bit_field_mem (op0, mode, bitsize, bitnum, &bitnum);
2268 : }
2269 : else
2270 48986 : mode = op0_mode.require ();
2271 :
2272 196678 : return extract_fixed_bit_field_1 (tmode, op0, mode, bitsize, bitnum,
2273 196678 : target, unsignedp, reverse);
2274 : }
2275 :
2276 : /* Helper function for extract_fixed_bit_field, extracts
2277 : the bit field always using MODE, which is the mode of OP0.
2278 : If UNSIGNEDP is -1, the result need not be sign or zero extended.
2279 : The other arguments are as for extract_fixed_bit_field. */
2280 :
2281 : static rtx
2282 196678 : extract_fixed_bit_field_1 (machine_mode tmode, rtx op0, scalar_int_mode mode,
2283 : unsigned HOST_WIDE_INT bitsize,
2284 : unsigned HOST_WIDE_INT bitnum, rtx target,
2285 : int unsignedp, bool reverse)
2286 : {
2287 : /* Note that bitsize + bitnum can be greater than GET_MODE_BITSIZE (mode)
2288 : for invalid input, such as extract equivalent of f5 from
2289 : gcc.dg/pr48335-2.c. */
2290 :
2291 196678 : if (reverse ? !BYTES_BIG_ENDIAN : BYTES_BIG_ENDIAN)
2292 : /* BITNUM is the distance between our msb and that of OP0.
2293 : Convert it to the distance from the lsb. */
2294 478 : bitnum = GET_MODE_BITSIZE (mode) - bitsize - bitnum;
2295 :
2296 : /* Now BITNUM is always the distance between the field's lsb and that of OP0.
2297 : We have reduced the big-endian case to the little-endian case. */
2298 196678 : if (reverse)
2299 239 : op0 = flip_storage_order (mode, op0);
2300 :
2301 196678 : if (unsignedp)
2302 : {
2303 137240 : if (bitnum)
2304 : {
2305 : /* If the field does not already start at the lsb,
2306 : shift it so it does. */
2307 : /* Maybe propagate the target for the shift. */
2308 61595 : rtx subtarget = (target != 0 && REG_P (target) ? target : 0);
2309 61595 : if (tmode != mode)
2310 41164 : subtarget = 0;
2311 61595 : op0 = expand_shift (RSHIFT_EXPR, mode, op0, bitnum, subtarget, 1);
2312 : }
2313 : /* Convert the value to the desired mode. TMODE must also be a
2314 : scalar integer for this conversion to make sense, since we
2315 : shouldn't reinterpret the bits. */
2316 137240 : scalar_int_mode new_mode = as_a <scalar_int_mode> (tmode);
2317 137240 : if (mode != new_mode)
2318 60620 : op0 = convert_to_mode (new_mode, op0, 1);
2319 :
2320 : /* Unless the msb of the field used to be the msb when we shifted,
2321 : mask out the upper bits. */
2322 :
2323 137240 : if (GET_MODE_BITSIZE (mode) != bitnum + bitsize
2324 137240 : && unsignedp != -1)
2325 93534 : return expand_binop (new_mode, and_optab, op0,
2326 : mask_rtx (new_mode, 0, bitsize, 0),
2327 93534 : target, 1, OPTAB_LIB_WIDEN);
2328 : return op0;
2329 : }
2330 :
2331 : /* To extract a signed bit-field, first shift its msb to the msb of the word,
2332 : then arithmetic-shift its lsb to the lsb of the word. */
2333 59438 : op0 = force_reg (mode, op0);
2334 :
2335 : /* Find the narrowest integer mode that contains the field. */
2336 :
2337 59438 : opt_scalar_int_mode mode_iter;
2338 147468 : FOR_EACH_MODE_IN_CLASS (mode_iter, MODE_INT)
2339 294936 : if (GET_MODE_BITSIZE (mode_iter.require ()) >= bitsize + bitnum)
2340 : break;
2341 :
2342 59438 : mode = mode_iter.require ();
2343 59438 : op0 = convert_to_mode (mode, op0, 0);
2344 :
2345 59438 : if (mode != tmode)
2346 4856 : target = 0;
2347 :
2348 118876 : if (GET_MODE_BITSIZE (mode) != (bitsize + bitnum))
2349 : {
2350 54432 : int amount = GET_MODE_BITSIZE (mode) - (bitsize + bitnum);
2351 : /* Maybe propagate the target for the shift. */
2352 54432 : rtx subtarget = (target != 0 && REG_P (target) ? target : 0);
2353 54432 : op0 = expand_shift (LSHIFT_EXPR, mode, op0, amount, subtarget, 1);
2354 : }
2355 :
2356 118876 : return expand_shift (RSHIFT_EXPR, mode, op0,
2357 59438 : GET_MODE_BITSIZE (mode) - bitsize, target, 0);
2358 : }
2359 :
2360 : /* Return a constant integer (CONST_INT or CONST_DOUBLE) rtx with the value
2361 : VALUE << BITPOS. */
2362 :
2363 : static rtx
2364 73173 : lshift_value (machine_mode mode, unsigned HOST_WIDE_INT value,
2365 : int bitpos)
2366 : {
2367 73173 : return immed_wide_int_const (wi::lshift (value, bitpos), mode);
2368 : }
2369 :
2370 : /* Extract a bit field that is split across two words
2371 : and return an RTX for the result.
2372 :
2373 : OP0 is the REG, SUBREG or MEM rtx for the first of the two words.
2374 : BITSIZE is the field width; BITPOS, position of its first bit, in the word.
2375 : UNSIGNEDP is 1 if should zero-extend the contents; else sign-extend.
2376 : If OP0_MODE is defined, it is the mode of OP0, otherwise OP0 is
2377 : a BLKmode MEM.
2378 :
2379 : If REVERSE is true, the extraction is to be done in reverse order. */
2380 :
2381 : static rtx
2382 4366 : extract_split_bit_field (rtx op0, opt_scalar_int_mode op0_mode,
2383 : unsigned HOST_WIDE_INT bitsize,
2384 : unsigned HOST_WIDE_INT bitpos, int unsignedp,
2385 : bool reverse)
2386 : {
2387 4366 : unsigned int unit;
2388 4366 : unsigned int bitsdone = 0;
2389 4366 : rtx result = NULL_RTX;
2390 4366 : int first = 1;
2391 :
2392 : /* Make sure UNIT isn't larger than BITS_PER_WORD, we can only handle that
2393 : much at a time. */
2394 4366 : if (REG_P (op0) || GET_CODE (op0) == SUBREG)
2395 64 : unit = BITS_PER_WORD;
2396 : else
2397 6385 : unit = MIN (MEM_ALIGN (op0), BITS_PER_WORD);
2398 :
2399 18197 : while (bitsdone < bitsize)
2400 : {
2401 13831 : unsigned HOST_WIDE_INT thissize;
2402 13831 : rtx part;
2403 13831 : unsigned HOST_WIDE_INT thispos;
2404 13831 : unsigned HOST_WIDE_INT offset;
2405 :
2406 13831 : offset = (bitpos + bitsdone) / unit;
2407 13831 : thispos = (bitpos + bitsdone) % unit;
2408 :
2409 : /* THISSIZE must not overrun a word boundary. Otherwise,
2410 : extract_fixed_bit_field will call us again, and we will mutually
2411 : recurse forever. */
2412 13831 : thissize = MIN (bitsize - bitsdone, BITS_PER_WORD);
2413 13831 : thissize = MIN (thissize, unit - thispos);
2414 :
2415 : /* If OP0 is a register, then handle OFFSET here. */
2416 13831 : rtx op0_piece = op0;
2417 13831 : opt_scalar_int_mode op0_piece_mode = op0_mode;
2418 13831 : if (SUBREG_P (op0) || REG_P (op0))
2419 : {
2420 128 : op0_piece = operand_subword_force (op0, offset, op0_mode.require ());
2421 128 : op0_piece_mode = word_mode;
2422 128 : offset = 0;
2423 : }
2424 :
2425 : /* Extract the parts in bit-counting order,
2426 : whose meaning is determined by BYTES_PER_UNIT.
2427 : OFFSET is in UNITs, and UNIT is in bits. */
2428 27662 : part = extract_fixed_bit_field (word_mode, op0_piece, op0_piece_mode,
2429 13831 : thissize, offset * unit + thispos,
2430 : 0, 1, reverse);
2431 13831 : bitsdone += thissize;
2432 :
2433 : /* Shift this part into place for the result. */
2434 13831 : if (reverse ? !BYTES_BIG_ENDIAN : BYTES_BIG_ENDIAN)
2435 : {
2436 4 : if (bitsize != bitsdone)
2437 2 : part = expand_shift (LSHIFT_EXPR, word_mode, part,
2438 2 : bitsize - bitsdone, 0, 1);
2439 : }
2440 : else
2441 : {
2442 13827 : if (bitsdone != thissize)
2443 9463 : part = expand_shift (LSHIFT_EXPR, word_mode, part,
2444 9463 : bitsdone - thissize, 0, 1);
2445 : }
2446 :
2447 13831 : if (first)
2448 : result = part;
2449 : else
2450 : /* Combine the parts with bitwise or. This works
2451 : because we extracted each part as an unsigned bit field. */
2452 9465 : result = expand_binop (word_mode, ior_optab, part, result, NULL_RTX, 1,
2453 : OPTAB_LIB_WIDEN);
2454 :
2455 13831 : first = 0;
2456 : }
2457 :
2458 : /* Unsigned bit field: we are done. */
2459 4366 : if (unsignedp)
2460 : return result;
2461 : /* Signed bit field: sign-extend with two arithmetic shifts. */
2462 1464 : result = expand_shift (LSHIFT_EXPR, word_mode, result,
2463 1464 : BITS_PER_WORD - bitsize, NULL_RTX, 0);
2464 1464 : return expand_shift (RSHIFT_EXPR, word_mode, result,
2465 1464 : BITS_PER_WORD - bitsize, NULL_RTX, 0);
2466 : }
2467 :
2468 : /* Try to read the low bits of SRC as an rvalue of mode MODE, preserving
2469 : the bit pattern. SRC_MODE is the mode of SRC; if this is smaller than
2470 : MODE, fill the upper bits with zeros. Fail if the layout of either
2471 : mode is unknown (as for CC modes) or if the extraction would involve
2472 : unprofitable mode punning. Return the value on success, otherwise
2473 : return null.
2474 :
2475 : This is different from gen_lowpart* in these respects:
2476 :
2477 : - the returned value must always be considered an rvalue
2478 :
2479 : - when MODE is wider than SRC_MODE, the extraction involves
2480 : a zero extension
2481 :
2482 : - when MODE is smaller than SRC_MODE, the extraction involves
2483 : a truncation (and is thus subject to TARGET_TRULY_NOOP_TRUNCATION).
2484 :
2485 : In other words, this routine performs a computation, whereas the
2486 : gen_lowpart* routines are conceptually lvalue or rvalue subreg
2487 : operations. */
2488 :
2489 : rtx
2490 118336 : extract_low_bits (machine_mode mode, machine_mode src_mode, rtx src)
2491 : {
2492 118336 : scalar_int_mode int_mode, src_int_mode;
2493 :
2494 118336 : if (mode == src_mode)
2495 : return src;
2496 :
2497 81597 : if (CONSTANT_P (src))
2498 : {
2499 : /* simplify_gen_subreg can't be used here, as if simplify_subreg
2500 : fails, it will happily create (subreg (symbol_ref)) or similar
2501 : invalid SUBREGs. */
2502 17040 : poly_uint64 byte = subreg_lowpart_offset (mode, src_mode);
2503 17040 : rtx ret = simplify_subreg (mode, src, src_mode, byte);
2504 17040 : if (ret)
2505 : return ret;
2506 :
2507 22 : if (GET_MODE (src) == VOIDmode
2508 22 : || !validate_subreg (mode, src_mode, src, byte))
2509 : return NULL_RTX;
2510 :
2511 17 : src = force_reg (GET_MODE (src), src);
2512 17 : return gen_rtx_SUBREG (mode, src, byte);
2513 : }
2514 :
2515 64557 : if (GET_MODE_CLASS (mode) == MODE_CC || GET_MODE_CLASS (src_mode) == MODE_CC)
2516 : return NULL_RTX;
2517 :
2518 129114 : if (known_eq (GET_MODE_BITSIZE (mode), GET_MODE_BITSIZE (src_mode))
2519 64557 : && targetm.modes_tieable_p (mode, src_mode))
2520 : {
2521 4153 : rtx x = gen_lowpart_common (mode, src);
2522 4153 : if (x)
2523 : return x;
2524 : }
2525 :
2526 60415 : if (!int_mode_for_mode (src_mode).exists (&src_int_mode)
2527 60415 : || !int_mode_for_mode (mode).exists (&int_mode))
2528 : return NULL_RTX;
2529 :
2530 60402 : if (!targetm.modes_tieable_p (src_int_mode, src_mode))
2531 : return NULL_RTX;
2532 59314 : if (!targetm.modes_tieable_p (int_mode, mode))
2533 : return NULL_RTX;
2534 :
2535 57246 : src = gen_lowpart (src_int_mode, src);
2536 57246 : if (!validate_subreg (int_mode, src_int_mode, src,
2537 : subreg_lowpart_offset (int_mode, src_int_mode)))
2538 : return NULL_RTX;
2539 :
2540 57234 : src = convert_modes (int_mode, src_int_mode, src, true);
2541 57234 : src = gen_lowpart (mode, src);
2542 57234 : return src;
2543 : }
2544 :
2545 : /* Add INC into TARGET. */
2546 :
2547 : void
2548 1186 : expand_inc (rtx target, rtx inc)
2549 : {
2550 1186 : rtx value = expand_binop (GET_MODE (target), add_optab,
2551 : target, inc,
2552 : target, 0, OPTAB_LIB_WIDEN);
2553 1186 : if (value != target)
2554 61 : emit_move_insn (target, value);
2555 1186 : }
2556 :
2557 : /* Subtract DEC from TARGET. */
2558 :
2559 : void
2560 1221 : expand_dec (rtx target, rtx dec)
2561 : {
2562 1221 : rtx value = expand_binop (GET_MODE (target), sub_optab,
2563 : target, dec,
2564 : target, 0, OPTAB_LIB_WIDEN);
2565 1221 : if (value != target)
2566 0 : emit_move_insn (target, value);
2567 1221 : }
2568 :
2569 : /* Output a shift instruction for expression code CODE,
2570 : with SHIFTED being the rtx for the value to shift,
2571 : and AMOUNT the rtx for the amount to shift by.
2572 : Store the result in the rtx TARGET, if that is convenient.
2573 : If UNSIGNEDP is nonzero, do a logical shift; otherwise, arithmetic.
2574 : Return the rtx for where the value is.
2575 : If that cannot be done, abort the compilation unless MAY_FAIL is true,
2576 : in which case 0 is returned. */
2577 :
2578 : static rtx
2579 1536035 : expand_shift_1 (enum tree_code code, machine_mode mode, rtx shifted,
2580 : rtx amount, rtx target, int unsignedp, bool may_fail = false)
2581 : {
2582 1536035 : rtx op1, temp = 0;
2583 1536035 : int left = (code == LSHIFT_EXPR || code == LROTATE_EXPR);
2584 1536035 : int rotate = (code == LROTATE_EXPR || code == RROTATE_EXPR);
2585 1536035 : optab lshift_optab = ashl_optab;
2586 1536035 : optab rshift_arith_optab = ashr_optab;
2587 1536035 : optab rshift_uns_optab = lshr_optab;
2588 1536035 : optab lrotate_optab = rotl_optab;
2589 1536035 : optab rrotate_optab = rotr_optab;
2590 1536035 : machine_mode op1_mode;
2591 1536035 : scalar_mode scalar_mode = GET_MODE_INNER (mode);
2592 1536035 : int attempt;
2593 1536035 : bool speed = optimize_insn_for_speed_p ();
2594 :
2595 1536035 : op1 = amount;
2596 1536035 : op1_mode = GET_MODE (op1);
2597 :
2598 : /* Determine whether the shift/rotate amount is a vector, or scalar. If the
2599 : shift amount is a vector, use the vector/vector shift patterns. */
2600 1536035 : if (VECTOR_MODE_P (mode) && VECTOR_MODE_P (op1_mode))
2601 : {
2602 1536035 : lshift_optab = vashl_optab;
2603 1536035 : rshift_arith_optab = vashr_optab;
2604 1536035 : rshift_uns_optab = vlshr_optab;
2605 1536035 : lrotate_optab = vrotl_optab;
2606 1536035 : rrotate_optab = vrotr_optab;
2607 : }
2608 :
2609 : /* Previously detected shift-counts computed by NEGATE_EXPR
2610 : and shifted in the other direction; but that does not work
2611 : on all machines. */
2612 :
2613 1536035 : if (SHIFT_COUNT_TRUNCATED)
2614 : {
2615 : if (CONST_INT_P (op1)
2616 : && ((unsigned HOST_WIDE_INT) INTVAL (op1) >=
2617 : (unsigned HOST_WIDE_INT) GET_MODE_BITSIZE (scalar_mode)))
2618 : op1 = gen_int_shift_amount (mode,
2619 : (unsigned HOST_WIDE_INT) INTVAL (op1)
2620 : % GET_MODE_BITSIZE (scalar_mode));
2621 : else if (GET_CODE (op1) == SUBREG
2622 : && subreg_lowpart_p (op1)
2623 : && SCALAR_INT_MODE_P (GET_MODE (SUBREG_REG (op1)))
2624 : && SCALAR_INT_MODE_P (GET_MODE (op1)))
2625 : op1 = SUBREG_REG (op1);
2626 : }
2627 :
2628 : /* Canonicalize rotates by constant amount. We may canonicalize
2629 : to reduce the immediate or if the ISA can rotate by constants
2630 : in only on direction. */
2631 1536035 : if (rotate && reverse_rotate_by_imm_p (scalar_mode, left, op1))
2632 : {
2633 3260 : op1 = gen_int_shift_amount (mode, (GET_MODE_BITSIZE (scalar_mode)
2634 3260 : - INTVAL (op1)));
2635 3260 : left = !left;
2636 3260 : code = left ? LROTATE_EXPR : RROTATE_EXPR;
2637 : }
2638 :
2639 : /* Rotation of 16bit values by 8 bits is effectively equivalent to a bswaphi.
2640 : Note that this is not the case for bigger values. For instance a rotation
2641 : of 0x01020304 by 16 bits gives 0x03040102 which is different from
2642 : 0x04030201 (bswapsi). */
2643 1536035 : if (rotate
2644 8624 : && CONST_INT_P (op1)
2645 5413 : && INTVAL (op1) == BITS_PER_UNIT
2646 1040 : && GET_MODE_SIZE (scalar_mode) == 2
2647 1536900 : && optab_handler (bswap_optab, mode) != CODE_FOR_nothing)
2648 864 : return expand_unop (mode, bswap_optab, shifted, NULL_RTX, unsignedp);
2649 :
2650 1535171 : if (op1 == const0_rtx)
2651 : return shifted;
2652 :
2653 : /* Check whether its cheaper to implement a left shift by a constant
2654 : bit count by a sequence of additions. */
2655 1487419 : if (code == LSHIFT_EXPR
2656 890220 : && CONST_INT_P (op1)
2657 859960 : && INTVAL (op1) > 0
2658 859935 : && INTVAL (op1) < GET_MODE_PRECISION (scalar_mode)
2659 859935 : && INTVAL (op1) < MAX_BITS_PER_WORD
2660 855210 : && (shift_cost (speed, mode, INTVAL (op1))
2661 855210 : > INTVAL (op1) * add_cost (speed, mode))
2662 1490404 : && shift_cost (speed, mode, INTVAL (op1)) != MAX_COST)
2663 : {
2664 : int i;
2665 6166 : for (i = 0; i < INTVAL (op1); i++)
2666 : {
2667 3181 : temp = force_reg (mode, shifted);
2668 3181 : shifted = expand_binop (mode, add_optab, temp, temp, NULL_RTX,
2669 : unsignedp, OPTAB_LIB_WIDEN);
2670 : }
2671 : return shifted;
2672 : }
2673 :
2674 2968902 : for (attempt = 0; temp == 0 && attempt < 3; attempt++)
2675 : {
2676 1484502 : enum optab_methods methods;
2677 :
2678 1484502 : if (attempt == 0)
2679 : methods = OPTAB_DIRECT;
2680 68 : else if (attempt == 1)
2681 : methods = OPTAB_WIDEN;
2682 : else
2683 34 : methods = OPTAB_LIB_WIDEN;
2684 :
2685 1484502 : if (rotate)
2686 : {
2687 : /* Widening does not work for rotation. */
2688 7828 : if (methods == OPTAB_WIDEN)
2689 34 : continue;
2690 7794 : else if (methods == OPTAB_LIB_WIDEN)
2691 : {
2692 : /* If we have been unable to open-code this by a rotation,
2693 : do it as the IOR or PLUS of two shifts. I.e., to rotate
2694 : A by N bits, compute
2695 : (A << N) | ((unsigned) A >> ((-N) & (C - 1)))
2696 : where C is the bitsize of A. If N cannot be zero,
2697 : use PLUS instead of IOR.
2698 :
2699 : It is theoretically possible that the target machine might
2700 : not be able to perform either shift and hence we would
2701 : be making two libcalls rather than just the one for the
2702 : shift (similarly if IOR could not be done). We will allow
2703 : this extremely unlikely lossage to avoid complicating the
2704 : code below. */
2705 :
2706 34 : rtx subtarget = target == shifted ? 0 : target;
2707 34 : rtx new_amount, other_amount;
2708 34 : rtx temp1;
2709 :
2710 34 : new_amount = op1;
2711 34 : if (op1 == const0_rtx)
2712 : return shifted;
2713 34 : else if (CONST_INT_P (op1))
2714 23 : other_amount = gen_int_shift_amount
2715 23 : (mode, GET_MODE_BITSIZE (scalar_mode) - INTVAL (op1));
2716 : else
2717 : {
2718 11 : other_amount
2719 22 : = simplify_gen_unary (NEG, GET_MODE (op1),
2720 11 : op1, GET_MODE (op1));
2721 11 : HOST_WIDE_INT mask = GET_MODE_PRECISION (scalar_mode) - 1;
2722 11 : other_amount
2723 11 : = simplify_gen_binary (AND, GET_MODE (op1), other_amount,
2724 11 : gen_int_mode (mask, GET_MODE (op1)));
2725 : }
2726 :
2727 34 : shifted = force_reg (mode, shifted);
2728 :
2729 45 : temp = expand_shift_1 (left ? LSHIFT_EXPR : RSHIFT_EXPR,
2730 : mode, shifted, new_amount, 0, 1);
2731 45 : temp1 = expand_shift_1 (left ? RSHIFT_EXPR : LSHIFT_EXPR,
2732 : mode, shifted, other_amount,
2733 : subtarget, 1);
2734 34 : return expand_binop (mode,
2735 34 : CONST_INT_P (op1) ? add_optab : ior_optab,
2736 34 : temp, temp1, target, unsignedp, methods);
2737 : }
2738 :
2739 11364 : temp = expand_binop (mode,
2740 : left ? lrotate_optab : rrotate_optab,
2741 : shifted, op1, target, unsignedp, methods);
2742 : }
2743 1476674 : else if (unsignedp)
2744 1229903 : temp = expand_binop (mode,
2745 : left ? lshift_optab : rshift_uns_optab,
2746 : shifted, op1, target, unsignedp, methods);
2747 :
2748 : /* Do arithmetic shifts.
2749 : Also, if we are going to widen the operand, we can just as well
2750 : use an arithmetic right-shift instead of a logical one. */
2751 1484434 : if (temp == 0 && ! rotate
2752 590373 : && (! unsignedp || (! left && methods == OPTAB_WIDEN)))
2753 : {
2754 : enum optab_methods methods1 = methods;
2755 :
2756 : /* If trying to widen a log shift to an arithmetic shift,
2757 : don't accept an arithmetic shift of the same size. */
2758 : if (unsignedp)
2759 : methods1 = OPTAB_MUST_WIDEN;
2760 :
2761 : /* Arithmetic shift */
2762 :
2763 836210 : temp = expand_binop (mode,
2764 : left ? lshift_optab : rshift_arith_optab,
2765 : shifted, op1, target, unsignedp, methods1);
2766 : }
2767 :
2768 : /* We used to try extzv here for logical right shifts, but that was
2769 : only useful for one machine, the VAX, and caused poor code
2770 : generation there for lshrdi3, so the code was deleted and a
2771 : define_expand for lshrsi3 was added to vax.md. */
2772 : }
2773 :
2774 1484400 : gcc_assert (temp != NULL_RTX || may_fail);
2775 : return temp;
2776 : }
2777 :
2778 : /* Output a shift instruction for expression code CODE,
2779 : with SHIFTED being the rtx for the value to shift,
2780 : and AMOUNT the amount to shift by.
2781 : Store the result in the rtx TARGET, if that is convenient.
2782 : If UNSIGNEDP is nonzero, do a logical shift; otherwise, arithmetic.
2783 : Return the rtx for where the value is. */
2784 :
2785 : rtx
2786 1230807 : expand_shift (enum tree_code code, machine_mode mode, rtx shifted,
2787 : poly_int64 amount, rtx target, int unsignedp)
2788 : {
2789 1230807 : return expand_shift_1 (code, mode, shifted,
2790 : gen_int_shift_amount (mode, amount),
2791 1230807 : target, unsignedp);
2792 : }
2793 :
2794 : /* Likewise, but return 0 if that cannot be done. */
2795 :
2796 : rtx
2797 321 : maybe_expand_shift (enum tree_code code, machine_mode mode, rtx shifted,
2798 : int amount, rtx target, int unsignedp)
2799 : {
2800 321 : return expand_shift_1 (code, mode,
2801 321 : shifted, GEN_INT (amount), target, unsignedp, true);
2802 : }
2803 :
2804 : /* Output a shift instruction for expression code CODE,
2805 : with SHIFTED being the rtx for the value to shift,
2806 : and AMOUNT the tree for the amount to shift by.
2807 : Store the result in the rtx TARGET, if that is convenient.
2808 : If UNSIGNEDP is nonzero, do a logical shift; otherwise, arithmetic.
2809 : Return the rtx for where the value is. */
2810 :
2811 : rtx
2812 304839 : expand_variable_shift (enum tree_code code, machine_mode mode, rtx shifted,
2813 : tree amount, rtx target, int unsignedp)
2814 : {
2815 304839 : return expand_shift_1 (code, mode,
2816 304839 : shifted, expand_normal (amount), target, unsignedp);
2817 : }
2818 :
2819 :
2820 : static void synth_mult (struct algorithm *, unsigned HOST_WIDE_INT,
2821 : const struct mult_cost *, machine_mode mode);
2822 : static rtx expand_mult_const (machine_mode, rtx, HOST_WIDE_INT, rtx,
2823 : const struct algorithm *, enum mult_variant);
2824 : static unsigned HOST_WIDE_INT invert_mod2n (unsigned HOST_WIDE_INT, int);
2825 : static rtx extract_high_half (scalar_int_mode, rtx);
2826 : static rtx expmed_mult_highpart (scalar_int_mode, rtx, rtx, rtx, int, int);
2827 :
2828 : /* Compute and return the best algorithm for multiplying by T.
2829 : The algorithm must cost less than cost_limit
2830 : If retval.cost >= COST_LIMIT, no algorithm was found and all
2831 : other field of the returned struct are undefined.
2832 : MODE is the machine mode of the multiplication. */
2833 :
2834 : static void
2835 38309832 : synth_mult (struct algorithm *alg_out, unsigned HOST_WIDE_INT t,
2836 : const struct mult_cost *cost_limit, machine_mode mode)
2837 : {
2838 38309832 : int m;
2839 38309832 : struct algorithm *alg_in, *best_alg;
2840 38309832 : struct mult_cost best_cost;
2841 38309832 : struct mult_cost new_limit;
2842 38309832 : int op_cost, op_latency;
2843 38309832 : unsigned HOST_WIDE_INT orig_t = t;
2844 38309832 : unsigned HOST_WIDE_INT q;
2845 38309832 : int maxm, hash_index;
2846 38309832 : bool cache_hit = false;
2847 38309832 : enum alg_code cache_alg = alg_zero;
2848 38309832 : bool speed = optimize_insn_for_speed_p ();
2849 38309832 : scalar_int_mode imode;
2850 38309832 : struct alg_hash_entry *entry_ptr;
2851 :
2852 : /* Indicate that no algorithm is yet found. If no algorithm
2853 : is found, this value will be returned and indicate failure. */
2854 38309832 : alg_out->cost.cost = cost_limit->cost + 1;
2855 38309832 : alg_out->cost.latency = cost_limit->latency + 1;
2856 :
2857 38309832 : if (cost_limit->cost < 0
2858 31280089 : || (cost_limit->cost == 0 && cost_limit->latency <= 0))
2859 30950864 : return;
2860 :
2861 : /* Be prepared for vector modes. */
2862 51804834 : imode = as_a <scalar_int_mode> (GET_MODE_INNER (mode));
2863 :
2864 76277524 : maxm = MIN (BITS_PER_WORD, GET_MODE_BITSIZE (imode));
2865 :
2866 : /* Restrict the bits of "t" to the multiplication's mode. */
2867 25902417 : t &= GET_MODE_MASK (imode);
2868 :
2869 : /* t == 1 can be done in zero cost. */
2870 25902417 : if (t == 1)
2871 : {
2872 5997940 : alg_out->ops = 1;
2873 5997940 : alg_out->cost.cost = 0;
2874 5997940 : alg_out->cost.latency = 0;
2875 5997940 : alg_out->op[0] = alg_m;
2876 5997940 : return;
2877 : }
2878 :
2879 : /* t == 0 sometimes has a cost. If it does and it exceeds our limit,
2880 : fail now. */
2881 19904477 : if (t == 0)
2882 : {
2883 534885 : if (MULT_COST_LESS (cost_limit, zero_cost (speed)))
2884 : return;
2885 : else
2886 : {
2887 534885 : alg_out->ops = 1;
2888 534885 : alg_out->cost.cost = zero_cost (speed);
2889 534885 : alg_out->cost.latency = zero_cost (speed);
2890 534885 : alg_out->op[0] = alg_zero;
2891 534885 : return;
2892 : }
2893 : }
2894 :
2895 : /* We'll be needing a couple extra algorithm structures now. */
2896 :
2897 19369592 : alg_in = XALLOCA (struct algorithm);
2898 19369592 : best_alg = XALLOCA (struct algorithm);
2899 19369592 : best_cost = *cost_limit;
2900 :
2901 : /* Compute the hash index. */
2902 19369592 : hash_index = (t ^ (unsigned int) mode ^ (speed * 256)) % NUM_ALG_HASH_ENTRIES;
2903 :
2904 : /* See if we already know what to do for T. */
2905 19369592 : entry_ptr = alg_hash_entry_ptr (hash_index);
2906 19369592 : if (entry_ptr->t == t
2907 15211211 : && entry_ptr->mode == mode
2908 15211211 : && entry_ptr->speed == speed
2909 15211211 : && entry_ptr->alg != alg_unknown)
2910 : {
2911 15211211 : cache_alg = entry_ptr->alg;
2912 :
2913 15211211 : if (cache_alg == alg_impossible)
2914 : {
2915 : /* The cache tells us that it's impossible to synthesize
2916 : multiplication by T within entry_ptr->cost. */
2917 7484519 : if (!CHEAPER_MULT_COST (&entry_ptr->cost, cost_limit))
2918 : /* COST_LIMIT is at least as restrictive as the one
2919 : recorded in the hash table, in which case we have no
2920 : hope of synthesizing a multiplication. Just
2921 : return. */
2922 : return;
2923 :
2924 : /* If we get here, COST_LIMIT is less restrictive than the
2925 : one recorded in the hash table, so we may be able to
2926 : synthesize a multiplication. Proceed as if we didn't
2927 : have the cache entry. */
2928 : }
2929 : else
2930 : {
2931 7726692 : if (CHEAPER_MULT_COST (cost_limit, &entry_ptr->cost))
2932 : /* The cached algorithm shows that this multiplication
2933 : requires more cost than COST_LIMIT. Just return. This
2934 : way, we don't clobber this cache entry with
2935 : alg_impossible but retain useful information. */
2936 : return;
2937 :
2938 6975389 : cache_hit = true;
2939 :
2940 6975389 : switch (cache_alg)
2941 : {
2942 4604658 : case alg_shift:
2943 4604658 : goto do_alg_shift;
2944 :
2945 939602 : case alg_add_t_m2:
2946 939602 : case alg_sub_t_m2:
2947 939602 : goto do_alg_addsub_t_m2;
2948 :
2949 117827 : case alg_add_factor:
2950 117827 : case alg_sub_factor:
2951 117827 : goto do_alg_addsub_factor;
2952 :
2953 1313291 : case alg_add_t2_m:
2954 1313291 : goto do_alg_add_t2_m;
2955 :
2956 11 : case alg_sub_t2_m:
2957 11 : goto do_alg_sub_t2_m;
2958 :
2959 0 : default:
2960 0 : gcc_unreachable ();
2961 : }
2962 : }
2963 : }
2964 :
2965 : /* If we have a group of zero bits at the low-order part of T, try
2966 : multiplying by the remaining bits and then doing a shift. */
2967 :
2968 5148207 : if ((t & 1) == 0)
2969 : {
2970 2643933 : do_alg_shift:
2971 7248591 : m = ctz_or_zero (t); /* m = number of low zero bits */
2972 7248591 : if (m < maxm)
2973 : {
2974 7246849 : q = t >> m;
2975 : /* The function expand_shift will choose between a shift and
2976 : a sequence of additions, so the observed cost is given as
2977 : MIN (m * add_cost(speed, mode), shift_cost(speed, mode, m)). */
2978 7246849 : op_cost = m * add_cost (speed, mode);
2979 7246849 : if (shift_cost (speed, mode, m) < op_cost)
2980 : op_cost = shift_cost (speed, mode, m);
2981 7246849 : new_limit.cost = best_cost.cost - op_cost;
2982 7246849 : new_limit.latency = best_cost.latency - op_cost;
2983 7246849 : synth_mult (alg_in, q, &new_limit, mode);
2984 :
2985 7246849 : alg_in->cost.cost += op_cost;
2986 7246849 : alg_in->cost.latency += op_cost;
2987 7246849 : if (CHEAPER_MULT_COST (&alg_in->cost, &best_cost))
2988 : {
2989 4228624 : best_cost = alg_in->cost;
2990 4228624 : std::swap (alg_in, best_alg);
2991 4228624 : best_alg->log[best_alg->ops] = m;
2992 4228624 : best_alg->op[best_alg->ops] = alg_shift;
2993 : }
2994 :
2995 : /* See if treating ORIG_T as a signed number yields a better
2996 : sequence. Try this sequence only for a negative ORIG_T
2997 : as it would be useless for a non-negative ORIG_T. */
2998 7246849 : if ((HOST_WIDE_INT) orig_t < 0)
2999 : {
3000 : /* Shift ORIG_T as follows because a right shift of a
3001 : negative-valued signed type is implementation
3002 : defined. */
3003 630913 : q = ~(~orig_t >> m);
3004 : /* The function expand_shift will choose between a shift
3005 : and a sequence of additions, so the observed cost is
3006 : given as MIN (m * add_cost(speed, mode),
3007 : shift_cost(speed, mode, m)). */
3008 630913 : op_cost = m * add_cost (speed, mode);
3009 630913 : if (shift_cost (speed, mode, m) < op_cost)
3010 : op_cost = shift_cost (speed, mode, m);
3011 630913 : new_limit.cost = best_cost.cost - op_cost;
3012 630913 : new_limit.latency = best_cost.latency - op_cost;
3013 630913 : synth_mult (alg_in, q, &new_limit, mode);
3014 :
3015 630913 : alg_in->cost.cost += op_cost;
3016 630913 : alg_in->cost.latency += op_cost;
3017 630913 : if (CHEAPER_MULT_COST (&alg_in->cost, &best_cost))
3018 : {
3019 599345 : best_cost = alg_in->cost;
3020 599345 : std::swap (alg_in, best_alg);
3021 599345 : best_alg->log[best_alg->ops] = m;
3022 599345 : best_alg->op[best_alg->ops] = alg_shift;
3023 : }
3024 : }
3025 : }
3026 1742 : else if (2 * BITS_PER_WORD <= HOST_BITS_PER_WIDE_INT
3027 1742 : && GET_MODE_BITSIZE (imode) == 2 * BITS_PER_WORD
3028 1730 : && m >= BITS_PER_WORD
3029 3472 : && imode == mode)
3030 : {
3031 1730 : q = t >> m;
3032 1730 : int op1_cost = shift_cost (speed, mode, m - BITS_PER_WORD);
3033 1730 : int op2_cost = zero_cost (speed);
3034 1730 : op_latency = MAX (op1_cost, op2_cost);
3035 1730 : op_cost = op1_cost + op2_cost;
3036 :
3037 1730 : new_limit.cost = best_cost.cost - op_cost;
3038 1730 : new_limit.latency = best_cost.latency - op_latency;
3039 1730 : synth_mult (alg_in, q, &new_limit, mode);
3040 1730 : alg_in->cost.cost += op_cost;
3041 1730 : alg_in->cost.latency += op_latency;
3042 1730 : if (CHEAPER_MULT_COST (&alg_in->cost, &best_cost))
3043 : {
3044 1666 : best_cost = alg_in->cost;
3045 1666 : std::swap (alg_in, best_alg);
3046 1666 : best_alg->log[best_alg->ops] = m;
3047 1666 : best_alg->op[best_alg->ops] = alg_shift;
3048 : }
3049 : }
3050 7248591 : if (cache_hit)
3051 4604658 : goto done;
3052 : }
3053 :
3054 : /* If we have an odd number, add or subtract one. */
3055 2643933 : if ((t & 1) != 0)
3056 : {
3057 3443876 : unsigned HOST_WIDE_INT w;
3058 :
3059 0 : do_alg_addsub_t_m2:
3060 42171708 : for (w = 1; (w & t) != 0; w <<= 1)
3061 : ;
3062 : /* If T was -1, then W will be zero after the loop. This is another
3063 : case where T ends with ...111. Handling this with (T + 1) and
3064 : subtract 1 produces slightly better code and results in algorithm
3065 : selection much faster than treating it like the ...0111 case
3066 : below. */
3067 3443876 : if (w == 0
3068 3025888 : || (w > 2
3069 : /* Reject the case where t is 3.
3070 : Thus we prefer addition in that case. */
3071 3025888 : && t != 3))
3072 : {
3073 : /* T ends with ...111. Multiply by (T + 1) and subtract T. */
3074 :
3075 1895650 : op_cost = add_cost (speed, mode);
3076 1895650 : new_limit.cost = best_cost.cost - op_cost;
3077 1895650 : new_limit.latency = best_cost.latency - op_cost;
3078 1895650 : synth_mult (alg_in, t + 1, &new_limit, mode);
3079 :
3080 1895650 : alg_in->cost.cost += op_cost;
3081 1895650 : alg_in->cost.latency += op_cost;
3082 1895650 : if (CHEAPER_MULT_COST (&alg_in->cost, &best_cost))
3083 : {
3084 711844 : best_cost = alg_in->cost;
3085 711844 : std::swap (alg_in, best_alg);
3086 711844 : best_alg->log[best_alg->ops] = 0;
3087 711844 : best_alg->op[best_alg->ops] = alg_sub_t_m2;
3088 : }
3089 : }
3090 : else
3091 : {
3092 : /* T ends with ...01 or ...011. Multiply by (T - 1) and add T. */
3093 :
3094 1548226 : op_cost = add_cost (speed, mode);
3095 1548226 : new_limit.cost = best_cost.cost - op_cost;
3096 1548226 : new_limit.latency = best_cost.latency - op_cost;
3097 1548226 : synth_mult (alg_in, t - 1, &new_limit, mode);
3098 :
3099 1548226 : alg_in->cost.cost += op_cost;
3100 1548226 : alg_in->cost.latency += op_cost;
3101 1548226 : if (CHEAPER_MULT_COST (&alg_in->cost, &best_cost))
3102 : {
3103 188546 : best_cost = alg_in->cost;
3104 188546 : std::swap (alg_in, best_alg);
3105 188546 : best_alg->log[best_alg->ops] = 0;
3106 188546 : best_alg->op[best_alg->ops] = alg_add_t_m2;
3107 : }
3108 : }
3109 :
3110 : /* We may be able to calculate a * -7, a * -15, a * -31, etc
3111 : quickly with a - a * n for some appropriate constant n. */
3112 3443876 : m = exact_log2 (-orig_t + 1);
3113 3443876 : if (m >= 0 && m < maxm)
3114 : {
3115 740417 : op_cost = add_cost (speed, mode) + shift_cost (speed, mode, m);
3116 : /* If the target has a cheap shift-and-subtract insn use
3117 : that in preference to a shift insn followed by a sub insn.
3118 : Assume that the shift-and-sub is "atomic" with a latency
3119 : equal to it's cost, otherwise assume that on superscalar
3120 : hardware the shift may be executed concurrently with the
3121 : earlier steps in the algorithm. */
3122 740417 : if (shiftsub1_cost (speed, mode, m) <= op_cost)
3123 : {
3124 : op_cost = shiftsub1_cost (speed, mode, m);
3125 : op_latency = op_cost;
3126 : }
3127 : else
3128 734662 : op_latency = add_cost (speed, mode);
3129 :
3130 740417 : new_limit.cost = best_cost.cost - op_cost;
3131 740417 : new_limit.latency = best_cost.latency - op_latency;
3132 740417 : synth_mult (alg_in, (unsigned HOST_WIDE_INT) (-orig_t + 1) >> m,
3133 : &new_limit, mode);
3134 :
3135 740417 : alg_in->cost.cost += op_cost;
3136 740417 : alg_in->cost.latency += op_latency;
3137 740417 : if (CHEAPER_MULT_COST (&alg_in->cost, &best_cost))
3138 : {
3139 189824 : best_cost = alg_in->cost;
3140 189824 : std::swap (alg_in, best_alg);
3141 189824 : best_alg->log[best_alg->ops] = m;
3142 189824 : best_alg->op[best_alg->ops] = alg_sub_t_m2;
3143 : }
3144 : }
3145 :
3146 3443876 : if (cache_hit)
3147 939602 : goto done;
3148 : }
3149 :
3150 : /* Look for factors of t of the form
3151 : t = q(2**m +- 1), 2 <= m <= floor(log2(t - 1)).
3152 : If we find such a factor, we can multiply by t using an algorithm that
3153 : multiplies by q, shift the result by m and add/subtract it to itself.
3154 :
3155 : We search for large factors first and loop down, even if large factors
3156 : are less probable than small; if we find a large factor we will find a
3157 : good sequence quickly, and therefore be able to prune (by decreasing
3158 : COST_LIMIT) the search. */
3159 :
3160 2643933 : do_alg_addsub_factor:
3161 111924789 : for (m = floor_log2 (t - 1); m >= 2; m--)
3162 : {
3163 109374892 : unsigned HOST_WIDE_INT d;
3164 :
3165 109374892 : d = (HOST_WIDE_INT_1U << m) + 1;
3166 109374892 : if (t % d == 0 && t > d && m < maxm
3167 1272905 : && (!cache_hit || cache_alg == alg_add_factor))
3168 : {
3169 1272905 : op_cost = add_cost (speed, mode) + shift_cost (speed, mode, m);
3170 1272905 : if (shiftadd_cost (speed, mode, m) <= op_cost)
3171 : op_cost = shiftadd_cost (speed, mode, m);
3172 :
3173 1272905 : op_latency = op_cost;
3174 :
3175 :
3176 1272905 : new_limit.cost = best_cost.cost - op_cost;
3177 1272905 : new_limit.latency = best_cost.latency - op_latency;
3178 1272905 : synth_mult (alg_in, t / d, &new_limit, mode);
3179 :
3180 1272905 : alg_in->cost.cost += op_cost;
3181 1272905 : alg_in->cost.latency += op_latency;
3182 1272905 : if (alg_in->cost.latency < op_cost)
3183 257027 : alg_in->cost.latency = op_cost;
3184 1272905 : if (CHEAPER_MULT_COST (&alg_in->cost, &best_cost))
3185 : {
3186 116459 : best_cost = alg_in->cost;
3187 116459 : std::swap (alg_in, best_alg);
3188 116459 : best_alg->log[best_alg->ops] = m;
3189 116459 : best_alg->op[best_alg->ops] = alg_add_factor;
3190 : }
3191 : /* Other factors will have been taken care of in the recursion. */
3192 : break;
3193 : }
3194 :
3195 108101987 : d = (HOST_WIDE_INT_1U << m) - 1;
3196 108101987 : if (t % d == 0 && t > d && m < maxm
3197 1443232 : && (!cache_hit || cache_alg == alg_sub_factor))
3198 : {
3199 1443232 : op_cost = add_cost (speed, mode) + shift_cost (speed, mode, m);
3200 1443232 : if (shiftsub0_cost (speed, mode, m) <= op_cost)
3201 : op_cost = shiftsub0_cost (speed, mode, m);
3202 :
3203 1443232 : op_latency = op_cost;
3204 :
3205 1443232 : new_limit.cost = best_cost.cost - op_cost;
3206 1443232 : new_limit.latency = best_cost.latency - op_latency;
3207 1443232 : synth_mult (alg_in, t / d, &new_limit, mode);
3208 :
3209 1443232 : alg_in->cost.cost += op_cost;
3210 1443232 : alg_in->cost.latency += op_latency;
3211 1443232 : if (alg_in->cost.latency < op_cost)
3212 351132 : alg_in->cost.latency = op_cost;
3213 1443232 : if (CHEAPER_MULT_COST (&alg_in->cost, &best_cost))
3214 : {
3215 31820 : best_cost = alg_in->cost;
3216 31820 : std::swap (alg_in, best_alg);
3217 31820 : best_alg->log[best_alg->ops] = m;
3218 31820 : best_alg->op[best_alg->ops] = alg_sub_factor;
3219 : }
3220 : break;
3221 : }
3222 : }
3223 5266034 : if (cache_hit)
3224 117827 : goto done;
3225 :
3226 : /* Try shift-and-add (load effective address) instructions,
3227 : i.e. do a*3, a*5, a*9. */
3228 5148207 : if ((t & 1) != 0)
3229 : {
3230 2504274 : do_alg_add_t2_m:
3231 3817565 : q = t - 1;
3232 3817565 : m = ctz_hwi (q);
3233 3817565 : if (q && m < maxm)
3234 : {
3235 3817543 : op_cost = shiftadd_cost (speed, mode, m);
3236 3817543 : new_limit.cost = best_cost.cost - op_cost;
3237 3817543 : new_limit.latency = best_cost.latency - op_cost;
3238 3817543 : synth_mult (alg_in, (t - 1) >> m, &new_limit, mode);
3239 :
3240 3817543 : alg_in->cost.cost += op_cost;
3241 3817543 : alg_in->cost.latency += op_cost;
3242 3817543 : if (CHEAPER_MULT_COST (&alg_in->cost, &best_cost))
3243 : {
3244 1366899 : best_cost = alg_in->cost;
3245 1366899 : std::swap (alg_in, best_alg);
3246 1366899 : best_alg->log[best_alg->ops] = m;
3247 1366899 : best_alg->op[best_alg->ops] = alg_add_t2_m;
3248 : }
3249 : }
3250 3817565 : if (cache_hit)
3251 1313291 : goto done;
3252 :
3253 2504274 : do_alg_sub_t2_m:
3254 2504285 : q = t + 1;
3255 2504285 : m = ctz_hwi (q);
3256 2504285 : if (q && m < maxm)
3257 : {
3258 2482021 : op_cost = shiftsub0_cost (speed, mode, m);
3259 2482021 : new_limit.cost = best_cost.cost - op_cost;
3260 2482021 : new_limit.latency = best_cost.latency - op_cost;
3261 2482021 : synth_mult (alg_in, (t + 1) >> m, &new_limit, mode);
3262 :
3263 2482021 : alg_in->cost.cost += op_cost;
3264 2482021 : alg_in->cost.latency += op_cost;
3265 2482021 : if (CHEAPER_MULT_COST (&alg_in->cost, &best_cost))
3266 : {
3267 95 : best_cost = alg_in->cost;
3268 95 : std::swap (alg_in, best_alg);
3269 95 : best_alg->log[best_alg->ops] = m;
3270 95 : best_alg->op[best_alg->ops] = alg_sub_t2_m;
3271 : }
3272 : }
3273 : if (cache_hit)
3274 : goto done;
3275 : }
3276 :
3277 2643933 : done:
3278 : /* If best_cost has not decreased, we have not found any algorithm. */
3279 12123596 : if (!CHEAPER_MULT_COST (&best_cost, cost_limit))
3280 : {
3281 : /* We failed to find an algorithm. Record alg_impossible for
3282 : this case (that is, <T, MODE, COST_LIMIT>) so that next time
3283 : we are asked to find an algorithm for T within the same or
3284 : lower COST_LIMIT, we can immediately return to the
3285 : caller. */
3286 4764628 : entry_ptr->t = t;
3287 4764628 : entry_ptr->mode = mode;
3288 4764628 : entry_ptr->speed = speed;
3289 4764628 : entry_ptr->alg = alg_impossible;
3290 4764628 : entry_ptr->cost = *cost_limit;
3291 4764628 : return;
3292 : }
3293 :
3294 : /* Cache the result. */
3295 7358968 : if (!cache_hit)
3296 : {
3297 695406 : entry_ptr->t = t;
3298 695406 : entry_ptr->mode = mode;
3299 695406 : entry_ptr->speed = speed;
3300 695406 : entry_ptr->alg = best_alg->op[best_alg->ops];
3301 695406 : entry_ptr->cost.cost = best_cost.cost;
3302 695406 : entry_ptr->cost.latency = best_cost.latency;
3303 : }
3304 :
3305 : /* If we are getting a too long sequence for `struct algorithm'
3306 : to record, make this search fail. */
3307 7358968 : if (best_alg->ops == MAX_BITS_PER_WORD)
3308 : return;
3309 :
3310 : /* Copy the algorithm from temporary space to the space at alg_out.
3311 : We avoid using structure assignment because the majority of
3312 : best_alg is normally undefined, and this is a critical function. */
3313 7358968 : alg_out->ops = best_alg->ops + 1;
3314 7358968 : alg_out->cost = best_cost;
3315 7358968 : memcpy (alg_out->op, best_alg->op,
3316 7358968 : alg_out->ops * sizeof *alg_out->op);
3317 7358968 : memcpy (alg_out->log, best_alg->log,
3318 : alg_out->ops * sizeof *alg_out->log);
3319 : }
3320 :
3321 : /* Find the cheapest way of multiplying a value of mode MODE by VAL.
3322 : Try three variations:
3323 :
3324 : - a shift/add sequence based on VAL itself
3325 : - a shift/add sequence based on -VAL, followed by a negation
3326 : - a shift/add sequence based on VAL - 1, followed by an addition.
3327 :
3328 : Return true if the cheapest of these cost less than MULT_COST,
3329 : describing the algorithm in *ALG and final fixup in *VARIANT. */
3330 :
3331 : bool
3332 7306510 : choose_mult_variant (machine_mode mode, HOST_WIDE_INT val,
3333 : struct algorithm *alg, enum mult_variant *variant,
3334 : int mult_cost)
3335 : {
3336 7306510 : struct algorithm alg2;
3337 7306510 : struct mult_cost limit;
3338 7306510 : int op_cost;
3339 7306510 : bool speed = optimize_insn_for_speed_p ();
3340 :
3341 : /* Fail quickly for impossible bounds. */
3342 7306510 : if (mult_cost < 0)
3343 : return false;
3344 :
3345 : /* Ensure that mult_cost provides a reasonable upper bound.
3346 : Any constant multiplication can be performed with less
3347 : than 2 * bits additions. */
3348 14609908 : op_cost = 2 * GET_MODE_UNIT_BITSIZE (mode) * add_cost (speed, mode);
3349 7304954 : if (mult_cost > op_cost)
3350 : mult_cost = op_cost;
3351 :
3352 7304954 : *variant = basic_variant;
3353 7304954 : limit.cost = mult_cost;
3354 7304954 : limit.latency = mult_cost;
3355 7304954 : synth_mult (alg, val, &limit, mode);
3356 :
3357 : /* This works only if the inverted value actually fits in an
3358 : `unsigned int' */
3359 14609908 : if (HOST_BITS_PER_INT >= GET_MODE_UNIT_BITSIZE (mode))
3360 : {
3361 2620438 : op_cost = neg_cost (speed, mode);
3362 2620438 : if (MULT_COST_LESS (&alg->cost, mult_cost))
3363 : {
3364 2513832 : limit.cost = alg->cost.cost - op_cost;
3365 2513832 : limit.latency = alg->cost.latency - op_cost;
3366 : }
3367 : else
3368 : {
3369 106606 : limit.cost = mult_cost - op_cost;
3370 106606 : limit.latency = mult_cost - op_cost;
3371 : }
3372 :
3373 2620438 : synth_mult (&alg2, -val, &limit, mode);
3374 2620438 : alg2.cost.cost += op_cost;
3375 2620438 : alg2.cost.latency += op_cost;
3376 2620438 : if (CHEAPER_MULT_COST (&alg2.cost, &alg->cost))
3377 14490 : *alg = alg2, *variant = negate_variant;
3378 : }
3379 :
3380 : /* This proves very useful for division-by-constant. */
3381 7304954 : op_cost = add_cost (speed, mode);
3382 7304954 : if (MULT_COST_LESS (&alg->cost, mult_cost))
3383 : {
3384 6452501 : limit.cost = alg->cost.cost - op_cost;
3385 6452501 : limit.latency = alg->cost.latency - op_cost;
3386 : }
3387 : else
3388 : {
3389 852453 : limit.cost = mult_cost - op_cost;
3390 852453 : limit.latency = mult_cost - op_cost;
3391 : }
3392 :
3393 7304954 : if (val != HOST_WIDE_INT_MIN
3394 7304964 : || GET_MODE_UNIT_PRECISION (mode) == HOST_BITS_PER_WIDE_INT)
3395 : {
3396 7304954 : synth_mult (&alg2, val - HOST_WIDE_INT_1U, &limit, mode);
3397 7304954 : alg2.cost.cost += op_cost;
3398 7304954 : alg2.cost.latency += op_cost;
3399 7304954 : if (CHEAPER_MULT_COST (&alg2.cost, &alg->cost))
3400 2814 : *alg = alg2, *variant = add_variant;
3401 : }
3402 :
3403 7304954 : return MULT_COST_LESS (&alg->cost, mult_cost);
3404 : }
3405 :
3406 : /* A subroutine of expand_mult, used for constant multiplications.
3407 : Multiply OP0 by VAL in mode MODE, storing the result in TARGET if
3408 : convenient. Use the shift/add sequence described by ALG and apply
3409 : the final fixup specified by VARIANT. */
3410 :
3411 : static rtx
3412 137505 : expand_mult_const (machine_mode mode, rtx op0, HOST_WIDE_INT val,
3413 : rtx target, const struct algorithm *alg,
3414 : enum mult_variant variant)
3415 : {
3416 137505 : unsigned HOST_WIDE_INT val_so_far;
3417 137505 : rtx_insn *insn;
3418 137505 : rtx accum, tem;
3419 137505 : int opno;
3420 137505 : machine_mode nmode;
3421 :
3422 : /* Avoid referencing memory over and over and invalid sharing
3423 : on SUBREGs. */
3424 137505 : op0 = force_reg (mode, op0);
3425 :
3426 : /* ACCUM starts out either as OP0 or as a zero, depending on
3427 : the first operation. */
3428 :
3429 137505 : if (alg->op[0] == alg_zero)
3430 : {
3431 5480 : accum = copy_to_mode_reg (mode, CONST0_RTX (mode));
3432 5480 : val_so_far = 0;
3433 : }
3434 132025 : else if (alg->op[0] == alg_m)
3435 : {
3436 132025 : accum = copy_to_mode_reg (mode, op0);
3437 132025 : val_so_far = 1;
3438 : }
3439 : else
3440 0 : gcc_unreachable ();
3441 :
3442 389055 : for (opno = 1; opno < alg->ops; opno++)
3443 : {
3444 251550 : int log = alg->log[opno];
3445 251550 : rtx shift_subtarget = optimize ? 0 : accum;
3446 235202 : rtx add_target
3447 137505 : = (opno == alg->ops - 1 && target != 0 && variant != add_variant
3448 40764 : && !optimize)
3449 251550 : ? target : 0;
3450 251550 : rtx accum_target = optimize ? 0 : accum;
3451 251550 : rtx accum_inner;
3452 :
3453 251550 : switch (alg->op[opno])
3454 : {
3455 109955 : case alg_shift:
3456 109955 : tem = expand_shift (LSHIFT_EXPR, mode, accum, log, NULL_RTX, 0);
3457 : /* REG_EQUAL note will be attached to the following insn. */
3458 109955 : emit_move_insn (accum, tem);
3459 109955 : val_so_far <<= log;
3460 109955 : break;
3461 :
3462 6198 : case alg_add_t_m2:
3463 6198 : tem = expand_shift (LSHIFT_EXPR, mode, op0, log, NULL_RTX, 0);
3464 12396 : accum = force_operand (gen_rtx_PLUS (mode, accum, tem),
3465 : add_target ? add_target : accum_target);
3466 6198 : val_so_far += HOST_WIDE_INT_1U << log;
3467 6198 : break;
3468 :
3469 19448 : case alg_sub_t_m2:
3470 19448 : tem = expand_shift (LSHIFT_EXPR, mode, op0, log, NULL_RTX, 0);
3471 38896 : accum = force_operand (gen_rtx_MINUS (mode, accum, tem),
3472 : add_target ? add_target : accum_target);
3473 19448 : val_so_far -= HOST_WIDE_INT_1U << log;
3474 19448 : break;
3475 :
3476 114994 : case alg_add_t2_m:
3477 114994 : accum = expand_shift (LSHIFT_EXPR, mode, accum,
3478 114994 : log, shift_subtarget, 0);
3479 229988 : accum = force_operand (gen_rtx_PLUS (mode, accum, op0),
3480 : add_target ? add_target : accum_target);
3481 114994 : val_so_far = (val_so_far << log) + 1;
3482 114994 : break;
3483 :
3484 0 : case alg_sub_t2_m:
3485 0 : accum = expand_shift (LSHIFT_EXPR, mode, accum,
3486 0 : log, shift_subtarget, 0);
3487 0 : accum = force_operand (gen_rtx_MINUS (mode, accum, op0),
3488 : add_target ? add_target : accum_target);
3489 0 : val_so_far = (val_so_far << log) - 1;
3490 0 : break;
3491 :
3492 862 : case alg_add_factor:
3493 862 : tem = expand_shift (LSHIFT_EXPR, mode, accum, log, NULL_RTX, 0);
3494 1724 : accum = force_operand (gen_rtx_PLUS (mode, accum, tem),
3495 : add_target ? add_target : accum_target);
3496 862 : val_so_far += val_so_far << log;
3497 862 : break;
3498 :
3499 93 : case alg_sub_factor:
3500 93 : tem = expand_shift (LSHIFT_EXPR, mode, accum, log, NULL_RTX, 0);
3501 186 : accum = force_operand (gen_rtx_MINUS (mode, tem, accum),
3502 : (add_target
3503 93 : ? add_target : (optimize ? 0 : tem)));
3504 93 : val_so_far = (val_so_far << log) - val_so_far;
3505 93 : break;
3506 :
3507 0 : default:
3508 0 : gcc_unreachable ();
3509 : }
3510 :
3511 251550 : if (SCALAR_INT_MODE_P (mode))
3512 : {
3513 : /* Write a REG_EQUAL note on the last insn so that we can cse
3514 : multiplication sequences. Note that if ACCUM is a SUBREG,
3515 : we've set the inner register and must properly indicate that. */
3516 244847 : tem = op0, nmode = mode;
3517 244847 : accum_inner = accum;
3518 244847 : if (GET_CODE (accum) == SUBREG)
3519 : {
3520 0 : accum_inner = SUBREG_REG (accum);
3521 0 : nmode = GET_MODE (accum_inner);
3522 0 : tem = gen_lowpart (nmode, op0);
3523 : }
3524 :
3525 : /* Don't add a REG_EQUAL note if tem is a paradoxical SUBREG.
3526 : In that case, only the low bits of accum would be guaranteed to
3527 : be equal to the content of the REG_EQUAL note, the upper bits
3528 : can be anything. */
3529 244847 : if (!paradoxical_subreg_p (tem))
3530 : {
3531 244847 : insn = get_last_insn ();
3532 244847 : wide_int wval_so_far
3533 244847 : = wi::uhwi (val_so_far,
3534 244847 : GET_MODE_PRECISION (as_a <scalar_mode> (nmode)));
3535 244847 : rtx c = immed_wide_int_const (wval_so_far, nmode);
3536 244847 : set_dst_reg_note (insn, REG_EQUAL, gen_rtx_MULT (nmode, tem, c),
3537 : accum_inner);
3538 244847 : }
3539 : }
3540 : }
3541 :
3542 137505 : if (variant == negate_variant)
3543 : {
3544 584 : val_so_far = -val_so_far;
3545 584 : accum = expand_unop (mode, neg_optab, accum, target, 0);
3546 : }
3547 136921 : else if (variant == add_variant)
3548 : {
3549 23 : val_so_far = val_so_far + 1;
3550 23 : accum = force_operand (gen_rtx_PLUS (mode, accum, op0), target);
3551 : }
3552 :
3553 : /* Compare only the bits of val and val_so_far that are significant
3554 : in the result mode, to avoid sign-/zero-extension confusion. */
3555 137505 : nmode = GET_MODE_INNER (mode);
3556 137505 : val &= GET_MODE_MASK (nmode);
3557 137505 : val_so_far &= GET_MODE_MASK (nmode);
3558 137505 : gcc_assert (val == (HOST_WIDE_INT) val_so_far);
3559 :
3560 137505 : return accum;
3561 : }
3562 :
3563 : /* Perform a multiplication and return an rtx for the result.
3564 : MODE is mode of value; OP0 and OP1 are what to multiply (rtx's);
3565 : TARGET is a suggestion for where to store the result (an rtx).
3566 :
3567 : We check specially for a constant integer as OP1.
3568 : If you want this check for OP0 as well, then before calling
3569 : you should swap the two operands if OP0 would be constant. */
3570 :
3571 : rtx
3572 1114301 : expand_mult (machine_mode mode, rtx op0, rtx op1, rtx target,
3573 : int unsignedp, bool no_libcall)
3574 : {
3575 1114301 : enum mult_variant variant;
3576 1114301 : struct algorithm algorithm;
3577 1114301 : rtx scalar_op1;
3578 1114301 : int max_cost;
3579 1114301 : bool speed = optimize_insn_for_speed_p ();
3580 1114301 : bool do_trapv = flag_trapv && SCALAR_INT_MODE_P (mode) && !unsignedp;
3581 :
3582 1114301 : if (CONSTANT_P (op0))
3583 275 : std::swap (op0, op1);
3584 :
3585 : /* For vectors, there are several simplifications that can be made if
3586 : all elements of the vector constant are identical. */
3587 1114301 : scalar_op1 = unwrap_const_vec_duplicate (op1);
3588 :
3589 1114301 : if (INTEGRAL_MODE_P (mode))
3590 : {
3591 995322 : rtx fake_reg;
3592 995322 : HOST_WIDE_INT coeff;
3593 995322 : bool is_neg;
3594 995322 : int mode_bitsize;
3595 :
3596 995322 : if (op1 == CONST0_RTX (mode))
3597 : return op1;
3598 995322 : if (op1 == CONST1_RTX (mode))
3599 : return op0;
3600 955843 : if (op1 == CONSTM1_RTX (mode))
3601 2822 : return expand_unop (mode, do_trapv ? negv_optab : neg_optab,
3602 1411 : op0, target, 0);
3603 :
3604 954432 : if (do_trapv)
3605 34 : goto skip_synth;
3606 :
3607 : /* If mode is integer vector mode, check if the backend supports
3608 : vector lshift (by scalar or vector) at all. If not, we can't use
3609 : synthesized multiply. */
3610 954398 : if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT
3611 14652 : && optab_handler (vashl_optab, mode) == CODE_FOR_nothing
3612 966655 : && optab_handler (ashl_optab, mode) == CODE_FOR_nothing)
3613 0 : goto skip_synth;
3614 :
3615 : /* These are the operations that are potentially turned into
3616 : a sequence of shifts and additions. */
3617 954398 : mode_bitsize = GET_MODE_UNIT_BITSIZE (mode);
3618 :
3619 : /* synth_mult does an `unsigned int' multiply. As long as the mode is
3620 : less than or equal in size to `unsigned int' this doesn't matter.
3621 : If the mode is larger than `unsigned int', then synth_mult works
3622 : only if the constant value exactly fits in an `unsigned int' without
3623 : any truncation. This means that multiplying by negative values does
3624 : not work; results are off by 2^32 on a 32 bit machine. */
3625 954398 : if (CONST_INT_P (scalar_op1))
3626 : {
3627 696541 : coeff = INTVAL (scalar_op1);
3628 696541 : is_neg = coeff < 0;
3629 : }
3630 : #if TARGET_SUPPORTS_WIDE_INT
3631 257857 : else if (CONST_WIDE_INT_P (scalar_op1))
3632 : #else
3633 : else if (CONST_DOUBLE_AS_INT_P (scalar_op1))
3634 : #endif
3635 : {
3636 1143 : int shift = wi::exact_log2 (rtx_mode_t (scalar_op1, mode));
3637 : /* Perfect power of 2 (other than 1, which is handled above). */
3638 1143 : if (shift > 0)
3639 106 : return expand_shift (LSHIFT_EXPR, mode, op0,
3640 106 : shift, target, unsignedp);
3641 : else
3642 1037 : goto skip_synth;
3643 : }
3644 : else
3645 256714 : goto skip_synth;
3646 :
3647 : /* We used to test optimize here, on the grounds that it's better to
3648 : produce a smaller program when -O is not used. But this causes
3649 : such a terrible slowdown sometimes that it seems better to always
3650 : use synth_mult. */
3651 :
3652 : /* Special case powers of two. */
3653 696541 : if (EXACT_POWER_OF_2_OR_ZERO_P (coeff)
3654 471367 : && !(is_neg && mode_bitsize > HOST_BITS_PER_WIDE_INT))
3655 471359 : return expand_shift (LSHIFT_EXPR, mode, op0,
3656 942718 : floor_log2 (coeff), target, unsignedp);
3657 :
3658 225182 : fake_reg = gen_raw_REG (mode, LAST_VIRTUAL_REGISTER + 1);
3659 :
3660 : /* Attempt to handle multiplication of DImode values by negative
3661 : coefficients, by performing the multiplication by a positive
3662 : multiplier and then inverting the result. */
3663 225182 : if (is_neg && mode_bitsize > HOST_BITS_PER_WIDE_INT)
3664 : {
3665 : /* Its safe to use -coeff even for INT_MIN, as the
3666 : result is interpreted as an unsigned coefficient.
3667 : Exclude cost of op0 from max_cost to match the cost
3668 : calculation of the synth_mult. */
3669 216 : coeff = -(unsigned HOST_WIDE_INT) coeff;
3670 216 : max_cost = (set_src_cost (gen_rtx_MULT (mode, fake_reg, op1),
3671 : mode, speed)
3672 216 : - neg_cost (speed, mode));
3673 216 : if (max_cost <= 0)
3674 0 : goto skip_synth;
3675 :
3676 : /* Special case powers of two. */
3677 216 : if (EXACT_POWER_OF_2_OR_ZERO_P (coeff))
3678 : {
3679 342 : rtx temp = expand_shift (LSHIFT_EXPR, mode, op0,
3680 171 : floor_log2 (coeff), target, unsignedp);
3681 171 : return expand_unop (mode, neg_optab, temp, target, 0);
3682 : }
3683 :
3684 45 : if (choose_mult_variant (mode, coeff, &algorithm, &variant,
3685 : max_cost))
3686 : {
3687 44 : rtx temp = expand_mult_const (mode, op0, coeff, NULL_RTX,
3688 : &algorithm, variant);
3689 44 : return expand_unop (mode, neg_optab, temp, target, 0);
3690 : }
3691 1 : goto skip_synth;
3692 : }
3693 :
3694 : /* Exclude cost of op0 from max_cost to match the cost
3695 : calculation of the synth_mult. */
3696 224966 : max_cost = set_src_cost (gen_rtx_MULT (mode, fake_reg, op1), mode, speed);
3697 224966 : if (choose_mult_variant (mode, coeff, &algorithm, &variant, max_cost))
3698 136334 : return expand_mult_const (mode, op0, coeff, target,
3699 136334 : &algorithm, variant);
3700 : }
3701 88632 : skip_synth:
3702 :
3703 : /* Expand x*2.0 as x+x. */
3704 38190 : if (CONST_DOUBLE_AS_FLOAT_P (scalar_op1)
3705 503587 : && real_equal (CONST_DOUBLE_REAL_VALUE (scalar_op1), &dconst2))
3706 : {
3707 5940 : op0 = force_reg (GET_MODE (op0), op0);
3708 11880 : return expand_binop (mode, add_optab, op0, op0,
3709 : target, unsignedp,
3710 5940 : no_libcall ? OPTAB_WIDEN : OPTAB_LIB_WIDEN);
3711 : }
3712 :
3713 : /* This used to use umul_optab if unsigned, but for non-widening multiply
3714 : there is no difference between signed and unsigned. */
3715 1378337 : op0 = expand_binop (mode, do_trapv ? smulv_optab : smul_optab,
3716 : op0, op1, target, unsignedp,
3717 : no_libcall ? OPTAB_WIDEN : OPTAB_LIB_WIDEN);
3718 459457 : gcc_assert (op0 || no_libcall);
3719 : return op0;
3720 : }
3721 :
3722 : /* Return a cost estimate for multiplying a register by the given
3723 : COEFFicient in the given MODE and SPEED. */
3724 :
3725 : int
3726 6747804 : mult_by_coeff_cost (HOST_WIDE_INT coeff, machine_mode mode, bool speed)
3727 : {
3728 6747804 : int max_cost;
3729 6747804 : struct algorithm algorithm;
3730 6747804 : enum mult_variant variant;
3731 :
3732 6747804 : rtx fake_reg = gen_raw_REG (mode, LAST_VIRTUAL_REGISTER + 1);
3733 6747804 : max_cost = set_src_cost (gen_rtx_MULT (mode, fake_reg, fake_reg),
3734 : mode, speed);
3735 6747804 : if (choose_mult_variant (mode, coeff, &algorithm, &variant, max_cost))
3736 5987649 : return algorithm.cost.cost;
3737 : else
3738 : return max_cost;
3739 : }
3740 :
3741 : /* Perform a widening multiplication and return an rtx for the result.
3742 : MODE is mode of value; OP0 and OP1 are what to multiply (rtx's);
3743 : TARGET is a suggestion for where to store the result (an rtx).
3744 : THIS_OPTAB is the optab we should use, it must be either umul_widen_optab
3745 : or smul_widen_optab.
3746 :
3747 : We check specially for a constant integer as OP1, comparing the
3748 : cost of a widening multiply against the cost of a sequence of shifts
3749 : and adds. */
3750 :
3751 : rtx
3752 24439 : expand_widening_mult (machine_mode mode, rtx op0, rtx op1, rtx target,
3753 : int unsignedp, optab this_optab)
3754 : {
3755 24439 : bool speed = optimize_insn_for_speed_p ();
3756 24439 : rtx cop1;
3757 :
3758 24439 : if (CONST_INT_P (op1)
3759 5785 : && GET_MODE (op0) != VOIDmode
3760 5785 : && (cop1 = convert_modes (mode, GET_MODE (op0), op1,
3761 : this_optab == umul_widen_optab))
3762 5785 : && CONST_INT_P (cop1)
3763 29703 : && (INTVAL (cop1) >= 0
3764 27092 : || HWI_COMPUTABLE_MODE_P (mode)))
3765 : {
3766 5031 : HOST_WIDE_INT coeff = INTVAL (cop1);
3767 5031 : int max_cost;
3768 5031 : enum mult_variant variant;
3769 5031 : struct algorithm algorithm;
3770 :
3771 5031 : if (coeff == 0)
3772 2378 : return CONST0_RTX (mode);
3773 :
3774 : /* Special case powers of two. */
3775 4327 : if (EXACT_POWER_OF_2_OR_ZERO_P (coeff))
3776 : {
3777 559 : op0 = convert_to_mode (mode, op0, this_optab == umul_widen_optab);
3778 559 : return expand_shift (LSHIFT_EXPR, mode, op0,
3779 559 : floor_log2 (coeff), target, unsignedp);
3780 : }
3781 :
3782 : /* Exclude cost of op0 from max_cost to match the cost
3783 : calculation of the synth_mult. */
3784 3768 : max_cost = mul_widen_cost (speed, mode);
3785 3768 : if (choose_mult_variant (mode, coeff, &algorithm, &variant,
3786 : max_cost))
3787 : {
3788 1115 : op0 = convert_to_mode (mode, op0, this_optab == umul_widen_optab);
3789 1115 : return expand_mult_const (mode, op0, coeff, target,
3790 1115 : &algorithm, variant);
3791 : }
3792 : }
3793 22061 : return expand_binop (mode, this_optab, op0, op1, target,
3794 22061 : unsignedp, OPTAB_LIB_WIDEN);
3795 : }
3796 :
3797 : /* Choose a minimal N + 1 bit approximation to 2**K / D that can be used to
3798 : replace division by D, put the least significant N bits of the result in
3799 : *MULTIPLIER_PTR, the value K - N in *POST_SHIFT_PTR, and return the most
3800 : significant bit.
3801 :
3802 : The width of operations is N (should be <= HOST_BITS_PER_WIDE_INT), the
3803 : needed precision is PRECISION (should be <= N).
3804 :
3805 : PRECISION should be as small as possible so this function can choose the
3806 : multiplier more freely. If PRECISION is <= N - 1, the most significant
3807 : bit returned by the function will be zero.
3808 :
3809 : Using this function, x / D is equal to (x*m) / 2**N >> (*POST_SHIFT_PTR),
3810 : where m is the full N + 1 bit multiplier. */
3811 :
3812 : unsigned HOST_WIDE_INT
3813 64251 : choose_multiplier (unsigned HOST_WIDE_INT d, int n, int precision,
3814 : unsigned HOST_WIDE_INT *multiplier_ptr,
3815 : int *post_shift_ptr)
3816 : {
3817 64251 : int lgup, post_shift;
3818 64251 : int pow1, pow2;
3819 :
3820 : /* lgup = ceil(log2(d)) */
3821 : /* Assuming d > 1, we have d >= 2^(lgup-1) + 1 */
3822 64251 : lgup = ceil_log2 (d);
3823 :
3824 64251 : gcc_assert (lgup <= n);
3825 64251 : gcc_assert (lgup <= precision);
3826 :
3827 64251 : pow1 = n + lgup;
3828 64251 : pow2 = n + lgup - precision;
3829 :
3830 : /* mlow = 2^(n + lgup)/d */
3831 : /* Trivially from above we have mlow < 2^(n+1) */
3832 64251 : wide_int val = wi::set_bit_in_zero (pow1, HOST_BITS_PER_DOUBLE_INT);
3833 64251 : wide_int mlow = wi::udiv_trunc (val, d);
3834 :
3835 : /* mhigh = (2^(n + lgup) + 2^(n + lgup - precision))/d */
3836 : /* From above we have mhigh < 2^(n+1) assuming lgup <= precision */
3837 : /* From precision <= n, the difference between the numerators of mhigh and
3838 : mlow is >= 2^lgup >= d. Therefore the difference of the quotients in
3839 : the Euclidean division by d is at least 1, so we have mlow < mhigh and
3840 : the exact value of 2^(n + lgup)/d lies in the interval [mlow; mhigh). */
3841 64251 : val |= wi::set_bit_in_zero (pow2, HOST_BITS_PER_DOUBLE_INT);
3842 64251 : wide_int mhigh = wi::udiv_trunc (val, d);
3843 :
3844 : /* Reduce to lowest terms. */
3845 : /* If precision <= n - 1, then the difference between the numerators of
3846 : mhigh and mlow is >= 2^(lgup + 1) >= 2 * 2^lgup >= 2 * d. Therefore
3847 : the difference of the quotients in the Euclidean division by d is at
3848 : least 2, which means that mhigh and mlow differ by at least one bit
3849 : not in the last place. The conclusion is that the first iteration of
3850 : the loop below completes and shifts mhigh and mlow by 1 bit, which in
3851 : particular means that mhigh < 2^n, that is to say, the most significant
3852 : bit in the n + 1 bit value is zero. */
3853 232339 : for (post_shift = lgup; post_shift > 0; post_shift--)
3854 : {
3855 162533 : unsigned HOST_WIDE_INT ml_lo = wi::extract_uhwi (mlow, 1,
3856 : HOST_BITS_PER_WIDE_INT);
3857 162533 : unsigned HOST_WIDE_INT mh_lo = wi::extract_uhwi (mhigh, 1,
3858 : HOST_BITS_PER_WIDE_INT);
3859 162533 : if (ml_lo >= mh_lo)
3860 : break;
3861 :
3862 103837 : mlow = wi::uhwi (ml_lo, HOST_BITS_PER_DOUBLE_INT);
3863 103837 : mhigh = wi::uhwi (mh_lo, HOST_BITS_PER_DOUBLE_INT);
3864 : }
3865 :
3866 64251 : *post_shift_ptr = post_shift;
3867 :
3868 64251 : if (n < HOST_BITS_PER_WIDE_INT)
3869 : {
3870 41057 : unsigned HOST_WIDE_INT mask = (HOST_WIDE_INT_1U << n) - 1;
3871 41057 : *multiplier_ptr = mhigh.to_uhwi () & mask;
3872 41057 : return mhigh.to_uhwi () > mask;
3873 : }
3874 : else
3875 : {
3876 23194 : *multiplier_ptr = mhigh.to_uhwi ();
3877 23194 : return wi::extract_uhwi (mhigh, HOST_BITS_PER_WIDE_INT, 1);
3878 : }
3879 64251 : }
3880 :
3881 : /* Compute the inverse of X mod 2**N, i.e., find Y such that X * Y is congruent
3882 : to 1 modulo 2**N, assuming that X is odd. Bézout's lemma guarantees that Y
3883 : exists for any given positive N. */
3884 :
3885 : static unsigned HOST_WIDE_INT
3886 46492 : invert_mod2n (unsigned HOST_WIDE_INT x, int n)
3887 : {
3888 46492 : gcc_assert ((x & 1) == 1);
3889 :
3890 : /* The algorithm notes that the choice Y = X satisfies X*Y == 1 mod 2^3,
3891 : since X is odd. Then each Newton-Raphson iteration doubles the number
3892 : of bits of significance in Y (Hensel's lemma). */
3893 :
3894 47513 : const unsigned HOST_WIDE_INT mask
3895 : = (n == HOST_BITS_PER_WIDE_INT
3896 46492 : ? HOST_WIDE_INT_M1U
3897 1021 : : (HOST_WIDE_INT_1U << n) - 1);
3898 46492 : unsigned HOST_WIDE_INT y = x;
3899 46492 : int nbit = 3;
3900 :
3901 277901 : while (nbit < n)
3902 : {
3903 231409 : y = y * (2 - x*y) & mask; /* Modulo 2^N */
3904 231409 : nbit *= 2;
3905 : }
3906 :
3907 46492 : return y;
3908 : }
3909 :
3910 : /* Emit code to adjust ADJ_OPERAND after multiplication of wrong signedness
3911 : flavor of OP0 and OP1. ADJ_OPERAND is already the high half of the
3912 : product OP0 x OP1. If UNSIGNEDP is nonzero, adjust the signed product
3913 : to become unsigned, if UNSIGNEDP is zero, adjust the unsigned product to
3914 : become signed.
3915 :
3916 : The result is put in TARGET if that is convenient.
3917 :
3918 : MODE is the mode of operation. */
3919 :
3920 : rtx
3921 0 : expand_mult_highpart_adjust (scalar_int_mode mode, rtx adj_operand, rtx op0,
3922 : rtx op1, rtx target, int unsignedp)
3923 : {
3924 0 : rtx tem;
3925 0 : enum rtx_code adj_code = unsignedp ? PLUS : MINUS;
3926 :
3927 0 : tem = expand_shift (RSHIFT_EXPR, mode, op0,
3928 0 : GET_MODE_BITSIZE (mode) - 1, NULL_RTX, 0);
3929 0 : tem = expand_and (mode, tem, op1, NULL_RTX);
3930 0 : adj_operand
3931 0 : = force_operand (gen_rtx_fmt_ee (adj_code, mode, adj_operand, tem),
3932 : adj_operand);
3933 :
3934 0 : tem = expand_shift (RSHIFT_EXPR, mode, op1,
3935 0 : GET_MODE_BITSIZE (mode) - 1, NULL_RTX, 0);
3936 0 : tem = expand_and (mode, tem, op0, NULL_RTX);
3937 0 : target = force_operand (gen_rtx_fmt_ee (adj_code, mode, adj_operand, tem),
3938 : target);
3939 :
3940 0 : return target;
3941 : }
3942 :
3943 : /* Subroutine of expmed_mult_highpart. Return the MODE high part of OP. */
3944 :
3945 : static rtx
3946 19407 : extract_high_half (scalar_int_mode mode, rtx op)
3947 : {
3948 19407 : if (mode == word_mode)
3949 0 : return gen_highpart (mode, op);
3950 :
3951 19407 : scalar_int_mode wider_mode = GET_MODE_WIDER_MODE (mode).require ();
3952 :
3953 38814 : op = expand_shift (RSHIFT_EXPR, wider_mode, op,
3954 19407 : GET_MODE_BITSIZE (mode), 0, 1);
3955 19407 : return convert_modes (mode, wider_mode, op, 0);
3956 : }
3957 :
3958 : /* Like expmed_mult_highpart, but only consider using multiplication optab. */
3959 :
3960 : rtx
3961 45761 : expmed_mult_highpart_optab (scalar_int_mode mode, rtx op0, rtx op1,
3962 : rtx target, int unsignedp, int max_cost)
3963 : {
3964 45761 : const scalar_int_mode wider_mode = GET_MODE_WIDER_MODE (mode).require ();
3965 45761 : const bool speed = optimize_insn_for_speed_p ();
3966 45761 : const int size = GET_MODE_BITSIZE (mode);
3967 45761 : optab moptab;
3968 45761 : rtx tem;
3969 :
3970 : /* Firstly, try using a multiplication insn that only generates the needed
3971 : high part of the product, and in the sign flavor of unsignedp. */
3972 45761 : if (mul_highpart_cost (speed, mode) < max_cost)
3973 : {
3974 43752 : moptab = unsignedp ? umul_highpart_optab : smul_highpart_optab;
3975 43752 : tem = expand_binop (mode, moptab, op0, op1, target, unsignedp,
3976 : OPTAB_DIRECT);
3977 43752 : if (tem)
3978 : return tem;
3979 : }
3980 :
3981 : /* Secondly, same as above, but use sign flavor opposite of unsignedp.
3982 : Need to adjust the result after the multiplication. */
3983 21580 : if (size - 1 < BITS_PER_WORD
3984 42904 : && (mul_highpart_cost (speed, mode)
3985 21324 : + 2 * shift_cost (speed, mode, size-1)
3986 21324 : + 4 * add_cost (speed, mode) < max_cost))
3987 : {
3988 5140 : moptab = unsignedp ? smul_highpart_optab : umul_highpart_optab;
3989 5140 : tem = expand_binop (mode, moptab, op0, op1, target, !unsignedp,
3990 : OPTAB_DIRECT);
3991 5140 : if (tem)
3992 : /* We used the wrong signedness. Adjust the result. */
3993 0 : return expand_mult_highpart_adjust (mode, tem, op0, op1, tem,
3994 0 : unsignedp);
3995 : }
3996 :
3997 : /* Try widening multiplication. */
3998 21580 : moptab = unsignedp ? umul_widen_optab : smul_widen_optab;
3999 21580 : if (convert_optab_handler (moptab, wider_mode, mode) != CODE_FOR_nothing
4000 21580 : && mul_widen_cost (speed, wider_mode) < max_cost)
4001 : {
4002 381 : tem = expand_binop (wider_mode, moptab, op0, op1, NULL_RTX, unsignedp,
4003 : OPTAB_WIDEN);
4004 381 : if (tem)
4005 381 : return extract_high_half (mode, tem);
4006 : }
4007 :
4008 : /* Try widening the mode and perform a non-widening multiplication. */
4009 21199 : if (optab_handler (smul_optab, wider_mode) != CODE_FOR_nothing
4010 20594 : && size - 1 < BITS_PER_WORD
4011 41790 : && (mul_cost (speed, wider_mode) + shift_cost (speed, mode, size-1)
4012 : < max_cost))
4013 : {
4014 19014 : rtx_insn *insns;
4015 19014 : rtx wop0, wop1;
4016 :
4017 : /* We need to widen the operands, for example to ensure the
4018 : constant multiplier is correctly sign or zero extended.
4019 : Use a sequence to clean-up any instructions emitted by
4020 : the conversions if things don't work out. */
4021 19014 : start_sequence ();
4022 19014 : wop0 = convert_modes (wider_mode, mode, op0, unsignedp);
4023 19014 : wop1 = convert_modes (wider_mode, mode, op1, unsignedp);
4024 19014 : tem = expand_binop (wider_mode, smul_optab, wop0, wop1, 0,
4025 : unsignedp, OPTAB_WIDEN);
4026 19014 : insns = end_sequence ();
4027 :
4028 19014 : if (tem)
4029 : {
4030 19014 : emit_insn (insns);
4031 19014 : return extract_high_half (mode, tem);
4032 : }
4033 : }
4034 :
4035 : /* Try widening multiplication of opposite signedness, and adjust. */
4036 2185 : moptab = unsignedp ? smul_widen_optab : umul_widen_optab;
4037 2185 : if (convert_optab_handler (moptab, wider_mode, mode) != CODE_FOR_nothing
4038 455 : && size - 1 < BITS_PER_WORD
4039 2977 : && (mul_widen_cost (speed, wider_mode)
4040 396 : + 2 * shift_cost (speed, mode, size-1)
4041 396 : + 4 * add_cost (speed, mode) < max_cost))
4042 : {
4043 0 : tem = expand_binop (wider_mode, moptab, op0, op1, NULL_RTX, !unsignedp,
4044 : OPTAB_WIDEN);
4045 0 : if (tem != 0)
4046 : {
4047 0 : tem = extract_high_half (mode, tem);
4048 : /* We used the wrong signedness. Adjust the result. */
4049 0 : return expand_mult_highpart_adjust (mode, tem, op0, op1, target,
4050 0 : unsignedp);
4051 : }
4052 : }
4053 :
4054 : return 0;
4055 : }
4056 :
4057 : /* Emit code to multiply OP0 and OP1 (where OP1 is an integer constant),
4058 : putting the high half of the result in TARGET if that is convenient,
4059 : and return where the result is. If the operation cannot be performed,
4060 : 0 is returned.
4061 :
4062 : MODE is the mode of operation and result.
4063 :
4064 : UNSIGNEDP nonzero means unsigned multiply.
4065 :
4066 : MAX_COST is the total allowed cost for the expanded RTL. */
4067 :
4068 : static rtx
4069 45761 : expmed_mult_highpart (scalar_int_mode mode, rtx op0, rtx op1,
4070 : rtx target, int unsignedp, int max_cost)
4071 : {
4072 45761 : const bool speed = optimize_insn_for_speed_p ();
4073 45761 : unsigned HOST_WIDE_INT cnst1;
4074 45761 : int extra_cost;
4075 45761 : bool sign_adjust = false;
4076 45761 : enum mult_variant variant;
4077 45761 : struct algorithm alg;
4078 45761 : rtx narrow_op1, tem;
4079 :
4080 : /* We can't support modes wider than HOST_BITS_PER_INT. */
4081 45761 : gcc_assert (HWI_COMPUTABLE_MODE_P (mode));
4082 :
4083 45761 : cnst1 = INTVAL (op1) & GET_MODE_MASK (mode);
4084 45761 : narrow_op1 = gen_int_mode (INTVAL (op1), mode);
4085 :
4086 : /* We can't optimize modes wider than BITS_PER_WORD.
4087 : ??? We might be able to perform double-word arithmetic if
4088 : mode == word_mode, however all the cost calculations in
4089 : synth_mult etc. assume single-word operations. */
4090 45761 : scalar_int_mode wider_mode = GET_MODE_WIDER_MODE (mode).require ();
4091 94661 : if (GET_MODE_BITSIZE (wider_mode) > BITS_PER_WORD)
4092 24789 : return expmed_mult_highpart_optab (mode, op0, narrow_op1, target,
4093 24789 : unsignedp, max_cost);
4094 :
4095 41944 : extra_cost = shift_cost (speed, mode, GET_MODE_BITSIZE (mode) - 1);
4096 :
4097 : /* Check whether we try to multiply by a negative constant. */
4098 31067 : if (!unsignedp && ((cnst1 >> (GET_MODE_BITSIZE (mode) - 1)) & 1))
4099 : {
4100 2266 : sign_adjust = true;
4101 2266 : extra_cost += add_cost (speed, mode);
4102 : }
4103 :
4104 : /* See whether shift/add multiplication is cheap enough. */
4105 20972 : if (choose_mult_variant (wider_mode, cnst1, &alg, &variant,
4106 : max_cost - extra_cost))
4107 : {
4108 : /* See whether the specialized multiplication optabs are
4109 : cheaper than the shift/add version. */
4110 38654 : tem = expmed_mult_highpart_optab (mode, op0, narrow_op1, target,
4111 : unsignedp,
4112 19327 : alg.cost.cost + extra_cost);
4113 19327 : if (tem)
4114 : return tem;
4115 :
4116 12 : tem = convert_to_mode (wider_mode, op0, unsignedp);
4117 12 : tem = expand_mult_const (wider_mode, tem, cnst1, 0, &alg, variant);
4118 12 : tem = extract_high_half (mode, tem);
4119 :
4120 : /* Adjust result for signedness. */
4121 12 : if (sign_adjust)
4122 0 : tem = force_operand (gen_rtx_MINUS (mode, tem, op0), tem);
4123 :
4124 : return tem;
4125 : }
4126 1645 : return expmed_mult_highpart_optab (mode, op0, narrow_op1, target,
4127 1645 : unsignedp, max_cost);
4128 : }
4129 :
4130 :
4131 : /* Expand signed modulus of OP0 by a power of two D in mode MODE. */
4132 :
4133 : static rtx
4134 2494 : expand_smod_pow2 (scalar_int_mode mode, rtx op0, HOST_WIDE_INT d)
4135 : {
4136 2494 : rtx result, temp, shift;
4137 2494 : rtx_code_label *label;
4138 2494 : int logd;
4139 2494 : int prec = GET_MODE_PRECISION (mode);
4140 :
4141 2494 : logd = floor_log2 (d);
4142 2494 : result = gen_reg_rtx (mode);
4143 :
4144 : /* Avoid conditional branches when they're expensive. */
4145 2494 : if (BRANCH_COST (optimize_insn_for_speed_p (), false) >= 2
4146 2494 : && optimize_insn_for_speed_p ())
4147 : {
4148 2492 : rtx signmask = emit_store_flag (result, LT, op0, const0_rtx,
4149 : mode, 0, -1);
4150 2492 : if (signmask)
4151 : {
4152 2492 : HOST_WIDE_INT masklow = (HOST_WIDE_INT_1 << logd) - 1;
4153 2492 : signmask = force_reg (mode, signmask);
4154 4984 : shift = gen_int_shift_amount (mode, GET_MODE_BITSIZE (mode) - logd);
4155 :
4156 : /* Use the rtx_cost of a LSHIFTRT instruction to determine
4157 : which instruction sequence to use. If logical right shifts
4158 : are expensive the use 2 XORs, 2 SUBs and an AND, otherwise
4159 : use a LSHIFTRT, 1 ADD, 1 SUB and an AND. */
4160 :
4161 2492 : temp = gen_rtx_LSHIFTRT (mode, result, shift);
4162 2492 : if (optab_handler (lshr_optab, mode) == CODE_FOR_nothing
4163 2492 : || (set_src_cost (temp, mode, optimize_insn_for_speed_p ())
4164 : > COSTS_N_INSNS (2)))
4165 : {
4166 106 : temp = expand_binop (mode, xor_optab, op0, signmask,
4167 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4168 106 : temp = expand_binop (mode, sub_optab, temp, signmask,
4169 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4170 106 : temp = expand_binop (mode, and_optab, temp,
4171 106 : gen_int_mode (masklow, mode),
4172 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4173 106 : temp = expand_binop (mode, xor_optab, temp, signmask,
4174 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4175 106 : temp = expand_binop (mode, sub_optab, temp, signmask,
4176 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4177 : }
4178 : else
4179 : {
4180 2386 : signmask = expand_binop (mode, lshr_optab, signmask, shift,
4181 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4182 2386 : signmask = force_reg (mode, signmask);
4183 :
4184 2386 : temp = expand_binop (mode, add_optab, op0, signmask,
4185 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4186 2386 : temp = expand_binop (mode, and_optab, temp,
4187 2386 : gen_int_mode (masklow, mode),
4188 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4189 2386 : temp = expand_binop (mode, sub_optab, temp, signmask,
4190 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4191 : }
4192 : return temp;
4193 : }
4194 : }
4195 :
4196 : /* Mask contains the mode's signbit and the significant bits of the
4197 : modulus. By including the signbit in the operation, many targets
4198 : can avoid an explicit compare operation in the following comparison
4199 : against zero. */
4200 2 : wide_int mask = wi::mask (logd, false, prec);
4201 2 : mask = wi::set_bit (mask, prec - 1);
4202 :
4203 4 : temp = expand_binop (mode, and_optab, op0,
4204 2 : immed_wide_int_const (mask, mode),
4205 : result, 1, OPTAB_LIB_WIDEN);
4206 2 : if (temp != result)
4207 0 : emit_move_insn (result, temp);
4208 :
4209 2 : label = gen_label_rtx ();
4210 2 : do_cmp_and_jump (result, const0_rtx, GE, mode, label);
4211 :
4212 2 : temp = expand_binop (mode, sub_optab, result, const1_rtx, result,
4213 : 0, OPTAB_LIB_WIDEN);
4214 :
4215 2 : mask = wi::mask (logd, true, prec);
4216 4 : temp = expand_binop (mode, ior_optab, temp,
4217 2 : immed_wide_int_const (mask, mode),
4218 : result, 1, OPTAB_LIB_WIDEN);
4219 2 : temp = expand_binop (mode, add_optab, temp, const1_rtx, result,
4220 : 0, OPTAB_LIB_WIDEN);
4221 2 : if (temp != result)
4222 0 : emit_move_insn (result, temp);
4223 2 : emit_label (label);
4224 2 : return result;
4225 2 : }
4226 :
4227 : /* Expand signed division of OP0 by a power of two D in mode MODE.
4228 : This routine is only called for positive values of D. */
4229 :
4230 : static rtx
4231 10306 : expand_sdiv_pow2 (scalar_int_mode mode, rtx op0, HOST_WIDE_INT d)
4232 : {
4233 10306 : rtx temp;
4234 10306 : rtx_code_label *label;
4235 10306 : int logd;
4236 :
4237 10306 : logd = floor_log2 (d);
4238 :
4239 10306 : if (d == 2
4240 10306 : && BRANCH_COST (optimize_insn_for_speed_p (),
4241 : false) >= 1)
4242 : {
4243 6657 : temp = gen_reg_rtx (mode);
4244 6657 : temp = emit_store_flag (temp, LT, op0, const0_rtx, mode, 0, 1);
4245 6657 : if (temp != NULL_RTX)
4246 : {
4247 6657 : temp = expand_binop (mode, add_optab, temp, op0, NULL_RTX,
4248 : 0, OPTAB_LIB_WIDEN);
4249 6657 : return expand_shift (RSHIFT_EXPR, mode, temp, logd, NULL_RTX, 0);
4250 : }
4251 : }
4252 :
4253 7297 : if (HAVE_conditional_move
4254 3649 : && BRANCH_COST (optimize_insn_for_speed_p (), false) >= 2)
4255 : {
4256 3649 : rtx temp2;
4257 :
4258 3649 : start_sequence ();
4259 3649 : temp2 = copy_to_mode_reg (mode, op0);
4260 3649 : temp = expand_binop (mode, add_optab, temp2, gen_int_mode (d - 1, mode),
4261 : NULL_RTX, 0, OPTAB_LIB_WIDEN);
4262 3649 : temp = force_reg (mode, temp);
4263 :
4264 : /* Construct "temp2 = (temp2 < 0) ? temp : temp2". */
4265 3649 : temp2 = emit_conditional_move (temp2, { LT, temp2, const0_rtx, mode },
4266 : temp, temp2, mode, 0);
4267 3649 : if (temp2)
4268 : {
4269 3584 : rtx_insn *seq = end_sequence ();
4270 3584 : emit_insn (seq);
4271 3584 : return expand_shift (RSHIFT_EXPR, mode, temp2, logd, NULL_RTX, 0);
4272 : }
4273 65 : end_sequence ();
4274 : }
4275 :
4276 65 : if (BRANCH_COST (optimize_insn_for_speed_p (),
4277 : false) >= 2)
4278 : {
4279 65 : int ushift = GET_MODE_BITSIZE (mode) - logd;
4280 :
4281 65 : temp = gen_reg_rtx (mode);
4282 65 : temp = emit_store_flag (temp, LT, op0, const0_rtx, mode, 0, -1);
4283 65 : if (temp != NULL_RTX)
4284 : {
4285 130 : if (GET_MODE_BITSIZE (mode) >= BITS_PER_WORD
4286 65 : || shift_cost (optimize_insn_for_speed_p (), mode, ushift)
4287 : > COSTS_N_INSNS (1))
4288 65 : temp = expand_binop (mode, and_optab, temp,
4289 65 : gen_int_mode (d - 1, mode),
4290 : NULL_RTX, 0, OPTAB_LIB_WIDEN);
4291 : else
4292 0 : temp = expand_shift (RSHIFT_EXPR, mode, temp,
4293 0 : ushift, NULL_RTX, 1);
4294 65 : temp = expand_binop (mode, add_optab, temp, op0, NULL_RTX,
4295 : 0, OPTAB_LIB_WIDEN);
4296 65 : return expand_shift (RSHIFT_EXPR, mode, temp, logd, NULL_RTX, 0);
4297 : }
4298 : }
4299 :
4300 0 : label = gen_label_rtx ();
4301 0 : temp = copy_to_mode_reg (mode, op0);
4302 0 : do_cmp_and_jump (temp, const0_rtx, GE, mode, label);
4303 0 : expand_inc (temp, gen_int_mode (d - 1, mode));
4304 0 : emit_label (label);
4305 0 : return expand_shift (RSHIFT_EXPR, mode, temp, logd, NULL_RTX, 0);
4306 : }
4307 :
4308 : /* Emit the code to divide OP0 by OP1, putting the result in TARGET
4309 : if that is convenient, and returning where the result is.
4310 : You may request either the quotient or the remainder as the result;
4311 : specify REM_FLAG nonzero to get the remainder.
4312 :
4313 : CODE is the expression code for which kind of division this is;
4314 : it controls how rounding is done. MODE is the machine mode to use.
4315 : UNSIGNEDP nonzero means do unsigned division. */
4316 :
4317 : /* ??? For CEIL_MOD_EXPR, can compute incorrect remainder with ANDI
4318 : and then correct it by or'ing in missing high bits
4319 : if result of ANDI is nonzero.
4320 : For ROUND_MOD_EXPR, can use ANDI and then sign-extend the result.
4321 : This could optimize to a bfexts instruction.
4322 : But C doesn't use these operations, so their optimizations are
4323 : left for later. */
4324 : /* ??? For modulo, we don't actually need the highpart of the first product,
4325 : the low part will do nicely. And for small divisors, the second multiply
4326 : can also be a low-part only multiply or even be completely left out.
4327 : E.g. to calculate the remainder of a division by 3 with a 32 bit
4328 : multiply, multiply with 0x55555556 and extract the upper two bits;
4329 : the result is exact for inputs up to 0x1fffffff.
4330 : The input range can be reduced by using cross-sum rules.
4331 : For odd divisors >= 3, the following table gives right shift counts
4332 : so that if a number is shifted by an integer multiple of the given
4333 : amount, the remainder stays the same:
4334 : 2, 4, 3, 6, 10, 12, 4, 8, 18, 6, 11, 20, 18, 0, 5, 10, 12, 0, 12, 20,
4335 : 14, 12, 23, 21, 8, 0, 20, 18, 0, 0, 6, 12, 0, 22, 0, 18, 20, 30, 0, 0,
4336 : 0, 8, 0, 11, 12, 10, 36, 0, 30, 0, 0, 12, 0, 0, 0, 0, 44, 12, 24, 0,
4337 : 20, 0, 7, 14, 0, 18, 36, 0, 0, 46, 60, 0, 42, 0, 15, 24, 20, 0, 0, 33,
4338 : 0, 20, 0, 0, 18, 0, 60, 0, 0, 0, 0, 0, 40, 18, 0, 0, 12
4339 :
4340 : Cross-sum rules for even numbers can be derived by leaving as many bits
4341 : to the right alone as the divisor has zeros to the right.
4342 : E.g. if x is an unsigned 32 bit number:
4343 : (x mod 12) == (((x & 1023) + ((x >> 8) & ~3)) * 0x15555558 >> 2 * 3) >> 28
4344 : */
4345 :
4346 : /* Helper for expand_divmod's unsigned constant division. For OP0 in
4347 : INT_MODE divided by a constant needing a (SIZE+1)-bit multiplier ML
4348 : with right shift POST_SHIFT (the mh != 0 case), try to obtain
4349 : the quotient from the high part of a single multiply in a mode twice
4350 : as wide as INT_MODE. Return the quotient in INT_MODE, having emitted
4351 : the insns, or NULL_RTX when the transformation is unavailable or not
4352 : cheaper than the classic sub/shift/add sequence. EXTRA_COST is the
4353 : cost of that sequence's follow-up ops, MAX_COST bounds the multiply
4354 : and SPEED selects the cost model.
4355 :
4356 : The magic constant occupies at most 2*SIZE bits and so must fit in a
4357 : HOST_WIDE_INT (always 64 bits today; checked below). A wider INT_MODE
4358 : such as DImode -- which would need a 128-bit magic and a single-word
4359 : high-part multiply in a 2x-wide mode that common targets lack -- is
4360 : therefore excluded. */
4361 :
4362 : static rtx
4363 2319 : expand_wide_mulh_udiv (scalar_int_mode int_mode, rtx op0,
4364 : unsigned HOST_WIDE_INT ml, int size, int post_shift,
4365 : int extra_cost, int max_cost, bool speed)
4366 : {
4367 2319 : scalar_int_mode wide_mode;
4368 :
4369 : /* We need POST_SHIFT >= 1, a wider integer mode that still fits in a
4370 : word, and the pre-shifted magic constant to fit in a HOST_WIDE_INT. */
4371 2319 : if (post_shift < 1
4372 2319 : || !GET_MODE_2XWIDER_MODE (int_mode).exists (&wide_mode)
4373 2412 : || GET_MODE_BITSIZE (wide_mode) > BITS_PER_WORD
4374 3349 : || GET_MODE_BITSIZE (wide_mode) > HOST_BITS_PER_WIDE_INT)
4375 : return NULL_RTX;
4376 :
4377 : /* The caller obtained ML and POST_SHIFT from choose_multiplier, which
4378 : guarantees POST_SHIFT <= ceil (log2 (d)) <= SIZE for a SIZE-bit
4379 : divisor d, so the shift count below is non-negative. */
4380 1030 : gcc_checking_assert (post_shift <= size);
4381 :
4382 : /* Pre-shift the (SIZE+1)-bit magic constant (2^SIZE + ML) by
4383 : (SIZE - POST_SHIFT) so that the quotient ends up in the high part
4384 : of the widened product. Since ML < 2^SIZE and POST_SHIFT >= 1, the
4385 : result is below 2^(2*SIZE) and thus fits in both WIDE_MODE and an
4386 : unsigned HOST_WIDE_INT (2*SIZE <= HOST_BITS_PER_WIDE_INT was
4387 : checked above). */
4388 1030 : unsigned HOST_WIDE_INT magic
4389 1030 : = ((HOST_WIDE_INT_1U << size) + ml) << (size - post_shift);
4390 :
4391 1030 : start_sequence ();
4392 1030 : rtx x_wide = convert_to_mode (wide_mode, op0, 1);
4393 1030 : rtx hi = expmed_mult_highpart (wide_mode, x_wide,
4394 1030 : gen_int_mode (magic, wide_mode),
4395 : NULL_RTX, 1, max_cost);
4396 1030 : rtx quotient = hi ? convert_to_mode (int_mode, hi, 1) : NULL_RTX;
4397 1030 : rtx_insn *insns = end_sequence ();
4398 :
4399 : /* Use the widened multiply only when it is no more expensive than
4400 : the classic sub/shift/add sequence. */
4401 1030 : unsigned classic_cost = mul_highpart_cost (speed, int_mode) + extra_cost;
4402 1030 : if (quotient == NULL_RTX || seq_cost (insns, speed) > classic_cost)
4403 : return NULL_RTX;
4404 :
4405 953 : emit_insn (insns);
4406 953 : return quotient;
4407 : }
4408 :
4409 : rtx
4410 240224 : expand_divmod (int rem_flag, enum tree_code code, machine_mode mode,
4411 : rtx op0, rtx op1, rtx target, int unsignedp,
4412 : enum optab_methods methods)
4413 : {
4414 240224 : machine_mode compute_mode;
4415 240224 : rtx tquotient;
4416 240224 : rtx quotient = 0, remainder = 0;
4417 240224 : rtx_insn *last;
4418 240224 : rtx_insn *insn;
4419 240224 : optab optab1, optab2;
4420 240224 : int op1_is_constant, op1_is_pow2 = 0;
4421 240224 : int max_cost, extra_cost;
4422 240224 : bool speed = optimize_insn_for_speed_p ();
4423 :
4424 240224 : op1_is_constant = CONST_INT_P (op1);
4425 240224 : if (op1_is_constant)
4426 : {
4427 137711 : wide_int ext_op1 = rtx_mode_t (op1, mode);
4428 167558 : op1_is_pow2 = (wi::popcount (ext_op1) == 1
4429 275422 : || (! unsignedp
4430 167558 : && wi::popcount (wi::neg (ext_op1)) == 1));
4431 137711 : }
4432 :
4433 : /*
4434 : This is the structure of expand_divmod:
4435 :
4436 : First comes code to fix up the operands so we can perform the operations
4437 : correctly and efficiently.
4438 :
4439 : Second comes a switch statement with code specific for each rounding mode.
4440 : For some special operands this code emits all RTL for the desired
4441 : operation, for other cases, it generates only a quotient and stores it in
4442 : QUOTIENT. The case for trunc division/remainder might leave quotient = 0,
4443 : to indicate that it has not done anything.
4444 :
4445 : Last comes code that finishes the operation. If QUOTIENT is set and
4446 : REM_FLAG is set, the remainder is computed as OP0 - QUOTIENT * OP1. If
4447 : QUOTIENT is not set, it is computed using trunc rounding.
4448 :
4449 : We try to generate special code for division and remainder when OP1 is a
4450 : constant. If |OP1| = 2**n we can use shifts and some other fast
4451 : operations. For other values of OP1, we compute a carefully selected
4452 : fixed-point approximation m = 1/OP1, and generate code that multiplies OP0
4453 : by m.
4454 :
4455 : In all cases but EXACT_DIV_EXPR, this multiplication requires the upper
4456 : half of the product. Different strategies for generating the product are
4457 : implemented in expmed_mult_highpart.
4458 :
4459 : If what we actually want is the remainder, we generate that by another
4460 : by-constant multiplication and a subtraction. */
4461 :
4462 : /* We shouldn't be called with OP1 == const1_rtx, but some of the
4463 : code below will malfunction if we are, so check here and handle
4464 : the special case if so. */
4465 240224 : if (op1 == const1_rtx)
4466 15 : return rem_flag ? const0_rtx : op0;
4467 :
4468 : /* When dividing by -1, we could get an overflow.
4469 : negv_optab can handle overflows. */
4470 240209 : if (! unsignedp && op1 == constm1_rtx)
4471 : {
4472 0 : if (rem_flag)
4473 0 : return const0_rtx;
4474 0 : return expand_unop (mode, flag_trapv && GET_MODE_CLASS (mode) == MODE_INT
4475 0 : ? negv_optab : neg_optab, op0, target, 0);
4476 : }
4477 :
4478 240209 : if (target
4479 : /* Don't use the function value register as a target
4480 : since we have to read it as well as write it,
4481 : and function-inlining gets confused by this. */
4482 240209 : && ((REG_P (target) && REG_FUNCTION_VALUE_P (target))
4483 : /* Don't clobber an operand while doing a multi-step calculation. */
4484 94813 : || ((rem_flag || op1_is_constant)
4485 74385 : && (reg_mentioned_p (target, op0)
4486 71963 : || (MEM_P (op0) && MEM_P (target))))
4487 91664 : || reg_mentioned_p (target, op1)
4488 91580 : || (MEM_P (op1) && MEM_P (target))))
4489 : target = 0;
4490 :
4491 : /* Get the mode in which to perform this computation. Normally it will
4492 : be MODE, but sometimes we can't do the desired operation in MODE.
4493 : If so, pick a wider mode in which we can do the operation. Convert
4494 : to that mode at the start to avoid repeated conversions.
4495 :
4496 : First see what operations we need. These depend on the expression
4497 : we are evaluating. (We assume that divxx3 insns exist under the
4498 : same conditions that modxx3 insns and that these insns don't normally
4499 : fail. If these assumptions are not correct, we may generate less
4500 : efficient code in some cases.)
4501 :
4502 : Then see if we find a mode in which we can open-code that operation
4503 : (either a division, modulus, or shift). Finally, check for the smallest
4504 : mode for which we can do the operation with a library call. */
4505 :
4506 : /* We might want to refine this now that we have division-by-constant
4507 : optimization. Since expmed_mult_highpart tries so many variants, it is
4508 : not straightforward to generalize this. Maybe we should make an array
4509 : of possible modes in init_expmed? Save this for GCC 2.7. */
4510 :
4511 134451 : optab1 = (op1_is_pow2
4512 240209 : ? (unsignedp ? lshr_optab : ashr_optab)
4513 159258 : : (unsignedp ? udiv_optab : sdiv_optab));
4514 321078 : optab2 = (op1_is_pow2 ? optab1
4515 159258 : : (unsignedp ? udivmod_optab : sdivmod_optab));
4516 :
4517 240209 : if (methods == OPTAB_WIDEN || methods == OPTAB_LIB_WIDEN)
4518 : {
4519 251938 : FOR_EACH_MODE_FROM (compute_mode, mode)
4520 247714 : if (optab_handler (optab1, compute_mode) != CODE_FOR_nothing
4521 247714 : || optab_handler (optab2, compute_mode) != CODE_FOR_nothing)
4522 : break;
4523 :
4524 238407 : if (compute_mode == VOIDmode && methods == OPTAB_LIB_WIDEN)
4525 4224 : FOR_EACH_MODE_FROM (compute_mode, mode)
4526 4224 : if (optab_libfunc (optab1, compute_mode)
4527 4224 : || optab_libfunc (optab2, compute_mode))
4528 : break;
4529 : }
4530 : else
4531 : compute_mode = mode;
4532 :
4533 : /* If we still couldn't find a mode, use MODE, but expand_binop will
4534 : probably die. */
4535 6026 : if (compute_mode == VOIDmode)
4536 0 : compute_mode = mode;
4537 :
4538 240209 : if (target && GET_MODE (target) == compute_mode)
4539 : tquotient = target;
4540 : else
4541 148796 : tquotient = gen_reg_rtx (compute_mode);
4542 :
4543 : #if 0
4544 : /* It should be possible to restrict the precision to GET_MODE_BITSIZE
4545 : (mode), and thereby get better code when OP1 is a constant. Do that
4546 : later. It will require going over all usages of SIZE below. */
4547 : size = GET_MODE_BITSIZE (mode);
4548 : #endif
4549 :
4550 : /* Only deduct something for a REM if the last divide done was
4551 : for a different constant. Then set the constant of the last
4552 : divide. */
4553 240209 : max_cost = (unsignedp
4554 240209 : ? udiv_cost (speed, compute_mode)
4555 134451 : : sdiv_cost (speed, compute_mode));
4556 240209 : if (rem_flag && ! (last_div_const != 0 && op1_is_constant
4557 6324 : && INTVAL (op1) == last_div_const))
4558 61923 : max_cost -= (mul_cost (speed, compute_mode)
4559 61923 : + add_cost (speed, compute_mode));
4560 :
4561 240209 : last_div_const = ! rem_flag && op1_is_constant ? INTVAL (op1) : 0;
4562 :
4563 : /* Now convert to the best mode to use. */
4564 240209 : if (compute_mode != mode)
4565 : {
4566 0 : op0 = convert_modes (compute_mode, mode, op0, unsignedp);
4567 0 : op1 = convert_modes (compute_mode, mode, op1, unsignedp);
4568 :
4569 : /* convert_modes may have placed op1 into a register, so we
4570 : must recompute the following. */
4571 0 : op1_is_constant = CONST_INT_P (op1);
4572 0 : if (op1_is_constant)
4573 : {
4574 0 : wide_int ext_op1 = rtx_mode_t (op1, compute_mode);
4575 0 : op1_is_pow2 = (wi::popcount (ext_op1) == 1
4576 0 : || (! unsignedp
4577 0 : && wi::popcount (wi::neg (ext_op1)) == 1));
4578 0 : }
4579 : else
4580 : op1_is_pow2 = 0;
4581 : }
4582 :
4583 : /* If one of the operands is a volatile MEM, copy it into a register. */
4584 :
4585 240209 : if (MEM_P (op0) && MEM_VOLATILE_P (op0))
4586 0 : op0 = force_reg (compute_mode, op0);
4587 240209 : if (MEM_P (op1) && MEM_VOLATILE_P (op1))
4588 0 : op1 = force_reg (compute_mode, op1);
4589 :
4590 : /* If we need the remainder or if OP1 is constant, we need to
4591 : put OP0 in a register in case it has any queued subexpressions. */
4592 240209 : if (rem_flag || op1_is_constant)
4593 176973 : op0 = force_reg (compute_mode, op0);
4594 :
4595 240209 : last = get_last_insn ();
4596 :
4597 : /* Promote floor rounding to trunc rounding for unsigned operations. */
4598 240209 : if (unsignedp)
4599 : {
4600 105758 : if (code == FLOOR_DIV_EXPR)
4601 : code = TRUNC_DIV_EXPR;
4602 105705 : if (code == FLOOR_MOD_EXPR)
4603 148 : code = TRUNC_MOD_EXPR;
4604 105758 : if (code == EXACT_DIV_EXPR && op1_is_pow2)
4605 80 : code = TRUNC_DIV_EXPR;
4606 : }
4607 :
4608 240209 : if (op1 != const0_rtx)
4609 239801 : switch (code)
4610 : {
4611 191150 : case TRUNC_MOD_EXPR:
4612 191150 : case TRUNC_DIV_EXPR:
4613 191150 : if (op1_is_constant)
4614 : {
4615 89751 : scalar_int_mode int_mode = as_a <scalar_int_mode> (compute_mode);
4616 89751 : int size = GET_MODE_BITSIZE (int_mode);
4617 89751 : if (unsignedp)
4618 : {
4619 52854 : unsigned HOST_WIDE_INT mh, ml;
4620 52854 : int pre_shift, post_shift;
4621 52854 : wide_int wd = rtx_mode_t (op1, int_mode);
4622 52854 : unsigned HOST_WIDE_INT d = wd.to_uhwi ();
4623 :
4624 52854 : if (wi::popcount (wd) == 1)
4625 : {
4626 27369 : pre_shift = floor_log2 (d);
4627 27369 : if (rem_flag)
4628 : {
4629 275 : unsigned HOST_WIDE_INT mask
4630 275 : = (HOST_WIDE_INT_1U << pre_shift) - 1;
4631 275 : remainder
4632 275 : = expand_binop (int_mode, and_optab, op0,
4633 275 : gen_int_mode (mask, int_mode),
4634 : remainder, 1, methods);
4635 275 : if (remainder)
4636 275 : return gen_lowpart (mode, remainder);
4637 : }
4638 27094 : quotient = expand_shift (RSHIFT_EXPR, int_mode, op0,
4639 27094 : pre_shift, tquotient, 1);
4640 : }
4641 25485 : else if (size <= HOST_BITS_PER_WIDE_INT)
4642 : {
4643 23881 : if (d >= (HOST_WIDE_INT_1U << (size - 1)))
4644 : {
4645 : /* Most significant bit of divisor is set; emit an scc
4646 : insn. */
4647 161 : quotient = emit_store_flag_force (tquotient, GEU, op0, op1,
4648 : int_mode, 1, 1);
4649 : }
4650 : else
4651 : {
4652 : /* Find a suitable multiplier and right shift count
4653 : instead of directly dividing by D. */
4654 23720 : mh = choose_multiplier (d, size, size,
4655 : &ml, &post_shift);
4656 :
4657 : /* If the suggested multiplier is more than SIZE bits,
4658 : we can do better for even divisors, using an
4659 : initial right shift. */
4660 23720 : if (mh != 0 && (d & 1) == 0)
4661 : {
4662 2341 : pre_shift = ctz_or_zero (d);
4663 2341 : mh = choose_multiplier (d >> pre_shift, size,
4664 : size - pre_shift,
4665 : &ml, &post_shift);
4666 2341 : gcc_assert (!mh);
4667 : }
4668 : else
4669 : pre_shift = 0;
4670 :
4671 2319 : if (mh != 0)
4672 : {
4673 2319 : rtx t1, t2, t3, t4;
4674 :
4675 2412 : if (post_shift - 1 >= BITS_PER_WORD)
4676 0 : goto fail1;
4677 :
4678 2319 : extra_cost
4679 2319 : = (shift_cost (speed, int_mode, post_shift - 1)
4680 2319 : + shift_cost (speed, int_mode, 1)
4681 2319 : + 2 * add_cost (speed, int_mode));
4682 :
4683 : /* Try a single widened multiply first; use it when
4684 : it is no more expensive. */
4685 2319 : quotient
4686 2319 : = expand_wide_mulh_udiv (int_mode, op0, ml, size,
4687 : post_shift, extra_cost,
4688 : max_cost, speed);
4689 2319 : if (quotient == NULL_RTX)
4690 : {
4691 1366 : t1 = expmed_mult_highpart
4692 1366 : (int_mode, op0, gen_int_mode (ml, int_mode),
4693 : NULL_RTX, 1, max_cost - extra_cost);
4694 1366 : if (t1 == 0)
4695 107 : goto fail1;
4696 1259 : t2 = force_operand (gen_rtx_MINUS (int_mode,
4697 : op0, t1),
4698 : NULL_RTX);
4699 1259 : t3 = expand_shift (RSHIFT_EXPR, int_mode,
4700 : t2, 1, NULL_RTX, 1);
4701 1259 : t4 = force_operand (gen_rtx_PLUS (int_mode,
4702 : t1, t3),
4703 : NULL_RTX);
4704 1259 : quotient = expand_shift
4705 1259 : (RSHIFT_EXPR, int_mode, t4,
4706 1259 : post_shift - 1, tquotient, 1);
4707 : }
4708 : }
4709 : else
4710 : {
4711 21401 : rtx t1, t2;
4712 :
4713 23246 : if (pre_shift >= BITS_PER_WORD
4714 21401 : || post_shift >= BITS_PER_WORD)
4715 3 : goto fail1;
4716 :
4717 21398 : t1 = expand_shift
4718 42796 : (RSHIFT_EXPR, int_mode, op0,
4719 21398 : pre_shift, NULL_RTX, 1);
4720 21398 : extra_cost
4721 21398 : = (shift_cost (speed, int_mode, pre_shift)
4722 21398 : + shift_cost (speed, int_mode, post_shift));
4723 21398 : t2 = expmed_mult_highpart
4724 21398 : (int_mode, t1,
4725 21398 : gen_int_mode (ml, int_mode),
4726 : NULL_RTX, 1, max_cost - extra_cost);
4727 21398 : if (t2 == 0)
4728 960 : goto fail1;
4729 20438 : quotient = expand_shift
4730 20438 : (RSHIFT_EXPR, int_mode, t2,
4731 20438 : post_shift, tquotient, 1);
4732 : }
4733 : }
4734 : }
4735 : else /* Too wide mode to use tricky code */
4736 : break;
4737 :
4738 49905 : insn = get_last_insn ();
4739 49905 : if (insn != last)
4740 49905 : set_dst_reg_note (insn, REG_EQUAL,
4741 : gen_rtx_UDIV (int_mode, op0, op1),
4742 : quotient);
4743 51250 : }
4744 : else /* TRUNC_DIV, signed */
4745 : {
4746 36897 : unsigned HOST_WIDE_INT ml;
4747 36897 : int post_shift;
4748 36897 : rtx mlr;
4749 36897 : HOST_WIDE_INT d = INTVAL (op1);
4750 36897 : unsigned HOST_WIDE_INT abs_d;
4751 :
4752 : /* Not prepared to handle division/remainder by
4753 : 0xffffffffffffffff8000000000000000 etc. */
4754 36897 : if (d == HOST_WIDE_INT_MIN && size > HOST_BITS_PER_WIDE_INT)
4755 : break;
4756 :
4757 : /* Since d might be INT_MIN, we have to cast to
4758 : unsigned HOST_WIDE_INT before negating to avoid
4759 : undefined signed overflow. */
4760 36897 : abs_d = (d >= 0
4761 36897 : ? (unsigned HOST_WIDE_INT) d
4762 : : - (unsigned HOST_WIDE_INT) d);
4763 :
4764 : /* n rem d = n rem -d */
4765 36897 : if (rem_flag && d < 0)
4766 : {
4767 141 : d = abs_d;
4768 141 : op1 = gen_int_mode (abs_d, int_mode);
4769 : }
4770 :
4771 36897 : if (d == 1)
4772 : quotient = op0;
4773 36897 : else if (d == -1)
4774 0 : quotient = expand_unop (int_mode, neg_optab, op0,
4775 : tquotient, 0);
4776 36897 : else if (size <= HOST_BITS_PER_WIDE_INT
4777 35498 : && abs_d == HOST_WIDE_INT_1U << (size - 1))
4778 : {
4779 : /* This case is not handled correctly below. */
4780 133 : quotient = emit_store_flag (tquotient, EQ, op0, op1,
4781 : int_mode, 1, 1);
4782 133 : if (quotient == 0)
4783 1289 : goto fail1;
4784 : }
4785 36764 : else if (EXACT_POWER_OF_2_OR_ZERO_P (d)
4786 12959 : && (size <= HOST_BITS_PER_WIDE_INT || d >= 0)
4787 2626 : && (rem_flag
4788 2626 : ? smod_pow2_cheap (speed, int_mode)
4789 10333 : : sdiv_pow2_cheap (speed, int_mode))
4790 : /* We assume that cheap metric is true if the
4791 : optab has an expander for this mode. */
4792 50919 : && ((optab_handler ((rem_flag ? smod_optab
4793 : : sdiv_optab),
4794 : int_mode)
4795 : != CODE_FOR_nothing)
4796 665 : || (optab_handler (sdivmod_optab, int_mode)
4797 : != CODE_FOR_nothing)))
4798 : ;
4799 36103 : else if (EXACT_POWER_OF_2_OR_ZERO_P (abs_d))
4800 : {
4801 12809 : if (rem_flag)
4802 : {
4803 2494 : remainder = expand_smod_pow2 (int_mode, op0, d);
4804 2494 : if (remainder)
4805 2494 : return gen_lowpart (mode, remainder);
4806 : }
4807 :
4808 10315 : if (sdiv_pow2_cheap (speed, int_mode)
4809 10315 : && ((optab_handler (sdiv_optab, int_mode)
4810 : != CODE_FOR_nothing)
4811 12 : || (optab_handler (sdivmod_optab, int_mode)
4812 : != CODE_FOR_nothing)))
4813 9 : quotient = expand_divmod (0, TRUNC_DIV_EXPR,
4814 : int_mode, op0,
4815 9 : gen_int_mode (abs_d,
4816 : int_mode),
4817 : NULL_RTX, 0);
4818 : else
4819 10306 : quotient = expand_sdiv_pow2 (int_mode, op0, abs_d);
4820 :
4821 : /* We have computed OP0 / abs(OP1). If OP1 is negative,
4822 : negate the quotient. */
4823 10315 : if (d < 0)
4824 : {
4825 511 : insn = get_last_insn ();
4826 511 : if (insn != last
4827 511 : && abs_d < (HOST_WIDE_INT_1U
4828 : << (HOST_BITS_PER_WIDE_INT - 1)))
4829 511 : set_dst_reg_note (insn, REG_EQUAL,
4830 511 : gen_rtx_DIV (int_mode, op0,
4831 : gen_int_mode
4832 : (abs_d,
4833 : int_mode)),
4834 : quotient);
4835 :
4836 511 : quotient = expand_unop (int_mode, neg_optab,
4837 : quotient, quotient, 0);
4838 : }
4839 : }
4840 23294 : else if (size <= HOST_BITS_PER_WIDE_INT)
4841 : {
4842 21956 : choose_multiplier (abs_d, size, size - 1,
4843 : &ml, &post_shift);
4844 21956 : if (ml < HOST_WIDE_INT_1U << (size - 1))
4845 : {
4846 16524 : rtx t1, t2, t3;
4847 :
4848 17637 : if (post_shift >= BITS_PER_WORD
4849 16524 : || size - 1 >= BITS_PER_WORD)
4850 248 : goto fail1;
4851 :
4852 16276 : extra_cost = (shift_cost (speed, int_mode, post_shift)
4853 16276 : + shift_cost (speed, int_mode, size - 1)
4854 16276 : + add_cost (speed, int_mode));
4855 16276 : t1 = expmed_mult_highpart
4856 16276 : (int_mode, op0, gen_int_mode (ml, int_mode),
4857 : NULL_RTX, 0, max_cost - extra_cost);
4858 16276 : if (t1 == 0)
4859 783 : goto fail1;
4860 15493 : t2 = expand_shift
4861 30986 : (RSHIFT_EXPR, int_mode, t1,
4862 15493 : post_shift, NULL_RTX, 0);
4863 15493 : t3 = expand_shift
4864 15493 : (RSHIFT_EXPR, int_mode, op0,
4865 15493 : size - 1, NULL_RTX, 0);
4866 15493 : if (d < 0)
4867 198 : quotient
4868 198 : = force_operand (gen_rtx_MINUS (int_mode, t3, t2),
4869 : tquotient);
4870 : else
4871 15295 : quotient
4872 15295 : = force_operand (gen_rtx_MINUS (int_mode, t2, t3),
4873 : tquotient);
4874 : }
4875 : else
4876 : {
4877 5432 : rtx t1, t2, t3, t4;
4878 :
4879 5792 : if (post_shift >= BITS_PER_WORD
4880 5427 : || size - 1 >= BITS_PER_WORD)
4881 43 : goto fail1;
4882 :
4883 5389 : ml |= HOST_WIDE_INT_M1U << (size - 1);
4884 5389 : mlr = gen_int_mode (ml, int_mode);
4885 5389 : extra_cost = (shift_cost (speed, int_mode, post_shift)
4886 5389 : + shift_cost (speed, int_mode, size - 1)
4887 5389 : + 2 * add_cost (speed, int_mode));
4888 5389 : t1 = expmed_mult_highpart (int_mode, op0, mlr,
4889 : NULL_RTX, 0,
4890 : max_cost - extra_cost);
4891 5389 : if (t1 == 0)
4892 215 : goto fail1;
4893 5174 : t2 = force_operand (gen_rtx_PLUS (int_mode, t1, op0),
4894 : NULL_RTX);
4895 5174 : t3 = expand_shift
4896 10348 : (RSHIFT_EXPR, int_mode, t2,
4897 5174 : post_shift, NULL_RTX, 0);
4898 5174 : t4 = expand_shift
4899 5174 : (RSHIFT_EXPR, int_mode, op0,
4900 5174 : size - 1, NULL_RTX, 0);
4901 5174 : if (d < 0)
4902 52 : quotient
4903 52 : = force_operand (gen_rtx_MINUS (int_mode, t4, t3),
4904 : tquotient);
4905 : else
4906 5122 : quotient
4907 5122 : = force_operand (gen_rtx_MINUS (int_mode, t3, t4),
4908 : tquotient);
4909 : }
4910 : }
4911 : else /* Too wide mode to use tricky code */
4912 : break;
4913 :
4914 31776 : insn = get_last_insn ();
4915 31776 : if (insn != last)
4916 31115 : set_dst_reg_note (insn, REG_EQUAL,
4917 : gen_rtx_DIV (int_mode, op0, op1),
4918 : quotient);
4919 : }
4920 : break;
4921 : }
4922 101399 : fail1:
4923 103758 : delete_insns_since (last);
4924 103758 : break;
4925 :
4926 1765 : case FLOOR_DIV_EXPR:
4927 1765 : case FLOOR_MOD_EXPR:
4928 : /* We will come here only for signed operations. */
4929 1765 : if (op1_is_constant && HWI_COMPUTABLE_MODE_P (compute_mode))
4930 : {
4931 971 : scalar_int_mode int_mode = as_a <scalar_int_mode> (compute_mode);
4932 971 : int size = GET_MODE_BITSIZE (int_mode);
4933 971 : unsigned HOST_WIDE_INT mh, ml;
4934 971 : int pre_shift, post_shift;
4935 971 : HOST_WIDE_INT d = INTVAL (op1);
4936 :
4937 971 : if (d > 0)
4938 : {
4939 : /* We could just as easily deal with negative constants here,
4940 : but it does not seem worth the trouble for GCC 2.6. */
4941 945 : if (EXACT_POWER_OF_2_OR_ZERO_P (d))
4942 : {
4943 641 : pre_shift = floor_log2 (d);
4944 641 : if (rem_flag)
4945 : {
4946 64 : unsigned HOST_WIDE_INT mask
4947 64 : = (HOST_WIDE_INT_1U << pre_shift) - 1;
4948 64 : remainder = expand_binop
4949 64 : (int_mode, and_optab, op0,
4950 64 : gen_int_mode (mask, int_mode),
4951 : remainder, 0, methods);
4952 64 : if (remainder)
4953 64 : return gen_lowpart (mode, remainder);
4954 : }
4955 577 : quotient = expand_shift
4956 577 : (RSHIFT_EXPR, int_mode, op0,
4957 577 : pre_shift, tquotient, 0);
4958 : }
4959 : else
4960 : {
4961 304 : rtx t1, t2, t3, t4;
4962 :
4963 304 : mh = choose_multiplier (d, size, size - 1,
4964 : &ml, &post_shift);
4965 304 : gcc_assert (!mh);
4966 :
4967 328 : if (post_shift < BITS_PER_WORD
4968 304 : && size - 1 < BITS_PER_WORD)
4969 : {
4970 302 : t1 = expand_shift
4971 302 : (RSHIFT_EXPR, int_mode, op0,
4972 302 : size - 1, NULL_RTX, 0);
4973 302 : t2 = expand_binop (int_mode, xor_optab, op0, t1,
4974 : NULL_RTX, 0, OPTAB_WIDEN);
4975 302 : extra_cost = (shift_cost (speed, int_mode, post_shift)
4976 302 : + shift_cost (speed, int_mode, size - 1)
4977 302 : + 2 * add_cost (speed, int_mode));
4978 302 : t3 = expmed_mult_highpart
4979 302 : (int_mode, t2, gen_int_mode (ml, int_mode),
4980 : NULL_RTX, 1, max_cost - extra_cost);
4981 302 : if (t3 != 0)
4982 : {
4983 271 : t4 = expand_shift
4984 542 : (RSHIFT_EXPR, int_mode, t3,
4985 271 : post_shift, NULL_RTX, 1);
4986 271 : quotient = expand_binop (int_mode, xor_optab,
4987 : t4, t1, tquotient, 0,
4988 : OPTAB_WIDEN);
4989 : }
4990 : }
4991 : }
4992 : }
4993 : else
4994 : {
4995 26 : rtx nsign, t1, t2, t3, t4;
4996 26 : t1 = force_operand (gen_rtx_PLUS (int_mode,
4997 : op0, constm1_rtx), NULL_RTX);
4998 26 : t2 = expand_binop (int_mode, ior_optab, op0, t1, NULL_RTX,
4999 : 0, OPTAB_WIDEN);
5000 52 : nsign = expand_shift (RSHIFT_EXPR, int_mode, t2,
5001 26 : size - 1, NULL_RTX, 0);
5002 26 : t3 = force_operand (gen_rtx_MINUS (int_mode, t1, nsign),
5003 : NULL_RTX);
5004 26 : t4 = expand_divmod (0, TRUNC_DIV_EXPR, int_mode, t3, op1,
5005 : NULL_RTX, 0);
5006 26 : if (t4)
5007 : {
5008 26 : rtx t5;
5009 26 : t5 = expand_unop (int_mode, one_cmpl_optab, nsign,
5010 : NULL_RTX, 0);
5011 26 : quotient = force_operand (gen_rtx_PLUS (int_mode, t4, t5),
5012 : tquotient);
5013 : }
5014 : }
5015 : }
5016 :
5017 907 : if (quotient != 0)
5018 : break;
5019 827 : delete_insns_since (last);
5020 :
5021 : /* Try using an instruction that produces both the quotient and
5022 : remainder, using truncation. We can easily compensate the quotient
5023 : or remainder to get floor rounding, once we have the remainder.
5024 : Notice that we compute also the final remainder value here,
5025 : and return the result right away. */
5026 827 : if (target == 0 || GET_MODE (target) != compute_mode)
5027 125 : target = gen_reg_rtx (compute_mode);
5028 :
5029 827 : if (rem_flag)
5030 : {
5031 329 : remainder
5032 329 : = REG_P (target) ? target : gen_reg_rtx (compute_mode);
5033 329 : quotient = gen_reg_rtx (compute_mode);
5034 : }
5035 : else
5036 : {
5037 498 : quotient
5038 498 : = REG_P (target) ? target : gen_reg_rtx (compute_mode);
5039 498 : remainder = gen_reg_rtx (compute_mode);
5040 : }
5041 :
5042 827 : if (expand_twoval_binop (sdivmod_optab, op0, op1,
5043 : quotient, remainder, 0))
5044 : {
5045 : /* This could be computed with a branch-less sequence.
5046 : Save that for later. */
5047 792 : rtx tem;
5048 792 : rtx_code_label *label = gen_label_rtx ();
5049 792 : do_cmp_and_jump (remainder, const0_rtx, EQ, compute_mode, label);
5050 792 : tem = expand_binop (compute_mode, xor_optab, op0, op1,
5051 : NULL_RTX, 0, OPTAB_WIDEN);
5052 792 : do_cmp_and_jump (tem, const0_rtx, GE, compute_mode, label);
5053 792 : expand_dec (quotient, const1_rtx);
5054 792 : expand_inc (remainder, op1);
5055 792 : emit_label (label);
5056 1281 : return gen_lowpart (mode, rem_flag ? remainder : quotient);
5057 : }
5058 :
5059 : /* No luck with division elimination or divmod. Have to do it
5060 : by conditionally adjusting op0 *and* the result. */
5061 35 : {
5062 35 : rtx_code_label *label1, *label2, *label3, *label4, *label5;
5063 35 : rtx adjusted_op0;
5064 35 : rtx tem;
5065 :
5066 35 : quotient = gen_reg_rtx (compute_mode);
5067 35 : adjusted_op0 = copy_to_mode_reg (compute_mode, op0);
5068 35 : label1 = gen_label_rtx ();
5069 35 : label2 = gen_label_rtx ();
5070 35 : label3 = gen_label_rtx ();
5071 35 : label4 = gen_label_rtx ();
5072 35 : label5 = gen_label_rtx ();
5073 35 : do_cmp_and_jump (op1, const0_rtx, LT, compute_mode, label2);
5074 35 : do_cmp_and_jump (adjusted_op0, const0_rtx, LT, compute_mode, label1);
5075 35 : tem = expand_binop (compute_mode, sdiv_optab, adjusted_op0, op1,
5076 : quotient, 0, methods);
5077 35 : if (tem != quotient)
5078 35 : emit_move_insn (quotient, tem);
5079 35 : emit_jump_insn (targetm.gen_jump (label5));
5080 35 : emit_barrier ();
5081 35 : emit_label (label1);
5082 35 : expand_inc (adjusted_op0, const1_rtx);
5083 35 : emit_jump_insn (targetm.gen_jump (label4));
5084 35 : emit_barrier ();
5085 35 : emit_label (label2);
5086 35 : do_cmp_and_jump (adjusted_op0, const0_rtx, GT, compute_mode, label3);
5087 35 : tem = expand_binop (compute_mode, sdiv_optab, adjusted_op0, op1,
5088 : quotient, 0, methods);
5089 35 : if (tem != quotient)
5090 35 : emit_move_insn (quotient, tem);
5091 35 : emit_jump_insn (targetm.gen_jump (label5));
5092 35 : emit_barrier ();
5093 35 : emit_label (label3);
5094 35 : expand_dec (adjusted_op0, const1_rtx);
5095 35 : emit_label (label4);
5096 35 : tem = expand_binop (compute_mode, sdiv_optab, adjusted_op0, op1,
5097 : quotient, 0, methods);
5098 35 : if (tem != quotient)
5099 35 : emit_move_insn (quotient, tem);
5100 35 : expand_dec (quotient, const1_rtx);
5101 35 : emit_label (label5);
5102 : }
5103 35 : break;
5104 :
5105 384 : case CEIL_DIV_EXPR:
5106 384 : case CEIL_MOD_EXPR:
5107 384 : if (unsignedp)
5108 : {
5109 0 : if (op1_is_constant
5110 0 : && EXACT_POWER_OF_2_OR_ZERO_P (INTVAL (op1))
5111 0 : && (HWI_COMPUTABLE_MODE_P (compute_mode)
5112 0 : || INTVAL (op1) >= 0))
5113 : {
5114 0 : scalar_int_mode int_mode
5115 0 : = as_a <scalar_int_mode> (compute_mode);
5116 0 : rtx t1, t2, t3;
5117 0 : unsigned HOST_WIDE_INT d = INTVAL (op1);
5118 0 : t1 = expand_shift (RSHIFT_EXPR, int_mode, op0,
5119 0 : floor_log2 (d), tquotient, 1);
5120 0 : t2 = expand_binop (int_mode, and_optab, op0,
5121 0 : gen_int_mode (d - 1, int_mode),
5122 : NULL_RTX, 1, methods);
5123 0 : t3 = gen_reg_rtx (int_mode);
5124 0 : t3 = emit_store_flag (t3, NE, t2, const0_rtx, int_mode, 1, 1);
5125 0 : if (t3 == 0)
5126 : {
5127 0 : rtx_code_label *lab;
5128 0 : lab = gen_label_rtx ();
5129 0 : do_cmp_and_jump (t2, const0_rtx, EQ, int_mode, lab);
5130 0 : expand_inc (t1, const1_rtx);
5131 0 : emit_label (lab);
5132 0 : quotient = t1;
5133 : }
5134 : else
5135 0 : quotient = force_operand (gen_rtx_PLUS (int_mode, t1, t3),
5136 : tquotient);
5137 : break;
5138 : }
5139 :
5140 : /* Try using an instruction that produces both the quotient and
5141 : remainder, using truncation. We can easily compensate the
5142 : quotient or remainder to get ceiling rounding, once we have the
5143 : remainder. Notice that we compute also the final remainder
5144 : value here, and return the result right away. */
5145 0 : if (target == 0 || GET_MODE (target) != compute_mode)
5146 0 : target = gen_reg_rtx (compute_mode);
5147 :
5148 0 : if (rem_flag)
5149 : {
5150 0 : remainder = (REG_P (target)
5151 0 : ? target : gen_reg_rtx (compute_mode));
5152 0 : quotient = gen_reg_rtx (compute_mode);
5153 : }
5154 : else
5155 : {
5156 0 : quotient = (REG_P (target)
5157 0 : ? target : gen_reg_rtx (compute_mode));
5158 0 : remainder = gen_reg_rtx (compute_mode);
5159 : }
5160 :
5161 0 : if (expand_twoval_binop (udivmod_optab, op0, op1, quotient,
5162 : remainder, 1))
5163 : {
5164 : /* This could be computed with a branch-less sequence.
5165 : Save that for later. */
5166 0 : rtx_code_label *label = gen_label_rtx ();
5167 0 : do_cmp_and_jump (remainder, const0_rtx, EQ,
5168 : compute_mode, label);
5169 0 : expand_inc (quotient, const1_rtx);
5170 0 : expand_dec (remainder, op1);
5171 0 : emit_label (label);
5172 0 : return gen_lowpart (mode, rem_flag ? remainder : quotient);
5173 : }
5174 :
5175 : /* No luck with division elimination or divmod. Have to do it
5176 : by conditionally adjusting op0 *and* the result. */
5177 0 : {
5178 0 : rtx_code_label *label1, *label2;
5179 0 : rtx adjusted_op0, tem;
5180 :
5181 0 : quotient = gen_reg_rtx (compute_mode);
5182 0 : adjusted_op0 = copy_to_mode_reg (compute_mode, op0);
5183 0 : label1 = gen_label_rtx ();
5184 0 : label2 = gen_label_rtx ();
5185 0 : do_cmp_and_jump (adjusted_op0, const0_rtx, NE,
5186 : compute_mode, label1);
5187 0 : emit_move_insn (quotient, const0_rtx);
5188 0 : emit_jump_insn (targetm.gen_jump (label2));
5189 0 : emit_barrier ();
5190 0 : emit_label (label1);
5191 0 : expand_dec (adjusted_op0, const1_rtx);
5192 0 : tem = expand_binop (compute_mode, udiv_optab, adjusted_op0, op1,
5193 : quotient, 1, methods);
5194 0 : if (tem != quotient)
5195 0 : emit_move_insn (quotient, tem);
5196 0 : expand_inc (quotient, const1_rtx);
5197 0 : emit_label (label2);
5198 : }
5199 : }
5200 : else /* signed */
5201 : {
5202 384 : if (op1_is_constant && EXACT_POWER_OF_2_OR_ZERO_P (INTVAL (op1))
5203 27 : && INTVAL (op1) >= 0)
5204 : {
5205 : /* This is extremely similar to the code for the unsigned case
5206 : above. For 2.7 we should merge these variants, but for
5207 : 2.6.1 I don't want to touch the code for unsigned since that
5208 : get used in C. The signed case will only be used by other
5209 : languages (Ada). */
5210 :
5211 27 : rtx t1, t2, t3;
5212 27 : unsigned HOST_WIDE_INT d = INTVAL (op1);
5213 54 : t1 = expand_shift (RSHIFT_EXPR, compute_mode, op0,
5214 27 : floor_log2 (d), tquotient, 0);
5215 27 : t2 = expand_binop (compute_mode, and_optab, op0,
5216 27 : gen_int_mode (d - 1, compute_mode),
5217 : NULL_RTX, 1, methods);
5218 27 : t3 = gen_reg_rtx (compute_mode);
5219 27 : t3 = emit_store_flag (t3, NE, t2, const0_rtx,
5220 : compute_mode, 1, 1);
5221 27 : if (t3 == 0)
5222 : {
5223 0 : rtx_code_label *lab;
5224 0 : lab = gen_label_rtx ();
5225 0 : do_cmp_and_jump (t2, const0_rtx, EQ, compute_mode, lab);
5226 0 : expand_inc (t1, const1_rtx);
5227 0 : emit_label (lab);
5228 0 : quotient = t1;
5229 : }
5230 : else
5231 27 : quotient = force_operand (gen_rtx_PLUS (compute_mode,
5232 : t1, t3),
5233 : tquotient);
5234 : break;
5235 : }
5236 :
5237 : /* Try using an instruction that produces both the quotient and
5238 : remainder, using truncation. We can easily compensate the
5239 : quotient or remainder to get ceiling rounding, once we have the
5240 : remainder. Notice that we compute also the final remainder
5241 : value here, and return the result right away. */
5242 357 : if (target == 0 || GET_MODE (target) != compute_mode)
5243 15 : target = gen_reg_rtx (compute_mode);
5244 357 : if (rem_flag)
5245 : {
5246 149 : remainder= (REG_P (target)
5247 149 : ? target : gen_reg_rtx (compute_mode));
5248 149 : quotient = gen_reg_rtx (compute_mode);
5249 : }
5250 : else
5251 : {
5252 208 : quotient = (REG_P (target)
5253 208 : ? target : gen_reg_rtx (compute_mode));
5254 208 : remainder = gen_reg_rtx (compute_mode);
5255 : }
5256 :
5257 357 : if (expand_twoval_binop (sdivmod_optab, op0, op1, quotient,
5258 : remainder, 0))
5259 : {
5260 : /* This could be computed with a branch-less sequence.
5261 : Save that for later. */
5262 357 : rtx tem;
5263 357 : rtx_code_label *label = gen_label_rtx ();
5264 357 : do_cmp_and_jump (remainder, const0_rtx, EQ,
5265 : compute_mode, label);
5266 357 : tem = expand_binop (compute_mode, xor_optab, op0, op1,
5267 : NULL_RTX, 0, OPTAB_WIDEN);
5268 357 : do_cmp_and_jump (tem, const0_rtx, LT, compute_mode, label);
5269 357 : expand_inc (quotient, const1_rtx);
5270 357 : expand_dec (remainder, op1);
5271 357 : emit_label (label);
5272 565 : return gen_lowpart (mode, rem_flag ? remainder : quotient);
5273 : }
5274 :
5275 : /* No luck with division elimination or divmod. Have to do it
5276 : by conditionally adjusting op0 *and* the result. */
5277 0 : {
5278 0 : rtx_code_label *label1, *label2, *label3, *label4, *label5;
5279 0 : rtx adjusted_op0;
5280 0 : rtx tem;
5281 :
5282 0 : quotient = gen_reg_rtx (compute_mode);
5283 0 : adjusted_op0 = copy_to_mode_reg (compute_mode, op0);
5284 0 : label1 = gen_label_rtx ();
5285 0 : label2 = gen_label_rtx ();
5286 0 : label3 = gen_label_rtx ();
5287 0 : label4 = gen_label_rtx ();
5288 0 : label5 = gen_label_rtx ();
5289 0 : do_cmp_and_jump (op1, const0_rtx, LT, compute_mode, label2);
5290 0 : do_cmp_and_jump (adjusted_op0, const0_rtx, GT,
5291 : compute_mode, label1);
5292 0 : tem = expand_binop (compute_mode, sdiv_optab, adjusted_op0, op1,
5293 : quotient, 0, methods);
5294 0 : if (tem != quotient)
5295 0 : emit_move_insn (quotient, tem);
5296 0 : emit_jump_insn (targetm.gen_jump (label5));
5297 0 : emit_barrier ();
5298 0 : emit_label (label1);
5299 0 : expand_dec (adjusted_op0, const1_rtx);
5300 0 : emit_jump_insn (targetm.gen_jump (label4));
5301 0 : emit_barrier ();
5302 0 : emit_label (label2);
5303 0 : do_cmp_and_jump (adjusted_op0, const0_rtx, LT,
5304 : compute_mode, label3);
5305 0 : tem = expand_binop (compute_mode, sdiv_optab, adjusted_op0, op1,
5306 : quotient, 0, methods);
5307 0 : if (tem != quotient)
5308 0 : emit_move_insn (quotient, tem);
5309 0 : emit_jump_insn (targetm.gen_jump (label5));
5310 0 : emit_barrier ();
5311 0 : emit_label (label3);
5312 0 : expand_inc (adjusted_op0, const1_rtx);
5313 0 : emit_label (label4);
5314 0 : tem = expand_binop (compute_mode, sdiv_optab, adjusted_op0, op1,
5315 : quotient, 0, methods);
5316 0 : if (tem != quotient)
5317 0 : emit_move_insn (quotient, tem);
5318 0 : expand_inc (quotient, const1_rtx);
5319 0 : emit_label (label5);
5320 : }
5321 : }
5322 : break;
5323 :
5324 46500 : case EXACT_DIV_EXPR:
5325 46500 : if (op1_is_constant && HWI_COMPUTABLE_MODE_P (compute_mode))
5326 : {
5327 46492 : scalar_int_mode int_mode = as_a <scalar_int_mode> (compute_mode);
5328 46492 : int size = GET_MODE_BITSIZE (int_mode);
5329 46492 : HOST_WIDE_INT d = INTVAL (op1);
5330 46492 : unsigned HOST_WIDE_INT ml;
5331 46492 : int pre_shift;
5332 46492 : rtx t1;
5333 :
5334 46492 : pre_shift = ctz_or_zero (d);
5335 46492 : ml = invert_mod2n (d >> pre_shift, size);
5336 46492 : t1 = expand_shift (RSHIFT_EXPR, int_mode, op0,
5337 46492 : pre_shift, NULL_RTX, unsignedp);
5338 46492 : quotient = expand_mult (int_mode, t1, gen_int_mode (ml, int_mode),
5339 : NULL_RTX, 1);
5340 :
5341 46492 : insn = get_last_insn ();
5342 92984 : set_dst_reg_note (insn, REG_EQUAL,
5343 : gen_rtx_fmt_ee (unsignedp ? UDIV : DIV,
5344 : int_mode, op0, op1),
5345 : quotient);
5346 : }
5347 : break;
5348 :
5349 2 : case ROUND_DIV_EXPR:
5350 2 : case ROUND_MOD_EXPR:
5351 2 : if (unsignedp)
5352 : {
5353 0 : scalar_int_mode int_mode = as_a <scalar_int_mode> (compute_mode);
5354 0 : rtx tem;
5355 0 : rtx_code_label *label;
5356 0 : label = gen_label_rtx ();
5357 0 : quotient = gen_reg_rtx (int_mode);
5358 0 : remainder = gen_reg_rtx (int_mode);
5359 0 : if (expand_twoval_binop (udivmod_optab, op0, op1, quotient, remainder, 1) == 0)
5360 : {
5361 0 : rtx tem;
5362 0 : quotient = expand_binop (int_mode, udiv_optab, op0, op1,
5363 : quotient, 1, methods);
5364 0 : tem = expand_mult (int_mode, quotient, op1, NULL_RTX, 1);
5365 0 : remainder = expand_binop (int_mode, sub_optab, op0, tem,
5366 : remainder, 1, methods);
5367 : }
5368 0 : tem = plus_constant (int_mode, op1, -1);
5369 0 : tem = expand_shift (RSHIFT_EXPR, int_mode, tem, 1, NULL_RTX, 1);
5370 0 : do_cmp_and_jump (remainder, tem, LEU, int_mode, label);
5371 0 : expand_inc (quotient, const1_rtx);
5372 0 : expand_dec (remainder, op1);
5373 0 : emit_label (label);
5374 : }
5375 : else
5376 : {
5377 2 : scalar_int_mode int_mode = as_a <scalar_int_mode> (compute_mode);
5378 2 : int size = GET_MODE_BITSIZE (int_mode);
5379 2 : rtx abs_rem, abs_op1, tem, mask;
5380 2 : rtx_code_label *label;
5381 2 : label = gen_label_rtx ();
5382 2 : quotient = gen_reg_rtx (int_mode);
5383 2 : remainder = gen_reg_rtx (int_mode);
5384 2 : if (expand_twoval_binop (sdivmod_optab, op0, op1, quotient, remainder, 0) == 0)
5385 : {
5386 0 : rtx tem;
5387 0 : quotient = expand_binop (int_mode, sdiv_optab, op0, op1,
5388 : quotient, 0, methods);
5389 0 : tem = expand_mult (int_mode, quotient, op1, NULL_RTX, 0);
5390 0 : remainder = expand_binop (int_mode, sub_optab, op0, tem,
5391 : remainder, 0, methods);
5392 : }
5393 2 : abs_rem = expand_abs (int_mode, remainder, NULL_RTX, 1, 0);
5394 2 : abs_op1 = expand_abs (int_mode, op1, NULL_RTX, 1, 0);
5395 2 : tem = expand_shift (LSHIFT_EXPR, int_mode, abs_rem,
5396 : 1, NULL_RTX, 1);
5397 2 : do_cmp_and_jump (tem, abs_op1, LTU, int_mode, label);
5398 2 : tem = expand_binop (int_mode, xor_optab, op0, op1,
5399 : NULL_RTX, 0, OPTAB_WIDEN);
5400 4 : mask = expand_shift (RSHIFT_EXPR, int_mode, tem,
5401 2 : size - 1, NULL_RTX, 0);
5402 2 : tem = expand_binop (int_mode, xor_optab, mask, const1_rtx,
5403 : NULL_RTX, 0, OPTAB_WIDEN);
5404 2 : tem = expand_binop (int_mode, sub_optab, tem, mask,
5405 : NULL_RTX, 0, OPTAB_WIDEN);
5406 2 : expand_inc (quotient, tem);
5407 2 : tem = expand_binop (int_mode, xor_optab, mask, op1,
5408 : NULL_RTX, 0, OPTAB_WIDEN);
5409 2 : tem = expand_binop (int_mode, sub_optab, tem, mask,
5410 : NULL_RTX, 0, OPTAB_WIDEN);
5411 2 : expand_dec (remainder, tem);
5412 2 : emit_label (label);
5413 : }
5414 3 : return gen_lowpart (mode, rem_flag ? remainder : quotient);
5415 :
5416 0 : default:
5417 0 : gcc_unreachable ();
5418 : }
5419 :
5420 234935 : if (quotient == 0)
5421 : {
5422 107777 : if (target && GET_MODE (target) != compute_mode)
5423 64510 : target = 0;
5424 :
5425 107777 : if (rem_flag)
5426 : {
5427 : /* Try to produce the remainder without producing the quotient.
5428 : If we seem to have a divmod pattern that does not require widening,
5429 : don't try widening here. We should really have a WIDEN argument
5430 : to expand_twoval_binop, since what we'd really like to do here is
5431 : 1) try a mod insn in compute_mode
5432 : 2) try a divmod insn in compute_mode
5433 : 3) try a div insn in compute_mode and multiply-subtract to get
5434 : remainder
5435 : 4) try the same things with widening allowed. */
5436 41396 : remainder
5437 42890 : = sign_expand_binop (compute_mode, umod_optab, smod_optab,
5438 : op0, op1, target,
5439 : unsignedp,
5440 41396 : ((optab_handler (optab2, compute_mode)
5441 : != CODE_FOR_nothing)
5442 : ? OPTAB_DIRECT : OPTAB_WIDEN));
5443 41396 : if (remainder == 0)
5444 : {
5445 : /* No luck there. Can we do remainder and divide at once
5446 : without a library call? */
5447 41189 : remainder = gen_reg_rtx (compute_mode);
5448 64979 : if (! expand_twoval_binop ((unsignedp
5449 : ? udivmod_optab
5450 : : sdivmod_optab),
5451 : op0, op1,
5452 : NULL_RTX, remainder, unsignedp))
5453 : remainder = 0;
5454 : }
5455 :
5456 39902 : if (remainder)
5457 40109 : return gen_lowpart (mode, remainder);
5458 : }
5459 :
5460 : /* Produce the quotient. Try a quotient insn, but not a library call.
5461 : If we have a divmod in this mode, use it in preference to widening
5462 : the div (for this test we assume it will not fail). Note that optab2
5463 : is set to the one of the two optabs that the call below will use. */
5464 67668 : quotient
5465 72939 : = sign_expand_binop (compute_mode, udiv_optab, sdiv_optab,
5466 : op0, op1, rem_flag ? NULL_RTX : target,
5467 : unsignedp,
5468 67668 : ((optab_handler (optab2, compute_mode)
5469 : != CODE_FOR_nothing)
5470 : ? OPTAB_DIRECT : OPTAB_WIDEN));
5471 :
5472 67668 : if (quotient == 0)
5473 : {
5474 : /* No luck there. Try a quotient-and-remainder insn,
5475 : keeping the quotient alone. */
5476 67238 : quotient = gen_reg_rtx (compute_mode);
5477 97874 : if (! expand_twoval_binop (unsignedp ? udivmod_optab : sdivmod_optab,
5478 : op0, op1,
5479 : quotient, NULL_RTX, unsignedp))
5480 : {
5481 3555 : quotient = 0;
5482 3555 : if (! rem_flag)
5483 : /* Still no luck. If we are not computing the remainder,
5484 : use a library call for the quotient. */
5485 2306 : quotient = sign_expand_binop (compute_mode,
5486 : udiv_optab, sdiv_optab,
5487 : op0, op1, target,
5488 : unsignedp, methods);
5489 : }
5490 : }
5491 : }
5492 :
5493 194867 : if (rem_flag)
5494 : {
5495 21689 : if (target && GET_MODE (target) != compute_mode)
5496 14670 : target = 0;
5497 :
5498 21689 : if (quotient == 0)
5499 : {
5500 : /* No divide instruction either. Use library for remainder. */
5501 1249 : remainder = sign_expand_binop (compute_mode, umod_optab, smod_optab,
5502 : op0, op1, target,
5503 : unsignedp, methods);
5504 : /* No remainder function. Try a quotient-and-remainder
5505 : function, keeping the remainder. */
5506 1249 : if (!remainder
5507 0 : && (methods == OPTAB_LIB || methods == OPTAB_LIB_WIDEN))
5508 : {
5509 0 : remainder = gen_reg_rtx (compute_mode);
5510 0 : if (!expand_twoval_binop_libfunc
5511 0 : (unsignedp ? udivmod_optab : sdivmod_optab,
5512 : op0, op1,
5513 : NULL_RTX, remainder,
5514 : unsignedp ? UMOD : MOD))
5515 0 : remainder = NULL_RTX;
5516 : }
5517 : }
5518 : else
5519 : {
5520 : /* We divided. Now finish doing X - Y * (X / Y). */
5521 20440 : remainder = expand_mult (compute_mode, quotient, op1,
5522 : NULL_RTX, unsignedp);
5523 20440 : remainder = expand_binop (compute_mode, sub_optab, op0,
5524 : remainder, target, unsignedp,
5525 : methods);
5526 : }
5527 : }
5528 :
5529 196116 : if (methods != OPTAB_LIB_WIDEN
5530 1802 : && (rem_flag ? remainder : quotient) == NULL_RTX)
5531 : return NULL_RTX;
5532 :
5533 370543 : return gen_lowpart (mode, rem_flag ? remainder : quotient);
5534 : }
5535 :
5536 : /* Return a tree node with data type TYPE, describing the value of X.
5537 : Usually this is an VAR_DECL, if there is no obvious better choice.
5538 : X may be an expression, however we only support those expressions
5539 : generated by loop.c. */
5540 :
5541 : tree
5542 692162 : make_tree (tree type, rtx x)
5543 : {
5544 692162 : tree t;
5545 :
5546 692162 : switch (GET_CODE (x))
5547 : {
5548 22873 : case CONST_INT:
5549 22873 : case CONST_WIDE_INT:
5550 22873 : t = wide_int_to_tree (type, rtx_mode_t (x, TYPE_MODE (type)));
5551 22873 : return t;
5552 :
5553 0 : case CONST_POLY_INT:
5554 0 : return wide_int_to_tree (type, const_poly_int_value (x));
5555 :
5556 0 : case CONST_DOUBLE:
5557 0 : STATIC_ASSERT (HOST_BITS_PER_WIDE_INT * 2 <= MAX_BITSIZE_MODE_ANY_INT);
5558 0 : if (TARGET_SUPPORTS_WIDE_INT == 0 && GET_MODE (x) == VOIDmode)
5559 : t = wide_int_to_tree (type,
5560 : wide_int::from_array (&CONST_DOUBLE_LOW (x), 2,
5561 : HOST_BITS_PER_WIDE_INT * 2));
5562 : else
5563 0 : t = build_real (type, *CONST_DOUBLE_REAL_VALUE (x));
5564 :
5565 0 : return t;
5566 :
5567 0 : case CONST_VECTOR:
5568 0 : {
5569 0 : unsigned int npatterns = CONST_VECTOR_NPATTERNS (x);
5570 0 : unsigned int nelts_per_pattern = CONST_VECTOR_NELTS_PER_PATTERN (x);
5571 0 : tree itype = TREE_TYPE (type);
5572 :
5573 : /* Build a tree with vector elements. */
5574 0 : tree_vector_builder elts (type, npatterns, nelts_per_pattern);
5575 0 : unsigned int count = elts.encoded_nelts ();
5576 0 : for (unsigned int i = 0; i < count; ++i)
5577 : {
5578 0 : rtx elt = CONST_VECTOR_ELT (x, i);
5579 0 : elts.quick_push (make_tree (itype, elt));
5580 : }
5581 :
5582 0 : return elts.build ();
5583 0 : }
5584 :
5585 0 : case PLUS:
5586 0 : return fold_build2 (PLUS_EXPR, type, make_tree (type, XEXP (x, 0)),
5587 : make_tree (type, XEXP (x, 1)));
5588 :
5589 0 : case MINUS:
5590 0 : return fold_build2 (MINUS_EXPR, type, make_tree (type, XEXP (x, 0)),
5591 : make_tree (type, XEXP (x, 1)));
5592 :
5593 0 : case NEG:
5594 0 : return fold_build1 (NEGATE_EXPR, type, make_tree (type, XEXP (x, 0)));
5595 :
5596 0 : case MULT:
5597 0 : return fold_build2 (MULT_EXPR, type, make_tree (type, XEXP (x, 0)),
5598 : make_tree (type, XEXP (x, 1)));
5599 :
5600 0 : case ASHIFT:
5601 0 : return fold_build2 (LSHIFT_EXPR, type, make_tree (type, XEXP (x, 0)),
5602 : make_tree (type, XEXP (x, 1)));
5603 :
5604 0 : case LSHIFTRT:
5605 0 : t = unsigned_type_for (type);
5606 0 : return fold_convert (type, build2 (RSHIFT_EXPR, t,
5607 : make_tree (t, XEXP (x, 0)),
5608 : make_tree (type, XEXP (x, 1))));
5609 :
5610 0 : case ASHIFTRT:
5611 0 : t = signed_type_for (type);
5612 0 : return fold_convert (type, build2 (RSHIFT_EXPR, t,
5613 : make_tree (t, XEXP (x, 0)),
5614 : make_tree (type, XEXP (x, 1))));
5615 :
5616 0 : case DIV:
5617 0 : if (TREE_CODE (type) != REAL_TYPE)
5618 0 : t = signed_type_for (type);
5619 : else
5620 : t = type;
5621 :
5622 0 : return fold_convert (type, build2 (TRUNC_DIV_EXPR, t,
5623 : make_tree (t, XEXP (x, 0)),
5624 : make_tree (t, XEXP (x, 1))));
5625 0 : case UDIV:
5626 0 : t = unsigned_type_for (type);
5627 0 : return fold_convert (type, build2 (TRUNC_DIV_EXPR, t,
5628 : make_tree (t, XEXP (x, 0)),
5629 : make_tree (t, XEXP (x, 1))));
5630 :
5631 0 : case SIGN_EXTEND:
5632 0 : case ZERO_EXTEND:
5633 0 : t = lang_hooks.types.type_for_mode (GET_MODE (XEXP (x, 0)),
5634 : GET_CODE (x) == ZERO_EXTEND);
5635 0 : return fold_convert (type, make_tree (t, XEXP (x, 0)));
5636 :
5637 0 : case CONST:
5638 0 : return make_tree (type, XEXP (x, 0));
5639 :
5640 0 : case SYMBOL_REF:
5641 0 : t = SYMBOL_REF_DECL (x);
5642 0 : if (t)
5643 0 : return fold_convert (type, build_fold_addr_expr (t));
5644 : /* fall through. */
5645 :
5646 669289 : default:
5647 669289 : t = build_decl (RTL_LOCATION (x), VAR_DECL, NULL_TREE, type);
5648 :
5649 : /* If TYPE is a POINTER_TYPE, we might need to convert X from
5650 : address mode to pointer mode. */
5651 669289 : if (POINTER_TYPE_P (type))
5652 829936 : x = convert_memory_address_addr_space
5653 414968 : (SCALAR_INT_TYPE_MODE (type), x, TYPE_ADDR_SPACE (TREE_TYPE (type)));
5654 :
5655 : /* Note that we do *not* use SET_DECL_RTL here, because we do not
5656 : want set_decl_rtl to go adjusting REG_ATTRS for this temporary. */
5657 669289 : t->decl_with_rtl.rtl = x;
5658 :
5659 669289 : return t;
5660 : }
5661 : }
5662 :
5663 : /* Compute the logical-and of OP0 and OP1, storing it in TARGET
5664 : and returning TARGET.
5665 :
5666 : If TARGET is 0, a pseudo-register or constant is returned. */
5667 :
5668 : rtx
5669 73109 : expand_and (machine_mode mode, rtx op0, rtx op1, rtx target)
5670 : {
5671 73109 : rtx tem = 0;
5672 :
5673 73109 : if (GET_MODE (op0) == VOIDmode && GET_MODE (op1) == VOIDmode)
5674 80 : tem = simplify_binary_operation (AND, mode, op0, op1);
5675 80 : if (tem == 0)
5676 73029 : tem = expand_binop (mode, and_optab, op0, op1, target, 0, OPTAB_LIB_WIDEN);
5677 :
5678 73109 : if (target == 0)
5679 : target = tem;
5680 44892 : else if (tem != target)
5681 13 : emit_move_insn (target, tem);
5682 73109 : return target;
5683 : }
5684 :
5685 : /* Helper function for emit_store_flag. */
5686 : rtx
5687 759644 : emit_cstore (rtx target, enum insn_code icode, enum rtx_code code,
5688 : machine_mode mode, machine_mode compare_mode,
5689 : int unsignedp, rtx x, rtx y, int normalizep,
5690 : machine_mode target_mode)
5691 : {
5692 759644 : class expand_operand ops[4];
5693 759644 : rtx op0, comparison, subtarget;
5694 759644 : rtx_insn *last;
5695 759644 : scalar_int_mode result_mode = targetm.cstore_mode (icode);
5696 759644 : scalar_int_mode int_target_mode;
5697 :
5698 759644 : last = get_last_insn ();
5699 759644 : x = prepare_operand (icode, x, 2, mode, compare_mode, unsignedp);
5700 759644 : y = prepare_operand (icode, y, 3, mode, compare_mode, unsignedp);
5701 759644 : if (!x || !y)
5702 : {
5703 284 : delete_insns_since (last);
5704 284 : return NULL_RTX;
5705 : }
5706 :
5707 759360 : if (target_mode == VOIDmode)
5708 : int_target_mode = result_mode;
5709 : else
5710 759344 : int_target_mode = as_a <scalar_int_mode> (target_mode);
5711 759360 : if (!target)
5712 69670 : target = gen_reg_rtx (int_target_mode);
5713 :
5714 759360 : comparison = gen_rtx_fmt_ee (code, result_mode, x, y);
5715 :
5716 759360 : create_output_operand (&ops[0], optimize ? NULL_RTX : target, result_mode);
5717 759360 : create_fixed_operand (&ops[1], comparison);
5718 759360 : create_fixed_operand (&ops[2], x);
5719 759360 : create_fixed_operand (&ops[3], y);
5720 759360 : if (!maybe_expand_insn (icode, 4, ops))
5721 : {
5722 155234 : delete_insns_since (last);
5723 155234 : return NULL_RTX;
5724 : }
5725 604126 : subtarget = ops[0].value;
5726 :
5727 : /* If we are converting to a wider mode, first convert to
5728 : INT_TARGET_MODE, then normalize. This produces better combining
5729 : opportunities on machines that have a SIGN_EXTRACT when we are
5730 : testing a single bit. This mostly benefits the 68k.
5731 :
5732 : If STORE_FLAG_VALUE does not have the sign bit set when
5733 : interpreted in MODE, we can do this conversion as unsigned, which
5734 : is usually more efficient. */
5735 604126 : if (GET_MODE_PRECISION (int_target_mode) > GET_MODE_PRECISION (result_mode))
5736 : {
5737 114913 : gcc_assert (GET_MODE_PRECISION (result_mode) != 1
5738 : || STORE_FLAG_VALUE == 1 || STORE_FLAG_VALUE == -1);
5739 :
5740 114913 : bool unsignedp = (STORE_FLAG_VALUE >= 0);
5741 114913 : convert_move (target, subtarget, unsignedp);
5742 :
5743 114913 : op0 = target;
5744 114913 : result_mode = int_target_mode;
5745 : }
5746 : else
5747 : op0 = subtarget;
5748 :
5749 : /* If we want to keep subexpressions around, don't reuse our last
5750 : target. */
5751 604126 : if (optimize)
5752 486336 : subtarget = 0;
5753 :
5754 : /* Now normalize to the proper value in MODE. Sometimes we don't
5755 : have to do anything. */
5756 604126 : if (normalizep == 0 || normalizep == STORE_FLAG_VALUE)
5757 : ;
5758 : /* STORE_FLAG_VALUE might be the most negative number, so write
5759 : the comparison this way to avoid a compiler-time warning. */
5760 352 : else if (- normalizep == STORE_FLAG_VALUE)
5761 352 : op0 = expand_unop (result_mode, neg_optab, op0, subtarget, 0);
5762 :
5763 : /* We don't want to use STORE_FLAG_VALUE < 0 below since this makes
5764 : it hard to use a value of just the sign bit due to ANSI integer
5765 : constant typing rules. */
5766 0 : else if (val_signbit_known_set_p (result_mode, STORE_FLAG_VALUE))
5767 0 : op0 = expand_shift (RSHIFT_EXPR, result_mode, op0,
5768 0 : GET_MODE_BITSIZE (result_mode) - 1, subtarget,
5769 : normalizep == 1);
5770 : else
5771 : {
5772 0 : gcc_assert (STORE_FLAG_VALUE & 1);
5773 :
5774 0 : op0 = expand_and (result_mode, op0, const1_rtx, subtarget);
5775 0 : if (normalizep == -1)
5776 : op0 = expand_unop (result_mode, neg_optab, op0, op0, 0);
5777 : }
5778 :
5779 : /* If we were converting to a smaller mode, do the conversion now. */
5780 604126 : if (int_target_mode != result_mode)
5781 : {
5782 0 : convert_move (target, op0, 0);
5783 0 : return target;
5784 : }
5785 : else
5786 : return op0;
5787 : }
5788 :
5789 :
5790 : /* A subroutine of emit_store_flag only including "tricks" that do not
5791 : need a recursive call. These are kept separate to avoid infinite
5792 : loops. */
5793 :
5794 : static rtx
5795 726813 : emit_store_flag_1 (rtx target, enum rtx_code code, rtx op0, rtx op1,
5796 : machine_mode mode, int unsignedp, int normalizep,
5797 : machine_mode target_mode)
5798 : {
5799 726813 : rtx subtarget;
5800 726813 : enum insn_code icode;
5801 726813 : machine_mode compare_mode;
5802 726813 : enum mode_class mclass;
5803 :
5804 726813 : if (unsignedp)
5805 166043 : code = unsigned_condition (code);
5806 :
5807 : /* If one operand is constant, make it the second one. Only do this
5808 : if the other operand is not constant as well. */
5809 :
5810 726813 : if (swap_commutative_operands_p (op0, op1))
5811 : {
5812 5128 : std::swap (op0, op1);
5813 5128 : code = swap_condition (code);
5814 : }
5815 :
5816 726813 : if (mode == VOIDmode)
5817 42033 : mode = GET_MODE (op0);
5818 :
5819 726813 : if (CONST_SCALAR_INT_P (op1))
5820 331223 : canonicalize_comparison (mode, &code, &op1);
5821 :
5822 : /* For some comparisons with 1 and -1, we can convert this to
5823 : comparisons with zero. This will often produce more opportunities for
5824 : store-flag insns. */
5825 :
5826 726813 : switch (code)
5827 : {
5828 44790 : case LT:
5829 44790 : if (op1 == const1_rtx)
5830 59 : op1 = const0_rtx, code = LE;
5831 : break;
5832 26637 : case LE:
5833 26637 : if (op1 == constm1_rtx)
5834 0 : op1 = const0_rtx, code = LT;
5835 : break;
5836 39709 : case GE:
5837 39709 : if (op1 == const1_rtx)
5838 0 : op1 = const0_rtx, code = GT;
5839 : break;
5840 33469 : case GT:
5841 33469 : if (op1 == constm1_rtx)
5842 135 : op1 = const0_rtx, code = GE;
5843 : break;
5844 6433 : case GEU:
5845 6433 : if (op1 == const1_rtx)
5846 0 : op1 = const0_rtx, code = NE;
5847 : break;
5848 10840 : case LTU:
5849 10840 : if (op1 == const1_rtx)
5850 10 : op1 = const0_rtx, code = EQ;
5851 : break;
5852 : default:
5853 : break;
5854 : }
5855 :
5856 : /* If this is A < 0 or A >= 0, we can do this by taking the ones
5857 : complement of A (for GE) and shifting the sign bit to the low bit. */
5858 726813 : scalar_int_mode int_mode;
5859 199052 : if (op1 == const0_rtx && (code == LT || code == GE)
5860 726813 : && is_int_mode (mode, &int_mode)
5861 726813 : && (normalizep || STORE_FLAG_VALUE == 1
5862 : || val_signbit_p (int_mode, STORE_FLAG_VALUE)))
5863 : {
5864 40568 : scalar_int_mode int_target_mode;
5865 40568 : subtarget = target;
5866 :
5867 40568 : if (!target)
5868 : int_target_mode = int_mode;
5869 : else
5870 : {
5871 : /* If the result is to be wider than OP0, it is best to convert it
5872 : first. If it is to be narrower, it is *incorrect* to convert it
5873 : first. */
5874 40568 : int_target_mode = as_a <scalar_int_mode> (target_mode);
5875 121704 : if (GET_MODE_SIZE (int_target_mode) > GET_MODE_SIZE (int_mode))
5876 : {
5877 453 : op0 = convert_modes (int_target_mode, int_mode, op0, 0);
5878 453 : int_mode = int_target_mode;
5879 : }
5880 : }
5881 :
5882 40568 : if (int_target_mode != int_mode)
5883 26953 : subtarget = 0;
5884 :
5885 40568 : if (code == GE)
5886 20764 : op0 = expand_unop (int_mode, one_cmpl_optab, op0,
5887 : ((STORE_FLAG_VALUE == 1 || normalizep)
5888 : ? 0 : subtarget), 0);
5889 :
5890 40568 : if (STORE_FLAG_VALUE == 1 || normalizep)
5891 : /* If we are supposed to produce a 0/1 value, we want to do
5892 : a logical shift from the sign bit to the low-order bit; for
5893 : a -1/0 value, we do an arithmetic shift. */
5894 81136 : op0 = expand_shift (RSHIFT_EXPR, int_mode, op0,
5895 40568 : GET_MODE_BITSIZE (int_mode) - 1,
5896 : subtarget, normalizep != -1);
5897 :
5898 40568 : if (int_mode != int_target_mode)
5899 26953 : op0 = convert_modes (int_target_mode, int_mode, op0, 0);
5900 :
5901 : return op0;
5902 : }
5903 :
5904 : /* Next try expanding this via the backend's cstore<mode>4. */
5905 686245 : mclass = GET_MODE_CLASS (mode);
5906 698246 : FOR_EACH_WIDER_MODE_FROM (compare_mode, mode)
5907 : {
5908 690953 : machine_mode optab_mode = mclass == MODE_CC ? CCmode : compare_mode;
5909 690953 : icode = optab_handler (cstore_optab, optab_mode);
5910 690953 : if (icode != CODE_FOR_nothing)
5911 : {
5912 678952 : do_pending_stack_adjust ();
5913 678952 : rtx tem = emit_cstore (target, icode, code, mode, compare_mode,
5914 : unsignedp, op0, op1, normalizep, target_mode);
5915 678952 : if (tem)
5916 : return tem;
5917 :
5918 90288 : if (GET_MODE_CLASS (mode) == MODE_FLOAT)
5919 : {
5920 80672 : enum rtx_code scode = swap_condition (code);
5921 :
5922 80672 : tem = emit_cstore (target, icode, scode, mode, compare_mode,
5923 : unsignedp, op1, op0, normalizep, target_mode);
5924 80672 : if (tem)
5925 : return tem;
5926 : }
5927 : break;
5928 : }
5929 : }
5930 :
5931 : /* If we are comparing a double-word integer with zero or -1, we can
5932 : convert the comparison into one involving a single word. */
5933 82139 : if (is_int_mode (mode, &int_mode)
5934 16910 : && GET_MODE_BITSIZE (int_mode) == BITS_PER_WORD * 2
5935 9332 : && (!MEM_P (op0) || ! MEM_VOLATILE_P (op0)))
5936 : {
5937 9332 : rtx tem;
5938 9332 : if ((code == EQ || code == NE)
5939 0 : && (op1 == const0_rtx || op1 == constm1_rtx))
5940 : {
5941 0 : rtx op00, op01;
5942 :
5943 : /* Do a logical OR or AND of the two words and compare the
5944 : result. */
5945 0 : op00 = force_subreg (word_mode, op0, int_mode, 0);
5946 0 : op01 = force_subreg (word_mode, op0, int_mode, UNITS_PER_WORD);
5947 0 : tem = expand_binop (word_mode,
5948 0 : op1 == const0_rtx ? ior_optab : and_optab,
5949 : op00, op01, NULL_RTX, unsignedp,
5950 : OPTAB_DIRECT);
5951 :
5952 0 : if (tem != 0)
5953 0 : tem = emit_store_flag (NULL_RTX, code, tem, op1, word_mode,
5954 : unsignedp, normalizep);
5955 : }
5956 9332 : else if ((code == LT || code == GE) && op1 == const0_rtx)
5957 : {
5958 0 : rtx op0h;
5959 :
5960 : /* If testing the sign bit, can just test on high word. */
5961 0 : op0h = force_highpart_subreg (word_mode, op0, int_mode);
5962 0 : tem = emit_store_flag (NULL_RTX, code, op0h, op1, word_mode,
5963 : unsignedp, normalizep);
5964 0 : }
5965 : else
5966 : tem = NULL_RTX;
5967 :
5968 0 : if (tem)
5969 : {
5970 0 : if (target_mode == VOIDmode || GET_MODE (tem) == target_mode)
5971 : return tem;
5972 0 : if (!target)
5973 0 : target = gen_reg_rtx (target_mode);
5974 :
5975 0 : convert_move (target, tem,
5976 0 : !val_signbit_known_set_p (word_mode,
5977 : (normalizep ? normalizep
5978 : : STORE_FLAG_VALUE)));
5979 0 : return target;
5980 : }
5981 : }
5982 :
5983 : return 0;
5984 : }
5985 :
5986 : /* Subroutine of emit_store_flag that handles cases in which the operands
5987 : are scalar integers. SUBTARGET is the target to use for temporary
5988 : operations and TRUEVAL is the value to store when the condition is
5989 : true. All other arguments are as for emit_store_flag. */
5990 :
5991 : rtx
5992 4808 : emit_store_flag_int (rtx target, rtx subtarget, enum rtx_code code, rtx op0,
5993 : rtx op1, scalar_int_mode mode, int unsignedp,
5994 : int normalizep, rtx trueval)
5995 : {
5996 4808 : machine_mode target_mode = target ? GET_MODE (target) : VOIDmode;
5997 4808 : rtx_insn *last = get_last_insn ();
5998 :
5999 : /* If this is an equality comparison of integers, we can try to exclusive-or
6000 : (or subtract) the two operands and use a recursive call to try the
6001 : comparison with zero. Don't do any of these cases if branches are
6002 : very cheap. */
6003 :
6004 4808 : if ((code == EQ || code == NE) && op1 != const0_rtx)
6005 : {
6006 0 : rtx tem = expand_binop (mode, xor_optab, op0, op1, subtarget, 1,
6007 : OPTAB_WIDEN);
6008 :
6009 0 : if (tem == 0)
6010 0 : tem = expand_binop (mode, sub_optab, op0, op1, subtarget, 1,
6011 : OPTAB_WIDEN);
6012 0 : if (tem != 0)
6013 0 : tem = emit_store_flag (target, code, tem, const0_rtx,
6014 : mode, unsignedp, normalizep);
6015 0 : if (tem != 0)
6016 : return tem;
6017 :
6018 0 : delete_insns_since (last);
6019 : }
6020 :
6021 : /* For integer comparisons, try the reverse comparison. However, for
6022 : small X and if we'd have anyway to extend, implementing "X != 0"
6023 : as "-(int)X >> 31" is still cheaper than inverting "(int)X == 0". */
6024 4808 : rtx_code rcode = reverse_condition (code);
6025 4808 : if (can_compare_p (rcode, mode, ccp_store_flag)
6026 4808 : && ! (optab_handler (cstore_optab, mode) == CODE_FOR_nothing
6027 0 : && code == NE
6028 0 : && GET_MODE_SIZE (mode) < UNITS_PER_WORD
6029 0 : && op1 == const0_rtx))
6030 : {
6031 4808 : int want_add = ((STORE_FLAG_VALUE == 1 && normalizep == -1)
6032 : || (STORE_FLAG_VALUE == -1 && normalizep == 1));
6033 :
6034 : /* Again, for the reverse comparison, use either an addition or a XOR. */
6035 4808 : if (want_add
6036 4808 : && rtx_cost (GEN_INT (normalizep), mode, PLUS, 1,
6037 : optimize_insn_for_speed_p ()) == 0)
6038 : {
6039 0 : rtx tem = emit_store_flag_1 (subtarget, rcode, op0, op1, mode, 0,
6040 : STORE_FLAG_VALUE, target_mode);
6041 0 : if (tem != 0)
6042 0 : tem = expand_binop (target_mode, add_optab, tem,
6043 0 : gen_int_mode (normalizep, target_mode),
6044 : target, 0, OPTAB_WIDEN);
6045 0 : if (tem != 0)
6046 : return tem;
6047 : }
6048 4808 : else if (!want_add
6049 4808 : && rtx_cost (trueval, mode, XOR, 1,
6050 : optimize_insn_for_speed_p ()) == 0)
6051 : {
6052 4808 : rtx tem = emit_store_flag_1 (subtarget, rcode, op0, op1, mode, 0,
6053 : normalizep, target_mode);
6054 4808 : if (tem != 0)
6055 0 : tem = expand_binop (target_mode, xor_optab, tem, trueval, target,
6056 0 : INTVAL (trueval) >= 0, OPTAB_WIDEN);
6057 0 : if (tem != 0)
6058 : return tem;
6059 : }
6060 :
6061 4808 : delete_insns_since (last);
6062 : }
6063 :
6064 : /* Some other cases we can do are EQ, NE, LE, and GT comparisons with
6065 : the constant zero. Reject all other comparisons at this point. Only
6066 : do LE and GT if branches are expensive since they are expensive on
6067 : 2-operand machines. */
6068 :
6069 4808 : if (op1 != const0_rtx
6070 4808 : || (code != EQ && code != NE
6071 89 : && (BRANCH_COST (optimize_insn_for_speed_p (),
6072 89 : false) <= 1 || (code != LE && code != GT))))
6073 : return 0;
6074 :
6075 : /* Try to put the result of the comparison in the sign bit. Assume we can't
6076 : do the necessary operation below. */
6077 :
6078 231 : rtx tem = 0;
6079 :
6080 : /* To see if A <= 0, compute (A | (A - 1)). A <= 0 iff that result has
6081 : the sign bit set. */
6082 :
6083 231 : if (code == LE)
6084 : {
6085 : /* This is destructive, so SUBTARGET can't be OP0. */
6086 56 : if (rtx_equal_p (subtarget, op0))
6087 0 : subtarget = 0;
6088 :
6089 56 : tem = expand_binop (mode, sub_optab, op0, const1_rtx, subtarget, 0,
6090 : OPTAB_WIDEN);
6091 56 : if (tem)
6092 56 : tem = expand_binop (mode, ior_optab, op0, tem, subtarget, 0,
6093 : OPTAB_WIDEN);
6094 : }
6095 :
6096 : /* To see if A > 0, compute (((signed) A) << BITS) - A, where BITS is the
6097 : number of bits in the mode of OP0, minus one. */
6098 :
6099 231 : if (code == GT)
6100 : {
6101 33 : if (rtx_equal_p (subtarget, op0))
6102 0 : subtarget = 0;
6103 :
6104 33 : tem = maybe_expand_shift (RSHIFT_EXPR, mode, op0,
6105 33 : GET_MODE_BITSIZE (mode) - 1,
6106 : subtarget, 0);
6107 33 : if (tem)
6108 33 : tem = expand_binop (mode, sub_optab, tem, op0, subtarget, 0,
6109 : OPTAB_WIDEN);
6110 : }
6111 :
6112 231 : if (code == EQ || code == NE)
6113 : {
6114 : /* For EQ or NE, one way to do the comparison is to apply an operation
6115 : that converts the operand into a positive number if it is nonzero
6116 : or zero if it was originally zero. Then, for EQ, we subtract 1 and
6117 : for NE we negate. This puts the result in the sign bit. Then we
6118 : normalize with a shift, if needed.
6119 :
6120 : Two operations that can do the above actions are ABS and FFS, so try
6121 : them. If that doesn't work, and MODE is smaller than a full word,
6122 : we can use zero-extension to the wider mode (an unsigned conversion)
6123 : as the operation. */
6124 :
6125 : /* Note that ABS doesn't yield a positive number for INT_MIN, but
6126 : that is compensated by the subsequent overflow when subtracting
6127 : one / negating. */
6128 :
6129 142 : if (optab_handler (abs_optab, mode) != CODE_FOR_nothing)
6130 142 : tem = expand_unop (mode, abs_optab, op0, subtarget, 1);
6131 0 : else if (optab_handler (ffs_optab, mode) != CODE_FOR_nothing)
6132 0 : tem = expand_unop (mode, ffs_optab, op0, subtarget, 1);
6133 0 : else if (GET_MODE_SIZE (mode) < UNITS_PER_WORD)
6134 : {
6135 0 : tem = convert_modes (word_mode, mode, op0, 1);
6136 0 : mode = word_mode;
6137 : }
6138 :
6139 142 : if (tem != 0)
6140 : {
6141 142 : if (code == EQ)
6142 0 : tem = expand_binop (mode, sub_optab, tem, const1_rtx, subtarget,
6143 : 0, OPTAB_WIDEN);
6144 : else
6145 142 : tem = expand_unop (mode, neg_optab, tem, subtarget, 0);
6146 : }
6147 :
6148 : /* If we couldn't do it that way, for NE we can "or" the two's complement
6149 : of the value with itself. For EQ, we take the one's complement of
6150 : that "or", which is an extra insn, so we only handle EQ if branches
6151 : are expensive. */
6152 :
6153 142 : if (tem == 0
6154 142 : && (code == NE
6155 0 : || BRANCH_COST (optimize_insn_for_speed_p (),
6156 : false) > 1))
6157 : {
6158 0 : if (rtx_equal_p (subtarget, op0))
6159 0 : subtarget = 0;
6160 :
6161 0 : tem = expand_unop (mode, neg_optab, op0, subtarget, 0);
6162 0 : tem = expand_binop (mode, ior_optab, tem, op0, subtarget, 0,
6163 : OPTAB_WIDEN);
6164 :
6165 0 : if (tem && code == EQ)
6166 0 : tem = expand_unop (mode, one_cmpl_optab, tem, subtarget, 0);
6167 : }
6168 : }
6169 :
6170 231 : if (tem && normalizep)
6171 231 : tem = maybe_expand_shift (RSHIFT_EXPR, mode, tem,
6172 231 : GET_MODE_BITSIZE (mode) - 1,
6173 : subtarget, normalizep == 1);
6174 :
6175 231 : if (tem)
6176 : {
6177 231 : if (!target)
6178 : ;
6179 231 : else if (GET_MODE (tem) != target_mode)
6180 : {
6181 89 : convert_move (target, tem, 0);
6182 89 : tem = target;
6183 : }
6184 142 : else if (!subtarget)
6185 : {
6186 73 : emit_move_insn (target, tem);
6187 73 : tem = target;
6188 : }
6189 : }
6190 : else
6191 0 : delete_insns_since (last);
6192 :
6193 : return tem;
6194 : }
6195 :
6196 : /* Emit a store-flags instruction for comparison CODE on OP0 and OP1
6197 : and storing in TARGET. Normally return TARGET.
6198 : Return 0 if that cannot be done.
6199 :
6200 : MODE is the mode to use for OP0 and OP1 should they be CONST_INTs. If
6201 : it is VOIDmode, they cannot both be CONST_INT.
6202 :
6203 : UNSIGNEDP is for the case where we have to widen the operands
6204 : to perform the operation. It says to use zero-extension.
6205 :
6206 : NORMALIZEP is 1 if we should convert the result to be either zero
6207 : or one. Normalize is -1 if we should convert the result to be
6208 : either zero or -1. If NORMALIZEP is zero, the result will be left
6209 : "raw" out of the scc insn. */
6210 :
6211 : rtx
6212 656564 : emit_store_flag (rtx target, enum rtx_code code, rtx op0, rtx op1,
6213 : machine_mode mode, int unsignedp, int normalizep)
6214 : {
6215 656564 : machine_mode target_mode = target ? GET_MODE (target) : VOIDmode;
6216 656564 : enum rtx_code rcode;
6217 656564 : rtx subtarget;
6218 656564 : rtx tem, trueval;
6219 656564 : rtx_insn *last;
6220 :
6221 : /* If we compare constants, we shouldn't use a store-flag operation,
6222 : but a constant load. We can get there via the vanilla route that
6223 : usually generates a compare-branch sequence, but will in this case
6224 : fold the comparison to a constant, and thus elide the branch. */
6225 656564 : if (CONSTANT_P (op0) && CONSTANT_P (op1))
6226 : return NULL_RTX;
6227 :
6228 656247 : tem = emit_store_flag_1 (target, code, op0, op1, mode, unsignedp, normalizep,
6229 : target_mode);
6230 656247 : if (tem)
6231 : return tem;
6232 :
6233 : /* If we reached here, we can't do this with a scc insn, however there
6234 : are some comparisons that can be done in other ways. Don't do any
6235 : of these cases if branches are very cheap. */
6236 76803 : if (BRANCH_COST (optimize_insn_for_speed_p (), false) == 0)
6237 : return 0;
6238 :
6239 : /* See what we need to return. We can only return a 1, -1, or the
6240 : sign bit. */
6241 :
6242 76803 : if (normalizep == 0)
6243 : {
6244 0 : if (STORE_FLAG_VALUE == 1 || STORE_FLAG_VALUE == -1)
6245 0 : normalizep = STORE_FLAG_VALUE;
6246 :
6247 : else if (val_signbit_p (mode, STORE_FLAG_VALUE))
6248 : ;
6249 : else
6250 : return 0;
6251 : }
6252 :
6253 76803 : last = get_last_insn ();
6254 :
6255 : /* If optimizing, use different pseudo registers for each insn, instead
6256 : of reusing the same pseudo. This leads to better CSE, but slows
6257 : down the compiler, since there are more pseudos. */
6258 76734 : subtarget = (!optimize
6259 76803 : && (target_mode == mode)) ? target : NULL_RTX;
6260 76803 : trueval = GEN_INT (normalizep ? normalizep : STORE_FLAG_VALUE);
6261 :
6262 : /* For floating-point comparisons, try the reverse comparison or try
6263 : changing the "orderedness" of the comparison. */
6264 76803 : if (GET_MODE_CLASS (mode) == MODE_FLOAT)
6265 : {
6266 68473 : enum rtx_code first_code;
6267 68473 : bool and_them;
6268 :
6269 68473 : rcode = reverse_condition_maybe_unordered (code);
6270 68473 : if (can_compare_p (rcode, mode, ccp_store_flag)
6271 68473 : && (code == ORDERED || code == UNORDERED
6272 0 : || (! HONOR_NANS (mode) && (code == LTGT || code == UNEQ))
6273 0 : || (! HONOR_SNANS (mode) && (code == EQ || code == NE))))
6274 : {
6275 0 : int want_add = ((STORE_FLAG_VALUE == 1 && normalizep == -1)
6276 : || (STORE_FLAG_VALUE == -1 && normalizep == 1));
6277 :
6278 : /* For the reverse comparison, use either an addition or a XOR. */
6279 0 : if (want_add
6280 0 : && rtx_cost (GEN_INT (normalizep), mode, PLUS, 1,
6281 : optimize_insn_for_speed_p ()) == 0)
6282 : {
6283 0 : tem = emit_store_flag_1 (subtarget, rcode, op0, op1, mode, 0,
6284 : STORE_FLAG_VALUE, target_mode);
6285 0 : if (tem)
6286 0 : return expand_binop (target_mode, add_optab, tem,
6287 0 : gen_int_mode (normalizep, target_mode),
6288 : target, 0, OPTAB_WIDEN);
6289 : }
6290 0 : else if (!want_add
6291 0 : && rtx_cost (trueval, mode, XOR, 1,
6292 : optimize_insn_for_speed_p ()) == 0)
6293 : {
6294 0 : tem = emit_store_flag_1 (subtarget, rcode, op0, op1, mode, 0,
6295 : normalizep, target_mode);
6296 0 : if (tem)
6297 0 : return expand_binop (target_mode, xor_optab, tem, trueval,
6298 0 : target, INTVAL (trueval) >= 0,
6299 0 : OPTAB_WIDEN);
6300 : }
6301 : }
6302 :
6303 68473 : delete_insns_since (last);
6304 :
6305 : /* Cannot split ORDERED and UNORDERED, only try the above trick. */
6306 68473 : if (code == ORDERED || code == UNORDERED)
6307 : return 0;
6308 :
6309 68331 : and_them = split_comparison (code, mode, &first_code, &code);
6310 :
6311 : /* If there are no NaNs, the first comparison should always fall through.
6312 : Effectively change the comparison to the other one. */
6313 68331 : if (!HONOR_NANS (mode))
6314 : {
6315 404 : gcc_assert (first_code == (and_them ? ORDERED : UNORDERED));
6316 243 : return emit_store_flag_1 (target, code, op0, op1, mode, 0, normalizep,
6317 243 : target_mode);
6318 : }
6319 :
6320 68088 : if (!HAVE_conditional_move)
6321 : return 0;
6322 :
6323 : /* Do not turn a trapping comparison into a non-trapping one. */
6324 68088 : if ((code != EQ && code != NE && code != UNEQ && code != LTGT)
6325 2573 : && flag_trapping_math)
6326 : return 0;
6327 :
6328 : /* Try using a setcc instruction for ORDERED/UNORDERED, followed by a
6329 : conditional move. */
6330 65515 : tem = emit_store_flag_1 (subtarget, first_code, op0, op1, mode, 0,
6331 : normalizep, target_mode);
6332 65515 : if (tem == 0)
6333 : return 0;
6334 :
6335 64988 : if (and_them)
6336 2683 : tem = emit_conditional_move (target, { code, op0, op1, mode },
6337 2683 : tem, const0_rtx, GET_MODE (tem), 0);
6338 : else
6339 62305 : tem = emit_conditional_move (target, { code, op0, op1, mode },
6340 62305 : trueval, tem, GET_MODE (tem), 0);
6341 :
6342 64988 : if (tem == 0)
6343 685 : delete_insns_since (last);
6344 : return tem;
6345 : }
6346 :
6347 : /* The remaining tricks only apply to integer comparisons. */
6348 :
6349 8330 : scalar_int_mode int_mode;
6350 8330 : if (is_int_mode (mode, &int_mode))
6351 4808 : return emit_store_flag_int (target, subtarget, code, op0, op1, int_mode,
6352 4808 : unsignedp, normalizep, trueval);
6353 :
6354 : return 0;
6355 : }
6356 :
6357 : /* Like emit_store_flag, but always succeeds. */
6358 :
6359 : rtx
6360 605141 : emit_store_flag_force (rtx target, enum rtx_code code, rtx op0, rtx op1,
6361 : machine_mode mode, int unsignedp, int normalizep)
6362 : {
6363 605141 : rtx tem;
6364 605141 : rtx_code_label *label;
6365 605141 : rtx trueval, falseval;
6366 :
6367 : /* First see if emit_store_flag can do the job. */
6368 605141 : tem = emit_store_flag (target, code, op0, op1, mode, unsignedp, normalizep);
6369 605141 : if (tem != 0)
6370 : return tem;
6371 :
6372 : /* If one operand is constant, make it the second one. Only do this
6373 : if the other operand is not constant as well. */
6374 12344 : if (swap_commutative_operands_p (op0, op1))
6375 : {
6376 8 : std::swap (op0, op1);
6377 8 : code = swap_condition (code);
6378 : }
6379 :
6380 12344 : if (mode == VOIDmode)
6381 0 : mode = GET_MODE (op0);
6382 :
6383 12344 : if (!target)
6384 0 : target = gen_reg_rtx (word_mode);
6385 :
6386 : /* If this failed, we have to do this with set/compare/jump/set code.
6387 : For foo != 0, if foo is in OP0, just replace it with 1 if nonzero. */
6388 12344 : trueval = normalizep ? GEN_INT (normalizep) : const1_rtx;
6389 12344 : if (code == NE
6390 1756 : && GET_MODE_CLASS (mode) == MODE_INT
6391 41 : && REG_P (target)
6392 41 : && op0 == target
6393 0 : && op1 == const0_rtx)
6394 : {
6395 0 : label = gen_label_rtx ();
6396 0 : do_compare_rtx_and_jump (target, const0_rtx, EQ, unsignedp, mode,
6397 : NULL_RTX, NULL, label,
6398 : profile_probability::uninitialized ());
6399 0 : emit_move_insn (target, trueval);
6400 0 : emit_label (label);
6401 0 : return target;
6402 : }
6403 :
6404 12344 : if (!REG_P (target)
6405 12344 : || reg_mentioned_p (target, op0) || reg_mentioned_p (target, op1))
6406 5 : target = gen_reg_rtx (GET_MODE (target));
6407 :
6408 : /* Jump in the right direction if the target cannot implement CODE
6409 : but can jump on its reverse condition. */
6410 12344 : falseval = const0_rtx;
6411 12344 : if (! can_compare_p (code, mode, ccp_jump)
6412 12344 : && (! FLOAT_MODE_P (mode)
6413 7410 : || code == ORDERED || code == UNORDERED
6414 7223 : || (! HONOR_NANS (mode) && (code == LTGT || code == UNEQ))
6415 7223 : || (! HONOR_SNANS (mode) && (code == EQ || code == NE))))
6416 : {
6417 2751 : enum rtx_code rcode;
6418 2751 : if (FLOAT_MODE_P (mode))
6419 2751 : rcode = reverse_condition_maybe_unordered (code);
6420 : else
6421 0 : rcode = reverse_condition (code);
6422 :
6423 : /* Canonicalize to UNORDERED for the libcall. */
6424 2751 : if (can_compare_p (rcode, mode, ccp_jump)
6425 2751 : || (code == ORDERED && ! can_compare_p (ORDERED, mode, ccp_jump)))
6426 : {
6427 155 : falseval = trueval;
6428 155 : trueval = const0_rtx;
6429 155 : code = rcode;
6430 : }
6431 : }
6432 :
6433 12344 : emit_move_insn (target, trueval);
6434 12344 : label = gen_label_rtx ();
6435 12344 : do_compare_rtx_and_jump (op0, op1, code, unsignedp, mode, NULL_RTX, NULL,
6436 : label, profile_probability::uninitialized ());
6437 :
6438 12344 : emit_move_insn (target, falseval);
6439 12344 : emit_label (label);
6440 :
6441 12344 : return target;
6442 : }
6443 :
6444 : /* Expand a vector (left) rotate of MODE of X by an immediate AMT as a vector
6445 : permute operation. Emit code to put the result in DST if successful and
6446 : return it. Otherwise return NULL. This is intended to implement vector
6447 : rotates by byte amounts using vector permutes when the target does not offer
6448 : native vector rotate operations. */
6449 : rtx
6450 0 : expand_rotate_as_vec_perm (machine_mode mode, rtx dst, rtx x, rtx amt)
6451 : {
6452 0 : rtx amt_unwrap = unwrap_const_vec_duplicate (amt);
6453 : /* For now handle only rotate by the same integer constant in all lanes.
6454 : In principle rotates by any constant vector are representable through
6455 : permutes as long as the individual rotate amounts are multiples of
6456 : BITS_PER_UNIT. */
6457 0 : if (!CONST_INT_P (amt_unwrap))
6458 : return NULL_RTX;
6459 :
6460 0 : int rotamnt = INTVAL (amt_unwrap);
6461 0 : if (rotamnt % BITS_PER_UNIT != 0)
6462 : return NULL_RTX;
6463 0 : machine_mode qimode;
6464 0 : if (!qimode_for_vec_perm (mode).exists (&qimode))
6465 0 : return NULL_RTX;
6466 :
6467 0 : vec_perm_builder builder;
6468 0 : unsigned nunits = GET_MODE_SIZE (GET_MODE_INNER (mode));
6469 0 : poly_uint64 total_units = GET_MODE_SIZE (mode);
6470 0 : builder.new_vector (total_units, nunits, 3);
6471 0 : unsigned rot_bytes = rotamnt / BITS_PER_UNIT;
6472 0 : unsigned rot_to_perm = BYTES_BIG_ENDIAN ? rot_bytes : nunits - rot_bytes;
6473 0 : for (unsigned j = 0; j < 3 * nunits; j += nunits)
6474 0 : for (unsigned i = 0; i < nunits; i++)
6475 0 : builder.quick_push ((rot_to_perm + i) % nunits + j);
6476 :
6477 0 : rtx perm_src = lowpart_subreg (qimode, x, mode);
6478 0 : rtx perm_dst = lowpart_subreg (qimode, dst, mode);
6479 0 : rtx res
6480 0 : = expand_vec_perm_const (qimode, perm_src, perm_src, builder,
6481 : qimode, perm_dst);
6482 0 : if (!res)
6483 : return NULL_RTX;
6484 0 : if (!rtx_equal_p (res, perm_dst))
6485 0 : emit_move_insn (dst, lowpart_subreg (mode, res, qimode));
6486 : return dst;
6487 0 : }
6488 :
6489 : /* Helper function for canonicalize_cmp_for_target. Swap between inclusive
6490 : and exclusive ranges in order to create an equivalent comparison. See
6491 : canonicalize_cmp_for_target for the possible cases. */
6492 :
6493 : static enum rtx_code
6494 47 : equivalent_cmp_code (enum rtx_code code)
6495 : {
6496 47 : switch (code)
6497 : {
6498 : case GT:
6499 : return GE;
6500 0 : case GE:
6501 0 : return GT;
6502 0 : case LT:
6503 0 : return LE;
6504 0 : case LE:
6505 0 : return LT;
6506 2 : case GTU:
6507 2 : return GEU;
6508 0 : case GEU:
6509 0 : return GTU;
6510 1 : case LTU:
6511 1 : return LEU;
6512 2 : case LEU:
6513 2 : return LTU;
6514 :
6515 0 : default:
6516 0 : return code;
6517 : }
6518 : }
6519 :
6520 : /* Choose the more appropriate immediate in scalar integer comparisons. The
6521 : purpose of this is to end up with an immediate which can be loaded into a
6522 : register in fewer moves, if possible.
6523 :
6524 : For each integer comparison there exists an equivalent choice:
6525 : i) a > b or a >= b + 1
6526 : ii) a <= b or a < b + 1
6527 : iii) a >= b or a > b - 1
6528 : iv) a < b or a <= b - 1
6529 :
6530 : MODE is the mode of the first operand.
6531 : CODE points to the comparison code.
6532 : IMM points to the rtx containing the immediate. *IMM must satisfy
6533 : CONST_SCALAR_INT_P on entry and continues to satisfy CONST_SCALAR_INT_P
6534 : on exit. */
6535 :
6536 : void
6537 4817368 : canonicalize_comparison (machine_mode mode, enum rtx_code *code, rtx *imm)
6538 : {
6539 4817368 : if (!SCALAR_INT_MODE_P (mode))
6540 3968529 : return;
6541 :
6542 4813549 : int to_add = 0;
6543 4813549 : enum signop sgn = unsigned_condition_p (*code) ? UNSIGNED : SIGNED;
6544 :
6545 : /* Extract the immediate value from the rtx. */
6546 4813549 : wide_int imm_val = rtx_mode_t (*imm, mode);
6547 :
6548 4813549 : if (*code == GT || *code == GTU || *code == LE || *code == LEU)
6549 : to_add = 1;
6550 : else if (*code == GE || *code == GEU || *code == LT || *code == LTU)
6551 : to_add = -1;
6552 : else
6553 : return;
6554 :
6555 : /* Check for overflow/underflow in the case of signed values and
6556 : wrapping around in the case of unsigned values. If any occur
6557 : cancel the optimization. */
6558 848983 : wi::overflow_type overflow = wi::OVF_NONE;
6559 848983 : wide_int imm_modif;
6560 :
6561 848983 : if (to_add == 1)
6562 614093 : imm_modif = wi::add (imm_val, 1, sgn, &overflow);
6563 : else
6564 234890 : imm_modif = wi::sub (imm_val, 1, sgn, &overflow);
6565 :
6566 848983 : if (overflow)
6567 144 : return;
6568 :
6569 848839 : rtx new_imm = immed_wide_int_const (imm_modif, mode);
6570 :
6571 848839 : int old_cost = rtx_cost (*imm, mode, COMPARE, 0, true);
6572 848839 : int new_cost = rtx_cost (new_imm, mode, COMPARE, 0, true);
6573 :
6574 848839 : if (dump_file && (dump_flags & TDF_DETAILS))
6575 : {
6576 7 : fprintf (dump_file, ";; cmp: %s, old cst: ",
6577 7 : GET_RTX_NAME (*code));
6578 7 : print_rtl (dump_file, *imm);
6579 7 : fprintf (dump_file, " new cst: ");
6580 7 : print_rtl (dump_file, new_imm);
6581 7 : fprintf (dump_file, "\n");
6582 7 : fprintf (dump_file, ";; old cst cost: %d, new cst cost: %d\n",
6583 : old_cost, new_cost);
6584 : }
6585 :
6586 : /* Update the immediate and the code. */
6587 848839 : if (old_cost > new_cost)
6588 : {
6589 47 : *code = equivalent_cmp_code (*code);
6590 47 : *imm = new_imm;
6591 : }
6592 4813693 : }
6593 :
6594 :
6595 :
6596 : /* Perform possibly multi-word comparison and conditional jump to LABEL
6597 : if ARG1 OP ARG2 true where ARG1 and ARG2 are of mode MODE. This is
6598 : now a thin wrapper around do_compare_rtx_and_jump. */
6599 :
6600 : static void
6601 2407 : do_cmp_and_jump (rtx arg1, rtx arg2, enum rtx_code op, machine_mode mode,
6602 : rtx_code_label *label)
6603 : {
6604 2407 : int unsignedp = (op == LTU || op == LEU || op == GTU || op == GEU);
6605 2407 : do_compare_rtx_and_jump (arg1, arg2, op, unsignedp, mode, NULL_RTX,
6606 : NULL, label, profile_probability::uninitialized ());
6607 2407 : }
|
