Line data Source code
1 : /* Lower _BitInt(N) operations to scalar operations.
2 : Copyright (C) 2023-2026 Free Software Foundation, Inc.
3 : Contributed by Jakub Jelinek <jakub@redhat.com>.
4 :
5 : This file is part of GCC.
6 :
7 : GCC is free software; you can redistribute it and/or modify it
8 : under the terms of the GNU General Public License as published by the
9 : Free Software Foundation; either version 3, or (at your option) any
10 : later version.
11 :
12 : GCC is distributed in the hope that it will be useful, but WITHOUT
13 : ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 : for more details.
16 :
17 : You should have received a copy of the GNU General Public License
18 : along with GCC; see the file COPYING3. If not see
19 : <http://www.gnu.org/licenses/>. */
20 :
21 : #include "config.h"
22 : #include "system.h"
23 : #include "coretypes.h"
24 : #include "backend.h"
25 : #include "rtl.h"
26 : #include "tree.h"
27 : #include "gimple.h"
28 : #include "cfghooks.h"
29 : #include "tree-pass.h"
30 : #include "ssa.h"
31 : #include "fold-const.h"
32 : #include "gimplify.h"
33 : #include "gimple-iterator.h"
34 : #include "tree-cfg.h"
35 : #include "tree-dfa.h"
36 : #include "cfgloop.h"
37 : #include "cfganal.h"
38 : #include "target.h"
39 : #include "tree-ssa-live.h"
40 : #include "tree-ssa-coalesce.h"
41 : #include "domwalk.h"
42 : #include "memmodel.h"
43 : #include "optabs.h"
44 : #include "varasm.h"
45 : #include "gimple-range.h"
46 : #include "value-range.h"
47 : #include "langhooks.h"
48 : #include "gimplify-me.h"
49 : #include "diagnostic-core.h"
50 : #include "tree-eh.h"
51 : #include "tree-pretty-print.h"
52 : #include "alloc-pool.h"
53 : #include "tree-into-ssa.h"
54 : #include "tree-cfgcleanup.h"
55 : #include "tree-switch-conversion.h"
56 : #include "ubsan.h"
57 : #include "stor-layout.h"
58 : #include "gimple-lower-bitint.h"
59 : #include "attribs.h"
60 : #include "asan.h"
61 : #include "tree-outof-ssa.h"
62 :
63 : /* Split BITINT_TYPE precisions in 4 categories. Small _BitInt, where
64 : target hook says it is a single limb, middle _BitInt which per ABI
65 : does not, but there is some INTEGER_TYPE in which arithmetics can be
66 : performed (operations on such _BitInt are lowered to casts to that
67 : arithmetic type and cast back; e.g. on x86_64 limb is DImode, but
68 : target supports TImode, so _BitInt(65) to _BitInt(128) are middle
69 : ones), large _BitInt which should by straight line code and
70 : finally huge _BitInt which should be handled by loops over the limbs. */
71 :
72 : enum bitint_prec_kind {
73 : bitint_prec_small,
74 : bitint_prec_middle,
75 : bitint_prec_large,
76 : bitint_prec_huge
77 : };
78 :
79 : /* Caches to speed up bitint_precision_kind. */
80 :
81 : static int small_max_prec, mid_min_prec, large_min_prec, huge_min_prec;
82 : static int limb_prec, abi_limb_prec;
83 : static bool bitint_big_endian;
84 : static enum bitint_ext bitint_extended;
85 :
86 : /* Categorize _BitInt(PREC) as small, middle, large or huge. */
87 :
88 : static bitint_prec_kind
89 497425 : bitint_precision_kind (int prec)
90 : {
91 497425 : if (prec <= small_max_prec)
92 : return bitint_prec_small;
93 475682 : if (huge_min_prec && prec >= huge_min_prec)
94 : return bitint_prec_huge;
95 268021 : if (large_min_prec && prec >= large_min_prec)
96 : return bitint_prec_large;
97 53503 : if (mid_min_prec && prec >= mid_min_prec)
98 : return bitint_prec_middle;
99 :
100 10473 : struct bitint_info info;
101 10473 : bool ok = targetm.c.bitint_type_info (prec, &info);
102 10473 : gcc_assert (ok);
103 10473 : scalar_int_mode limb_mode = as_a <scalar_int_mode> (info.limb_mode);
104 10473 : if (prec <= GET_MODE_PRECISION (limb_mode))
105 : {
106 2881 : small_max_prec = prec;
107 2881 : return bitint_prec_small;
108 : }
109 7592 : bitint_big_endian = info.big_endian;
110 7592 : bitint_extended = info.extended;
111 7592 : if (info.limb_mode == info.abi_limb_mode && bitint_extended == bitint_ext_full)
112 0 : bitint_extended = bitint_ext_partial;
113 7592 : if (!large_min_prec
114 15080 : && GET_MODE_PRECISION (limb_mode) <= MAX_FIXED_MODE_SIZE)
115 14976 : large_min_prec = MAX_FIXED_MODE_SIZE + 1;
116 7592 : if (!limb_prec)
117 7488 : limb_prec = GET_MODE_PRECISION (limb_mode);
118 7592 : if (!abi_limb_prec)
119 7488 : abi_limb_prec
120 7488 : = GET_MODE_PRECISION (as_a <scalar_int_mode> (info.abi_limb_mode));
121 : /* For bitint_ext_full with different limb_mode from abi_limb_mode we
122 : currently only support only abi_limb_mode twice the precision of
123 : limb_mode, and don't support big endian in that case either. */
124 7592 : gcc_assert (bitint_extended != bitint_ext_full
125 : || (abi_limb_prec == 2 * limb_prec
126 : && !bitint_big_endian));
127 7592 : if (!huge_min_prec)
128 : {
129 14976 : if (4 * limb_prec >= MAX_FIXED_MODE_SIZE)
130 7488 : huge_min_prec = 4 * limb_prec;
131 : else
132 0 : huge_min_prec = MAX_FIXED_MODE_SIZE + 1;
133 : }
134 15184 : if (prec <= MAX_FIXED_MODE_SIZE)
135 : {
136 1655 : if (!mid_min_prec || prec < mid_min_prec)
137 1655 : mid_min_prec = prec;
138 : return bitint_prec_middle;
139 : }
140 5937 : if (huge_min_prec && prec >= huge_min_prec)
141 2405 : return bitint_prec_huge;
142 : return bitint_prec_large;
143 : }
144 :
145 : /* Same for a TYPE. */
146 :
147 : static bitint_prec_kind
148 491959 : bitint_precision_kind (tree type)
149 : {
150 491959 : return bitint_precision_kind (TYPE_PRECISION (type));
151 : }
152 :
153 : /* Return minimum precision needed to describe INTEGER_CST
154 : CST. All bits above that precision up to precision of
155 : TREE_TYPE (CST) are cleared if EXT is set to 0, or set
156 : if EXT is set to -1. */
157 :
158 : static unsigned
159 5502 : bitint_min_cst_precision (tree cst, int &ext)
160 : {
161 5502 : ext = tree_int_cst_sgn (cst) < 0 ? -1 : 0;
162 5502 : wide_int w = wi::to_wide (cst);
163 5502 : unsigned min_prec = wi::min_precision (w, TYPE_SIGN (TREE_TYPE (cst)));
164 : /* For signed values, we don't need to count the sign bit,
165 : we'll use constant 0 or -1 for the upper bits. */
166 5502 : if (!TYPE_UNSIGNED (TREE_TYPE (cst)))
167 3321 : --min_prec;
168 : else
169 : {
170 : /* For unsigned values, also try signed min_precision
171 : in case the constant has lots of most significant bits set. */
172 2181 : unsigned min_prec2 = wi::min_precision (w, SIGNED) - 1;
173 2181 : if (min_prec2 < min_prec)
174 : {
175 1010 : ext = -1;
176 1010 : return min_prec2;
177 : }
178 : }
179 : return min_prec;
180 5502 : }
181 :
182 : namespace {
183 :
184 : /* If OP is middle _BitInt, cast it to corresponding INTEGER_TYPE
185 : cached in TYPE and return it. */
186 :
187 : tree
188 8109 : maybe_cast_middle_bitint (gimple_stmt_iterator *gsi, tree op, tree &type)
189 : {
190 8109 : if (op == NULL_TREE
191 7996 : || !BITINT_TYPE_P (TREE_TYPE (op))
192 15150 : || bitint_precision_kind (TREE_TYPE (op)) != bitint_prec_middle)
193 : return op;
194 :
195 7023 : int prec = TYPE_PRECISION (TREE_TYPE (op));
196 7023 : int uns = TYPE_UNSIGNED (TREE_TYPE (op));
197 7023 : if (type == NULL_TREE
198 2628 : || TYPE_PRECISION (type) != prec
199 9651 : || TYPE_UNSIGNED (type) != uns)
200 4395 : type = build_nonstandard_integer_type (prec, uns);
201 :
202 7023 : if (TREE_CODE (op) != SSA_NAME)
203 : {
204 2436 : tree nop = fold_convert (type, op);
205 2436 : if (is_gimple_val (nop))
206 : return nop;
207 : }
208 :
209 4587 : tree nop = make_ssa_name (type);
210 4587 : gimple *g = gimple_build_assign (nop, NOP_EXPR, op);
211 4587 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
212 4587 : return nop;
213 : }
214 :
215 : /* Return true if STMT can be handled in a loop from least to most
216 : significant limb together with its dependencies. */
217 :
218 : bool
219 48640 : mergeable_op (gimple *stmt)
220 : {
221 48640 : if (!is_gimple_assign (stmt))
222 : return false;
223 39626 : switch (gimple_assign_rhs_code (stmt))
224 : {
225 : case PLUS_EXPR:
226 : case MINUS_EXPR:
227 : case NEGATE_EXPR:
228 : case BIT_AND_EXPR:
229 : case BIT_IOR_EXPR:
230 : case BIT_XOR_EXPR:
231 : case BIT_NOT_EXPR:
232 : case SSA_NAME:
233 : case INTEGER_CST:
234 : case BIT_FIELD_REF:
235 : return true;
236 410 : case LSHIFT_EXPR:
237 410 : {
238 410 : tree cnt = gimple_assign_rhs2 (stmt);
239 410 : if (tree_fits_uhwi_p (cnt)
240 198 : && tree_to_uhwi (cnt) < (unsigned HOST_WIDE_INT) limb_prec)
241 : return true;
242 : }
243 : break;
244 6361 : CASE_CONVERT:
245 6361 : case VIEW_CONVERT_EXPR:
246 6361 : {
247 6361 : tree lhs_type = TREE_TYPE (gimple_assign_lhs (stmt));
248 6361 : tree rhs_type = TREE_TYPE (gimple_assign_rhs1 (stmt));
249 6361 : if (TREE_CODE (gimple_assign_rhs1 (stmt)) == SSA_NAME
250 6319 : && BITINT_TYPE_P (lhs_type)
251 3744 : && BITINT_TYPE_P (rhs_type)
252 3371 : && bitint_precision_kind (lhs_type) >= bitint_prec_large
253 3335 : && bitint_precision_kind (rhs_type) >= bitint_prec_large
254 9533 : && (CEIL (TYPE_PRECISION (lhs_type), limb_prec)
255 3172 : == CEIL (TYPE_PRECISION (rhs_type), limb_prec)))
256 : {
257 2198 : if (TYPE_PRECISION (rhs_type) >= TYPE_PRECISION (lhs_type))
258 : return true;
259 183 : if ((unsigned) TYPE_PRECISION (lhs_type) % (2 * limb_prec) != 0)
260 : return true;
261 17 : if (bitint_precision_kind (lhs_type) == bitint_prec_large)
262 : return true;
263 : }
264 : break;
265 : }
266 : default:
267 : break;
268 : }
269 : return false;
270 : }
271 :
272 : /* Return non-zero if stmt is .{ADD,SUB,MUL}_OVERFLOW call with
273 : _Complex large/huge _BitInt lhs which has at most two immediate uses,
274 : at most one use in REALPART_EXPR stmt in the same bb and exactly one
275 : IMAGPART_EXPR use in the same bb with a single use which casts it to
276 : non-BITINT_TYPE integral type. If there is a REALPART_EXPR use,
277 : return 2. Such cases (most common uses of those builtins) can be
278 : optimized by marking their lhs and lhs of IMAGPART_EXPR and maybe lhs
279 : of REALPART_EXPR as not needed to be backed up by a stack variable.
280 : For .UBSAN_CHECK_{ADD,SUB,MUL} return 3. */
281 :
282 : int
283 21926 : optimizable_arith_overflow (gimple *stmt)
284 : {
285 21926 : bool is_ubsan = false;
286 21926 : if (!is_gimple_call (stmt) || !gimple_call_internal_p (stmt))
287 : return false;
288 5306 : switch (gimple_call_internal_fn (stmt))
289 : {
290 : case IFN_ADD_OVERFLOW:
291 : case IFN_SUB_OVERFLOW:
292 : case IFN_MUL_OVERFLOW:
293 : break;
294 63 : case IFN_UBSAN_CHECK_ADD:
295 63 : case IFN_UBSAN_CHECK_SUB:
296 63 : case IFN_UBSAN_CHECK_MUL:
297 63 : is_ubsan = true;
298 63 : break;
299 : default:
300 : return 0;
301 : }
302 5210 : tree lhs = gimple_call_lhs (stmt);
303 5210 : if (!lhs)
304 : return 0;
305 5210 : if (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (lhs))
306 : return 0;
307 5195 : tree type = is_ubsan ? TREE_TYPE (lhs) : TREE_TYPE (TREE_TYPE (lhs));
308 0 : if (!BITINT_TYPE_P (type)
309 5195 : || bitint_precision_kind (type) < bitint_prec_large)
310 : return 0;
311 :
312 5195 : if (is_ubsan)
313 : {
314 48 : use_operand_p use_p;
315 48 : gimple *use_stmt;
316 48 : if (!single_imm_use (lhs, &use_p, &use_stmt)
317 48 : || gimple_bb (use_stmt) != gimple_bb (stmt)
318 48 : || !gimple_store_p (use_stmt)
319 48 : || !is_gimple_assign (use_stmt)
320 48 : || gimple_has_volatile_ops (use_stmt)
321 96 : || stmt_ends_bb_p (use_stmt))
322 0 : return 0;
323 : return 3;
324 : }
325 :
326 5147 : imm_use_iterator ui;
327 5147 : use_operand_p use_p;
328 5147 : int seen = 0;
329 5147 : gimple *realpart = NULL, *cast = NULL;
330 14740 : FOR_EACH_IMM_USE_FAST (use_p, ui, lhs)
331 : {
332 9807 : gimple *g = USE_STMT (use_p);
333 9807 : if (is_gimple_debug (g))
334 0 : continue;
335 9807 : if (!is_gimple_assign (g) || gimple_bb (g) != gimple_bb (stmt))
336 : return 0;
337 9807 : if (gimple_assign_rhs_code (g) == REALPART_EXPR)
338 : {
339 4660 : if ((seen & 1) != 0)
340 : return 0;
341 4660 : seen |= 1;
342 4660 : realpart = g;
343 : }
344 5147 : else if (gimple_assign_rhs_code (g) == IMAGPART_EXPR)
345 : {
346 5147 : if ((seen & 2) != 0)
347 214 : return 0;
348 5147 : seen |= 2;
349 :
350 5147 : use_operand_p use2_p;
351 5147 : gimple *use_stmt;
352 5147 : tree lhs2 = gimple_assign_lhs (g);
353 5147 : if (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (lhs2))
354 : return 0;
355 5147 : if (!single_imm_use (lhs2, &use2_p, &use_stmt)
356 5147 : || gimple_bb (use_stmt) != gimple_bb (stmt)
357 10294 : || !gimple_assign_cast_p (use_stmt))
358 : return 0;
359 :
360 4937 : lhs2 = gimple_assign_lhs (use_stmt);
361 9874 : if (!INTEGRAL_TYPE_P (TREE_TYPE (lhs2))
362 9874 : || BITINT_TYPE_P (TREE_TYPE (lhs2)))
363 : return 0;
364 4933 : cast = use_stmt;
365 : }
366 : else
367 : return 0;
368 214 : }
369 4933 : if ((seen & 2) == 0)
370 : return 0;
371 4933 : if (seen == 3)
372 : {
373 : /* Punt if the cast stmt appears before realpart stmt, because
374 : if both appear, the lowering wants to emit all the code
375 : at the location of realpart stmt. */
376 4660 : gimple_stmt_iterator gsi = gsi_for_stmt (realpart);
377 4660 : unsigned int cnt = 0;
378 4663 : do
379 : {
380 4663 : gsi_prev_nondebug (&gsi);
381 4663 : if (gsi_end_p (gsi) || gsi_stmt (gsi) == cast)
382 : return 0;
383 4660 : if (gsi_stmt (gsi) == stmt)
384 : return 2;
385 : /* If realpart is too far from stmt, punt as well.
386 : Usually it will appear right after it. */
387 3 : if (++cnt == 32)
388 : return 0;
389 : }
390 : while (1);
391 : }
392 : return 1;
393 : }
394 :
395 : /* If STMT is some kind of comparison (GIMPLE_COND, comparison assignment)
396 : comparing large/huge _BitInt types, return the comparison code and if
397 : non-NULL fill in the comparison operands to *POP1 and *POP2. */
398 :
399 : tree_code
400 36315 : comparison_op (gimple *stmt, tree *pop1, tree *pop2)
401 : {
402 36315 : tree op1 = NULL_TREE, op2 = NULL_TREE;
403 36315 : tree_code code = ERROR_MARK;
404 36315 : if (gimple_code (stmt) == GIMPLE_COND)
405 : {
406 6667 : code = gimple_cond_code (stmt);
407 6667 : op1 = gimple_cond_lhs (stmt);
408 6667 : op2 = gimple_cond_rhs (stmt);
409 : }
410 29648 : else if (is_gimple_assign (stmt))
411 : {
412 29621 : code = gimple_assign_rhs_code (stmt);
413 29621 : op1 = gimple_assign_rhs1 (stmt);
414 29621 : if (TREE_CODE_CLASS (code) == tcc_comparison
415 29621 : || TREE_CODE_CLASS (code) == tcc_binary)
416 2275 : op2 = gimple_assign_rhs2 (stmt);
417 : }
418 36315 : if (TREE_CODE_CLASS (code) != tcc_comparison)
419 : return ERROR_MARK;
420 7449 : tree type = TREE_TYPE (op1);
421 10 : if (!BITINT_TYPE_P (type)
422 7459 : || bitint_precision_kind (type) < bitint_prec_large)
423 : return ERROR_MARK;
424 7449 : if (pop1)
425 : {
426 7387 : *pop1 = op1;
427 7387 : *pop2 = op2;
428 : }
429 : return code;
430 : }
431 :
432 : /* Class used during large/huge _BitInt lowering containing all the
433 : state for the methods. */
434 :
435 : struct bitint_large_huge
436 : {
437 7486 : bitint_large_huge ()
438 7486 : : m_names (NULL), m_loads (NULL), m_preserved (NULL),
439 7486 : m_single_use_names (NULL), m_map (NULL), m_vars (NULL),
440 7486 : m_limb_type (NULL_TREE), m_data (vNULL),
441 7486 : m_returns_twice_calls (vNULL) {}
442 :
443 : ~bitint_large_huge ();
444 :
445 : void insert_before (gimple *);
446 : tree limb_access_type (tree, tree);
447 : tree limb_access (tree, tree, tree, bool, bool = false);
448 : tree build_bit_field_ref (tree, tree, unsigned HOST_WIDE_INT,
449 : unsigned HOST_WIDE_INT);
450 : void if_then (gimple *, profile_probability, edge &, edge &);
451 : void if_then_else (gimple *, profile_probability, edge &, edge &);
452 : void if_then_if_then_else (gimple *g, gimple *,
453 : profile_probability, profile_probability,
454 : edge &, edge &, edge &);
455 : tree handle_operand (tree, tree);
456 : tree prepare_data_in_out (tree, tree, tree *, tree = NULL_TREE);
457 : tree add_cast (tree, tree);
458 : tree handle_plus_minus (tree_code, tree, tree, tree);
459 : tree handle_lshift (tree, tree, tree);
460 : tree handle_cast (tree, tree, tree);
461 : tree handle_bit_field_ref (tree, tree);
462 : tree handle_load (gimple *, tree);
463 : tree handle_stmt (gimple *, tree);
464 : tree handle_operand_addr (tree, gimple *, int *, int *);
465 : tree create_loop (tree, tree *);
466 : tree lower_mergeable_stmt (gimple *, tree_code &, tree, tree);
467 : tree lower_comparison_stmt (gimple *, tree_code &, tree, tree);
468 : void lower_shift_stmt (tree, gimple *);
469 : void lower_muldiv_stmt (tree, gimple *);
470 : void lower_float_conv_stmt (tree, gimple *);
471 : tree arith_overflow_extract_bits (unsigned int, unsigned int, tree,
472 : unsigned int, bool);
473 : void finish_arith_overflow (tree, tree, tree, tree, tree, tree, gimple *,
474 : unsigned, tree_code);
475 : void lower_addsub_overflow (tree, gimple *);
476 : void lower_mul_overflow (tree, gimple *);
477 : void lower_cplxpart_stmt (tree, gimple *);
478 : void lower_complexexpr_stmt (gimple *);
479 : void lower_bit_query (gimple *);
480 : void lower_bswap_bitreverse (tree, gimple *);
481 : void lower_clmul (tree, gimple *);
482 : void lower_call (tree, gimple *);
483 : void lower_asm (gimple *);
484 : void lower_stmt (gimple *);
485 :
486 : /* Bitmap of large/huge _BitInt SSA_NAMEs except those can be
487 : merged with their uses. */
488 : bitmap m_names;
489 : /* Subset of those for lhs of load statements. These will be
490 : cleared in m_names if the loads will be mergeable with all
491 : their uses. */
492 : bitmap m_loads;
493 : /* Bitmap of large/huge _BitInt SSA_NAMEs that should survive
494 : to later passes (arguments or return values of calls). */
495 : bitmap m_preserved;
496 : /* Subset of m_names which have a single use. As the lowering
497 : can replace various original statements with their lowered
498 : form even before it is done iterating over all basic blocks,
499 : testing has_single_use for the purpose of emitting clobbers
500 : doesn't work properly. */
501 : bitmap m_single_use_names;
502 : /* Used for coalescing/partitioning of large/huge _BitInt SSA_NAMEs
503 : set in m_names. */
504 : var_map m_map;
505 : /* Mapping of the partitions to corresponding decls. */
506 : tree *m_vars;
507 : /* Unsigned integer type with limb precision. */
508 : tree m_limb_type;
509 : /* Its TYPE_SIZE_UNIT. */
510 : unsigned HOST_WIDE_INT m_limb_size;
511 : /* Location of a gimple stmt which is being currently lowered. */
512 : location_t m_loc;
513 : /* Current stmt iterator where code is being lowered currently. */
514 : gimple_stmt_iterator m_gsi;
515 : /* Statement after which any clobbers should be added if non-NULL. */
516 : gimple *m_after_stmt;
517 : /* Set when creating loops to the loop header bb and its preheader. */
518 : basic_block m_bb, m_preheader_bb;
519 : /* Stmt iterator after which initialization statements should be emitted. */
520 : gimple_stmt_iterator m_init_gsi;
521 : /* Decl into which a mergeable statement stores result. */
522 : tree m_lhs;
523 : /* handle_operand/handle_stmt can be invoked in various ways.
524 :
525 : lower_mergeable_stmt for large _BitInt calls those with constant
526 : idx only, expanding to straight line code, for huge _BitInt
527 : emits a loop from least significant limb upwards, where each loop
528 : iteration handles 2 limbs, plus there can be up to one full limb
529 : and one partial limb processed after the loop, where handle_operand
530 : and/or handle_stmt are called with constant idx. m_upwards_2limb
531 : is set for this case, false otherwise. m_upwards is true if it
532 : is either large or huge _BitInt handled by lower_mergeable_stmt,
533 : i.e. indexes always increase.
534 :
535 : Another way is used by lower_comparison_stmt, which walks limbs
536 : from most significant to least significant, partial limb if any
537 : processed first with constant idx and then loop processing a single
538 : limb per iteration with non-constant idx.
539 :
540 : Another way is used in lower_shift_stmt, where for LSHIFT_EXPR
541 : destination limbs are processed from most significant to least
542 : significant or for RSHIFT_EXPR the other way around, in loops or
543 : straight line code, but idx usually is non-constant (so from
544 : handle_operand/handle_stmt POV random access). The LSHIFT_EXPR
545 : handling there can access even partial limbs using non-constant
546 : idx (then m_var_msb should be true, for all the other cases
547 : including lower_mergeable_stmt/lower_comparison_stmt that is
548 : not the case and so m_var_msb should be false.
549 :
550 : m_first should be set the first time handle_operand/handle_stmt
551 : is called and clear when it is called for some other limb with
552 : the same argument. If the lowering of an operand (e.g. INTEGER_CST)
553 : or statement (e.g. +/-/<< with < limb_prec constant) needs some
554 : state between the different calls, when m_first is true it should
555 : push some trees to m_data vector and also make sure m_data_cnt is
556 : incremented by how many trees were pushed, and when m_first is
557 : false, it can use the m_data[m_data_cnt] etc. data or update them,
558 : just needs to bump m_data_cnt by the same amount as when it was
559 : called with m_first set. The toplevel calls to
560 : handle_operand/handle_stmt should set m_data_cnt to 0 and truncate
561 : m_data vector when setting m_first to true.
562 :
563 : m_cast_conditional and m_bitfld_load are used when handling a
564 : bit-field load inside of a widening cast. handle_cast sometimes
565 : needs to do runtime comparisons and handle_operand only conditionally
566 : or even in two separate conditional blocks for one idx (once with
567 : constant index after comparing the runtime one for equality with the
568 : constant). In these cases, m_cast_conditional is set to true and
569 : the bit-field load then communicates its m_data_cnt to handle_cast
570 : using m_bitfld_load. */
571 : bool m_first;
572 : bool m_var_msb;
573 : unsigned m_upwards_2limb;
574 : bool m_upwards;
575 : bool m_cast_conditional;
576 : unsigned m_bitfld_load;
577 : vec<tree> m_data;
578 : unsigned int m_data_cnt;
579 : vec<gimple *> m_returns_twice_calls;
580 : };
581 :
582 7486 : bitint_large_huge::~bitint_large_huge ()
583 : {
584 7486 : BITMAP_FREE (m_names);
585 7486 : BITMAP_FREE (m_loads);
586 7486 : BITMAP_FREE (m_preserved);
587 7486 : BITMAP_FREE (m_single_use_names);
588 7486 : if (m_map)
589 5788 : delete_var_map (m_map);
590 7486 : XDELETEVEC (m_vars);
591 7486 : m_data.release ();
592 7486 : m_returns_twice_calls.release ();
593 7486 : }
594 :
595 : /* Insert gimple statement G before current location
596 : and set its gimple_location. */
597 :
598 : void
599 368806 : bitint_large_huge::insert_before (gimple *g)
600 : {
601 368806 : gimple_set_location (g, m_loc);
602 368806 : gsi_insert_before (&m_gsi, g, GSI_SAME_STMT);
603 368806 : }
604 :
605 : /* Return type for accessing limb IDX of BITINT_TYPE TYPE.
606 : This is normally m_limb_type, except for a partial most
607 : significant limb if any. */
608 :
609 : tree
610 131869 : bitint_large_huge::limb_access_type (tree type, tree idx)
611 : {
612 131869 : if (type == NULL_TREE)
613 5943 : return m_limb_type;
614 125926 : unsigned HOST_WIDE_INT i = tree_to_uhwi (idx);
615 125926 : unsigned int prec = TYPE_PRECISION (type);
616 125926 : gcc_assert (i * limb_prec < prec
617 : || (bitint_extended == bitint_ext_full
618 : && abi_limb_prec > limb_prec
619 : && i * limb_prec
620 : < CEIL (prec, abi_limb_prec) * abi_limb_prec));
621 251852 : if (bitint_big_endian
622 125926 : ? (i != 0 || (prec % limb_prec) == 0)
623 125926 : : (i + 1) * limb_prec <= prec)
624 82089 : return m_limb_type;
625 : else
626 87674 : return build_nonstandard_integer_type (prec % limb_prec,
627 43837 : TYPE_UNSIGNED (type));
628 : }
629 :
630 : /* Return a tree how to access limb IDX of VAR corresponding to BITINT_TYPE
631 : TYPE. If WRITE_P is true, it will be a store, otherwise a read. */
632 :
633 : tree
634 157784 : bitint_large_huge::limb_access (tree type, tree var, tree idx, bool write_p,
635 : bool abi_load_p)
636 : {
637 157784 : tree atype = (tree_fits_uhwi_p (idx)
638 157784 : ? limb_access_type (type, idx) : m_limb_type);
639 :
640 157784 : tree ltype = (bitint_extended && abi_load_p) ? atype : m_limb_type;
641 :
642 157784 : addr_space_t as = TYPE_ADDR_SPACE (TREE_TYPE (var));
643 157784 : tree ret;
644 157784 : if (DECL_P (var) && tree_fits_uhwi_p (idx))
645 : {
646 96728 : if (as != TYPE_ADDR_SPACE (ltype))
647 0 : ltype = build_qualified_type (ltype,
648 0 : TYPE_QUALS (ltype).with_as (as));
649 96728 : tree ptype = build_pointer_type (strip_array_types (TREE_TYPE (var)));
650 96728 : unsigned HOST_WIDE_INT off = tree_to_uhwi (idx) * m_limb_size;
651 96728 : if (bitint_big_endian)
652 0 : off += m_limb_size - tree_to_uhwi (TYPE_SIZE_UNIT (ltype));
653 96728 : ret = build2 (MEM_REF, ltype,
654 : build_fold_addr_expr (var),
655 96728 : build_int_cst (ptype, off));
656 96728 : TREE_THIS_VOLATILE (ret) = TREE_THIS_VOLATILE (var);
657 96728 : TREE_SIDE_EFFECTS (ret) = TREE_SIDE_EFFECTS (var);
658 96728 : }
659 61056 : else if (TREE_CODE (var) == MEM_REF && tree_fits_uhwi_p (idx))
660 : {
661 4316 : if (as != TYPE_ADDR_SPACE (ltype))
662 0 : ltype = build_qualified_type (ltype,
663 0 : TYPE_QUALS (ltype).with_as (as));
664 4316 : unsigned HOST_WIDE_INT off = tree_to_uhwi (idx) * m_limb_size;
665 4316 : if (bitint_big_endian)
666 0 : off += m_limb_size - tree_to_uhwi (TYPE_SIZE_UNIT (ltype));
667 4316 : ret
668 8632 : = build2 (MEM_REF, ltype, unshare_expr (TREE_OPERAND (var, 0)),
669 8632 : size_binop (PLUS_EXPR, TREE_OPERAND (var, 1),
670 : build_int_cst (TREE_TYPE (TREE_OPERAND (var, 1)),
671 : off)));
672 4316 : TREE_THIS_VOLATILE (ret) = TREE_THIS_VOLATILE (var);
673 4316 : TREE_SIDE_EFFECTS (ret) = TREE_SIDE_EFFECTS (var);
674 4316 : TREE_THIS_NOTRAP (ret) = TREE_THIS_NOTRAP (var);
675 4316 : }
676 : else
677 : {
678 56740 : ltype = m_limb_type;
679 56740 : if (as != TYPE_ADDR_SPACE (ltype))
680 17 : ltype = build_qualified_type (ltype,
681 17 : TYPE_QUALS (ltype).with_as (as));
682 56740 : var = unshare_expr (var);
683 56740 : if (TREE_CODE (TREE_TYPE (var)) != ARRAY_TYPE
684 85216 : || !useless_type_conversion_p (m_limb_type,
685 28476 : TREE_TYPE (TREE_TYPE (var))))
686 : {
687 29260 : unsigned HOST_WIDE_INT nelts
688 29260 : = CEIL (tree_to_uhwi (TYPE_SIZE (TREE_TYPE (var))), limb_prec);
689 29260 : tree atype = build_array_type_nelts (ltype, nelts);
690 29260 : var = build1 (VIEW_CONVERT_EXPR, atype, var);
691 : }
692 56740 : ret = build4 (ARRAY_REF, ltype, var, idx, NULL_TREE, NULL_TREE);
693 : }
694 157784 : if (!write_p && !useless_type_conversion_p (atype, ltype))
695 : {
696 19230 : gimple *g = gimple_build_assign (make_ssa_name (m_limb_type), ret);
697 19230 : insert_before (g);
698 19230 : ret = gimple_assign_lhs (g);
699 19230 : ret = build1 (NOP_EXPR, atype, ret);
700 : }
701 157784 : return ret;
702 : }
703 :
704 : /* Build a BIT_FIELD_REF to access BITSIZE bits with FTYPE type at
705 : offset BITPOS inside of OBJ. */
706 :
707 : tree
708 273 : bitint_large_huge::build_bit_field_ref (tree ftype, tree obj,
709 : unsigned HOST_WIDE_INT bitsize,
710 : unsigned HOST_WIDE_INT bitpos)
711 : {
712 546 : if (INTEGRAL_TYPE_P (TREE_TYPE (obj))
713 282 : && !type_has_mode_precision_p (TREE_TYPE (obj)))
714 : {
715 9 : unsigned HOST_WIDE_INT nelts
716 9 : = CEIL (tree_to_uhwi (TYPE_SIZE (TREE_TYPE (obj))), limb_prec);
717 9 : tree ltype = m_limb_type;
718 9 : addr_space_t as = TYPE_ADDR_SPACE (TREE_TYPE (obj));
719 9 : if (as != TYPE_ADDR_SPACE (ltype))
720 0 : ltype = build_qualified_type (ltype,
721 0 : TYPE_QUALS (ltype).with_as (as));
722 9 : tree atype = build_array_type_nelts (ltype, nelts);
723 9 : obj = build1 (VIEW_CONVERT_EXPR, atype, obj);
724 : }
725 273 : return build3 (BIT_FIELD_REF, ftype, obj, bitsize_int (bitsize),
726 273 : bitsize_int (bitpos));
727 : }
728 :
729 : /* Emit a half diamond,
730 : if (COND)
731 : |\
732 : | \
733 : | \
734 : | new_bb1
735 : | /
736 : | /
737 : |/
738 : or if (COND) new_bb1;
739 : PROB is the probability that the condition is true.
740 : Updates m_gsi to start of new_bb1.
741 : Sets EDGE_TRUE to edge from new_bb1 to successor and
742 : EDGE_FALSE to the EDGE_FALSE_VALUE edge from if (COND) bb. */
743 :
744 : void
745 4436 : bitint_large_huge::if_then (gimple *cond, profile_probability prob,
746 : edge &edge_true, edge &edge_false)
747 : {
748 4436 : insert_before (cond);
749 4436 : edge e1 = split_block (gsi_bb (m_gsi), cond);
750 4436 : edge e2 = split_block (e1->dest, (gimple *) NULL);
751 4436 : edge e3 = make_edge (e1->src, e2->dest, EDGE_FALSE_VALUE);
752 4436 : e1->flags = EDGE_TRUE_VALUE;
753 4436 : e1->probability = prob;
754 4436 : e3->probability = prob.invert ();
755 4436 : set_immediate_dominator (CDI_DOMINATORS, e2->dest, e1->src);
756 4436 : edge_true = e2;
757 4436 : edge_false = e3;
758 4436 : m_gsi = gsi_after_labels (e1->dest);
759 4436 : }
760 :
761 : /* Emit a full diamond,
762 : if (COND)
763 : /\
764 : / \
765 : / \
766 : new_bb1 new_bb2
767 : \ /
768 : \ /
769 : \/
770 : or if (COND) new_bb2; else new_bb1;
771 : PROB is the probability that the condition is true.
772 : Updates m_gsi to start of new_bb2.
773 : Sets EDGE_TRUE to edge from new_bb1 to successor and
774 : EDGE_FALSE to the EDGE_FALSE_VALUE edge from if (COND) bb. */
775 :
776 : void
777 112 : bitint_large_huge::if_then_else (gimple *cond, profile_probability prob,
778 : edge &edge_true, edge &edge_false)
779 : {
780 112 : insert_before (cond);
781 112 : edge e1 = split_block (gsi_bb (m_gsi), cond);
782 112 : edge e2 = split_block (e1->dest, (gimple *) NULL);
783 112 : basic_block bb = create_empty_bb (e1->dest);
784 112 : add_bb_to_loop (bb, e1->dest->loop_father);
785 112 : edge e3 = make_edge (e1->src, bb, EDGE_TRUE_VALUE);
786 112 : e1->flags = EDGE_FALSE_VALUE;
787 112 : e3->probability = prob;
788 112 : e1->probability = prob.invert ();
789 112 : bb->count = e1->src->count.apply_probability (prob);
790 112 : set_immediate_dominator (CDI_DOMINATORS, bb, e1->src);
791 112 : set_immediate_dominator (CDI_DOMINATORS, e2->dest, e1->src);
792 112 : edge_true = make_single_succ_edge (bb, e2->dest, EDGE_FALLTHRU);
793 112 : edge_false = e2;
794 112 : m_gsi = gsi_after_labels (bb);
795 112 : }
796 :
797 : /* Emit a half diamond with full diamond in it
798 : if (COND1)
799 : |\
800 : | \
801 : | \
802 : | if (COND2)
803 : | / \
804 : | / \
805 : |new_bb1 new_bb2
806 : | | /
807 : \ | /
808 : \ | /
809 : \ | /
810 : \|/
811 : or if (COND1) { if (COND2) new_bb2; else new_bb1; }
812 : PROB1 is the probability that the condition 1 is true.
813 : PROB2 is the probability that the condition 2 is true.
814 : Updates m_gsi to start of new_bb1.
815 : Sets EDGE_TRUE_TRUE to edge from new_bb2 to successor,
816 : EDGE_TRUE_FALSE to edge from new_bb1 to successor and
817 : EDGE_FALSE to the EDGE_FALSE_VALUE edge from if (COND1) bb.
818 : If COND2 is NULL, this is equivalent to
819 : if_then (COND1, PROB1, EDGE_TRUE_FALSE, EDGE_FALSE);
820 : EDGE_TRUE_TRUE = NULL; */
821 :
822 : void
823 2098 : bitint_large_huge::if_then_if_then_else (gimple *cond1, gimple *cond2,
824 : profile_probability prob1,
825 : profile_probability prob2,
826 : edge &edge_true_true,
827 : edge &edge_true_false,
828 : edge &edge_false)
829 : {
830 2098 : edge e2, e3, e4 = NULL;
831 2098 : if_then (cond1, prob1, e2, e3);
832 2098 : if (cond2 == NULL)
833 : {
834 1257 : edge_true_true = NULL;
835 1257 : edge_true_false = e2;
836 1257 : edge_false = e3;
837 1257 : return;
838 : }
839 841 : insert_before (cond2);
840 841 : e2 = split_block (gsi_bb (m_gsi), cond2);
841 841 : basic_block bb = create_empty_bb (e2->dest);
842 841 : add_bb_to_loop (bb, e2->dest->loop_father);
843 841 : e4 = make_edge (e2->src, bb, EDGE_TRUE_VALUE);
844 841 : set_immediate_dominator (CDI_DOMINATORS, bb, e2->src);
845 841 : e4->probability = prob2;
846 841 : e2->flags = EDGE_FALSE_VALUE;
847 841 : e2->probability = prob2.invert ();
848 841 : bb->count = e2->src->count.apply_probability (prob2);
849 841 : e4 = make_single_succ_edge (bb, e3->dest, EDGE_FALLTHRU);
850 841 : e2 = find_edge (e2->dest, e3->dest);
851 841 : edge_true_true = e4;
852 841 : edge_true_false = e2;
853 841 : edge_false = e3;
854 841 : m_gsi = gsi_after_labels (e2->src);
855 : }
856 :
857 : /* Emit code to access limb IDX from OP. */
858 :
859 : tree
860 112879 : bitint_large_huge::handle_operand (tree op, tree idx)
861 : {
862 112879 : switch (TREE_CODE (op))
863 : {
864 78053 : case SSA_NAME:
865 78053 : if (m_names == NULL
866 78053 : || !bitmap_bit_p (m_names, SSA_NAME_VERSION (op)))
867 : {
868 14619 : if (SSA_NAME_IS_DEFAULT_DEF (op))
869 : {
870 47 : if (m_first)
871 : {
872 16 : tree v = create_tmp_reg (m_limb_type);
873 16 : if (SSA_NAME_VAR (op) && VAR_P (SSA_NAME_VAR (op)))
874 : {
875 16 : DECL_NAME (v) = DECL_NAME (SSA_NAME_VAR (op));
876 16 : DECL_SOURCE_LOCATION (v)
877 16 : = DECL_SOURCE_LOCATION (SSA_NAME_VAR (op));
878 : }
879 16 : v = get_or_create_ssa_default_def (cfun, v);
880 16 : m_data.safe_push (v);
881 : }
882 47 : tree ret = m_data[m_data_cnt];
883 47 : m_data_cnt++;
884 47 : if (tree_fits_uhwi_p (idx))
885 : {
886 45 : tree type = limb_access_type (TREE_TYPE (op), idx);
887 45 : ret = add_cast (type, ret);
888 : }
889 : return ret;
890 : }
891 14572 : location_t loc_save = m_loc;
892 14572 : m_loc = gimple_location (SSA_NAME_DEF_STMT (op));
893 14572 : tree ret = handle_stmt (SSA_NAME_DEF_STMT (op), idx);
894 14572 : m_loc = loc_save;
895 14572 : return ret;
896 : }
897 63434 : int p;
898 63434 : gimple *g;
899 63434 : tree t;
900 63434 : p = var_to_partition (m_map, op);
901 63434 : gcc_assert (m_vars[p] != NULL_TREE);
902 63434 : t = limb_access (TREE_TYPE (op), m_vars[p], idx, false);
903 63434 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (t)), t);
904 63434 : insert_before (g);
905 63434 : t = gimple_assign_lhs (g);
906 63434 : if (m_first
907 22965 : && m_single_use_names
908 22044 : && m_vars[p] != m_lhs
909 21935 : && m_after_stmt
910 72265 : && bitmap_bit_p (m_single_use_names, SSA_NAME_VERSION (op)))
911 : {
912 8508 : tree clobber = build_clobber (TREE_TYPE (m_vars[p]),
913 : CLOBBER_STORAGE_END);
914 8508 : g = gimple_build_assign (m_vars[p], clobber);
915 8508 : gimple_stmt_iterator gsi = gsi_for_stmt (m_after_stmt);
916 8508 : gsi_insert_after (&gsi, g, GSI_SAME_STMT);
917 : }
918 : return t;
919 34826 : case INTEGER_CST:
920 34826 : if (tree_fits_uhwi_p (idx))
921 : {
922 24053 : tree c, type = limb_access_type (TREE_TYPE (op), idx);
923 24053 : unsigned HOST_WIDE_INT i = tree_to_uhwi (idx);
924 24053 : if (m_first)
925 : {
926 6265 : m_data.safe_push (NULL_TREE);
927 6265 : m_data.safe_push (NULL_TREE);
928 : }
929 24053 : if (bitint_big_endian)
930 0 : i = CEIL (TYPE_PRECISION (TREE_TYPE (op)), limb_prec) - 1 - i;
931 24053 : if (limb_prec != HOST_BITS_PER_WIDE_INT)
932 : {
933 0 : wide_int w = wi::rshift (wi::to_wide (op), i * limb_prec,
934 0 : TYPE_SIGN (TREE_TYPE (op)));
935 0 : c = wide_int_to_tree (type,
936 0 : wide_int::from (w, TYPE_PRECISION (type),
937 : UNSIGNED));
938 0 : }
939 24053 : else if (i >= TREE_INT_CST_EXT_NUNITS (op))
940 7634 : c = build_int_cst (type,
941 13580 : tree_int_cst_sgn (op) < 0 ? -1 : 0);
942 : else
943 16419 : c = build_int_cst (type, TREE_INT_CST_ELT (op, i));
944 24053 : m_data_cnt += 2;
945 24053 : return c;
946 : }
947 10773 : if (m_first
948 10773 : || (m_data[m_data_cnt] == NULL_TREE
949 159 : && m_data[m_data_cnt + 1] == NULL_TREE))
950 : {
951 5432 : unsigned int prec = TYPE_PRECISION (TREE_TYPE (op));
952 5432 : unsigned int rem = prec % ((m_upwards_2limb ? 2 : 1) * limb_prec);
953 5432 : int ext;
954 5432 : unsigned min_prec = bitint_min_cst_precision (op, ext);
955 5432 : if (m_first)
956 : {
957 5273 : m_data.safe_push (NULL_TREE);
958 5273 : m_data.safe_push (NULL_TREE);
959 : }
960 5432 : if (integer_zerop (op))
961 : {
962 867 : tree c = build_zero_cst (m_limb_type);
963 867 : m_data[m_data_cnt] = c;
964 867 : m_data[m_data_cnt + 1] = c;
965 : }
966 4565 : else if (integer_all_onesp (op))
967 : {
968 679 : tree c = build_all_ones_cst (m_limb_type);
969 679 : m_data[m_data_cnt] = c;
970 679 : m_data[m_data_cnt + 1] = c;
971 : }
972 3886 : else if (m_upwards_2limb && min_prec <= (unsigned) limb_prec)
973 : {
974 : /* Single limb constant. Use a phi with that limb from
975 : the preheader edge and 0 or -1 constant from the other edge
976 : and for the second limb in the loop. */
977 925 : tree out;
978 925 : gcc_assert (m_first);
979 925 : m_data.pop ();
980 925 : m_data.pop ();
981 925 : prepare_data_in_out (fold_convert (m_limb_type, op), idx, &out,
982 925 : build_int_cst (m_limb_type, ext));
983 925 : }
984 2961 : else if (min_prec > prec - rem - 2 * limb_prec)
985 : {
986 : /* Constant which has enough significant bits that it isn't
987 : worth trying to save .rodata space by extending from smaller
988 : number. */
989 2434 : tree type;
990 2434 : if (m_var_msb)
991 25 : type = TREE_TYPE (op);
992 : else
993 : /* If we have a guarantee the most significant partial limb
994 : (if any) will be only accessed through handle_operand
995 : with INTEGER_CST idx, we don't need to include the partial
996 : limb in .rodata. */
997 2409 : type = build_bitint_type (prec - rem, 1);
998 2434 : tree c = tree_output_constant_def (fold_convert (type, op));
999 2434 : m_data[m_data_cnt] = c;
1000 2434 : m_data[m_data_cnt + 1] = NULL_TREE;
1001 : }
1002 527 : else if (m_upwards_2limb)
1003 : {
1004 : /* Constant with smaller number of bits. Trade conditional
1005 : code for .rodata space by extending from smaller number. */
1006 452 : min_prec = CEIL (min_prec, 2 * limb_prec) * (2 * limb_prec);
1007 452 : tree type = build_bitint_type (min_prec, 1);
1008 452 : tree c = tree_output_constant_def (fold_convert (type, op));
1009 452 : tree ridx = idx;
1010 452 : if (bitint_big_endian)
1011 : {
1012 0 : ridx = make_ssa_name (sizetype);
1013 0 : g = gimple_build_assign (ridx, PLUS_EXPR, idx,
1014 0 : size_int (min_prec / limb_prec
1015 : - ((HOST_WIDE_INT)
1016 : CEIL (prec,
1017 : limb_prec))));
1018 0 : insert_before (g);
1019 : }
1020 452 : tree ridx2 = make_ssa_name (sizetype);
1021 452 : g = gimple_build_assign (ridx2, PLUS_EXPR, ridx,
1022 : bitint_big_endian
1023 0 : ? size_int (-1) : size_one_node);
1024 452 : insert_before (g);
1025 452 : if (bitint_big_endian)
1026 0 : g = gimple_build_cond (GE_EXPR, idx,
1027 0 : size_int (CEIL (prec, limb_prec)
1028 : - min_prec / limb_prec),
1029 : NULL_TREE, NULL_TREE);
1030 : else
1031 452 : g = gimple_build_cond (LT_EXPR, idx,
1032 452 : size_int (min_prec / limb_prec),
1033 : NULL_TREE, NULL_TREE);
1034 452 : edge edge_true, edge_false;
1035 904 : if_then (g, (min_prec >= (prec - rem) / 2
1036 320 : ? profile_probability::likely ()
1037 132 : : profile_probability::unlikely ()),
1038 : edge_true, edge_false);
1039 452 : tree c1 = limb_access (TREE_TYPE (op), c, ridx, false);
1040 452 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (c1)), c1);
1041 452 : insert_before (g);
1042 452 : c1 = gimple_assign_lhs (g);
1043 452 : tree c2 = limb_access (TREE_TYPE (op), c, ridx2, false);
1044 452 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (c2)), c2);
1045 452 : insert_before (g);
1046 452 : c2 = gimple_assign_lhs (g);
1047 452 : tree c3 = build_int_cst (m_limb_type, ext);
1048 452 : m_gsi = gsi_after_labels (edge_true->dest);
1049 452 : m_data[m_data_cnt] = make_ssa_name (m_limb_type);
1050 452 : m_data[m_data_cnt + 1] = make_ssa_name (m_limb_type);
1051 452 : gphi *phi = create_phi_node (m_data[m_data_cnt],
1052 : edge_true->dest);
1053 452 : add_phi_arg (phi, c1, edge_true, UNKNOWN_LOCATION);
1054 452 : add_phi_arg (phi, c3, edge_false, UNKNOWN_LOCATION);
1055 452 : phi = create_phi_node (m_data[m_data_cnt + 1], edge_true->dest);
1056 452 : add_phi_arg (phi, c2, edge_true, UNKNOWN_LOCATION);
1057 452 : add_phi_arg (phi, c3, edge_false, UNKNOWN_LOCATION);
1058 : }
1059 : else
1060 : {
1061 : /* Constant with smaller number of bits. Trade conditional
1062 : code for .rodata space by extending from smaller number.
1063 : Version for loops with random access to the limbs or
1064 : downwards loops. */
1065 75 : min_prec = CEIL (min_prec, limb_prec) * limb_prec;
1066 75 : tree c;
1067 75 : if (min_prec <= (unsigned) limb_prec)
1068 21 : c = fold_convert (m_limb_type, op);
1069 : else
1070 : {
1071 54 : tree type = build_bitint_type (min_prec, 1);
1072 54 : c = tree_output_constant_def (fold_convert (type, op));
1073 : }
1074 75 : m_data[m_data_cnt] = c;
1075 75 : m_data[m_data_cnt + 1] = integer_type_node;
1076 : }
1077 5432 : t = m_data[m_data_cnt];
1078 : }
1079 : else
1080 5341 : t = m_data[m_data_cnt + 1];
1081 10773 : if (m_data[m_data_cnt + 1] == NULL_TREE)
1082 : {
1083 4812 : tree ridx = idx;
1084 4812 : unsigned int prec = TYPE_PRECISION (TREE_TYPE (op));
1085 4812 : tree c = m_data[m_data_cnt];
1086 4812 : unsigned int min_prec = TYPE_PRECISION (TREE_TYPE (c));
1087 4812 : if (bitint_big_endian
1088 0 : && CEIL (min_prec, limb_prec) != CEIL (prec, limb_prec))
1089 : {
1090 0 : ridx = make_ssa_name (sizetype);
1091 0 : g = gimple_build_assign (ridx, PLUS_EXPR, idx,
1092 0 : size_int (CEIL (min_prec, limb_prec)
1093 : - ((HOST_WIDE_INT)
1094 : CEIL (prec, limb_prec))));
1095 0 : insert_before (g);
1096 : }
1097 4812 : t = limb_access (TREE_TYPE (op), c, ridx, false);
1098 4812 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (t)), t);
1099 4812 : insert_before (g);
1100 4812 : t = gimple_assign_lhs (g);
1101 : }
1102 5961 : else if (m_data[m_data_cnt + 1] == integer_type_node)
1103 : {
1104 115 : unsigned int prec = TYPE_PRECISION (TREE_TYPE (op));
1105 115 : unsigned rem = prec % ((m_upwards_2limb ? 2 : 1) * limb_prec);
1106 115 : int ext = wi::neg_p (wi::to_wide (op)) ? -1 : 0;
1107 115 : tree c = m_data[m_data_cnt];
1108 115 : unsigned min_prec = TYPE_PRECISION (TREE_TYPE (c));
1109 115 : if (bitint_big_endian)
1110 0 : g = gimple_build_cond (GE_EXPR, idx,
1111 0 : size_int (CEIL (prec, limb_prec)
1112 : - min_prec / limb_prec),
1113 : NULL_TREE, NULL_TREE);
1114 : else
1115 115 : g = gimple_build_cond (LT_EXPR, idx,
1116 115 : size_int (min_prec / limb_prec),
1117 : NULL_TREE, NULL_TREE);
1118 115 : edge edge_true, edge_false;
1119 230 : if_then (g, (min_prec >= (prec - rem) / 2
1120 29 : ? profile_probability::likely ()
1121 86 : : profile_probability::unlikely ()),
1122 : edge_true, edge_false);
1123 115 : if (min_prec > (unsigned) limb_prec)
1124 : {
1125 70 : tree ridx = idx;
1126 70 : if (bitint_big_endian)
1127 : {
1128 0 : ridx = make_ssa_name (sizetype);
1129 0 : g = gimple_build_assign (ridx, PLUS_EXPR, idx,
1130 0 : size_int (min_prec / limb_prec
1131 : - ((HOST_WIDE_INT)
1132 : CEIL (prec,
1133 : limb_prec))));
1134 0 : insert_before (g);
1135 : }
1136 70 : c = limb_access (TREE_TYPE (op), c, ridx, false);
1137 70 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (c)), c);
1138 70 : insert_before (g);
1139 70 : c = gimple_assign_lhs (g);
1140 : }
1141 115 : tree c2 = build_int_cst (m_limb_type, ext);
1142 115 : m_gsi = gsi_after_labels (edge_true->dest);
1143 115 : t = make_ssa_name (m_limb_type);
1144 115 : gphi *phi = create_phi_node (t, edge_true->dest);
1145 115 : add_phi_arg (phi, c, edge_true, UNKNOWN_LOCATION);
1146 115 : add_phi_arg (phi, c2, edge_false, UNKNOWN_LOCATION);
1147 : }
1148 10773 : m_data_cnt += 2;
1149 10773 : return t;
1150 0 : default:
1151 0 : gcc_unreachable ();
1152 : }
1153 : }
1154 :
1155 : /* Helper method, add a PHI node with VAL from preheader edge if
1156 : inside of a loop and m_first. Keep state in a pair of m_data
1157 : elements. If VAL_OUT is non-NULL, use that as PHI argument from
1158 : the latch edge, otherwise create a new SSA_NAME for it and let
1159 : caller initialize it. */
1160 :
1161 : tree
1162 16045 : bitint_large_huge::prepare_data_in_out (tree val, tree idx, tree *data_out,
1163 : tree val_out)
1164 : {
1165 16045 : if (!m_first)
1166 : {
1167 9520 : *data_out = tree_fits_uhwi_p (idx) ? NULL_TREE : m_data[m_data_cnt + 1];
1168 9520 : return m_data[m_data_cnt];
1169 : }
1170 :
1171 6525 : *data_out = NULL_TREE;
1172 6525 : if (tree_fits_uhwi_p (idx))
1173 : {
1174 2024 : m_data.safe_push (val);
1175 2024 : m_data.safe_push (NULL_TREE);
1176 2024 : return val;
1177 : }
1178 :
1179 4501 : tree in = make_ssa_name (TREE_TYPE (val));
1180 4501 : gphi *phi = create_phi_node (in, m_bb);
1181 4501 : edge e1 = find_edge (m_preheader_bb, m_bb);
1182 4501 : edge e2 = EDGE_PRED (m_bb, 0);
1183 4501 : if (e1 == e2)
1184 4501 : e2 = EDGE_PRED (m_bb, 1);
1185 4501 : add_phi_arg (phi, val, e1, UNKNOWN_LOCATION);
1186 4501 : tree out = val_out ? val_out : make_ssa_name (TREE_TYPE (val));
1187 4501 : add_phi_arg (phi, out, e2, UNKNOWN_LOCATION);
1188 4501 : m_data.safe_push (in);
1189 4501 : m_data.safe_push (out);
1190 4501 : return in;
1191 : }
1192 :
1193 : /* Return VAL cast to TYPE. If VAL is INTEGER_CST, just
1194 : convert it without emitting any code, otherwise emit
1195 : the conversion statement before the current location. */
1196 :
1197 : tree
1198 39041 : bitint_large_huge::add_cast (tree type, tree val)
1199 : {
1200 39041 : if (TREE_CODE (val) == INTEGER_CST)
1201 4628 : return fold_convert (type, val);
1202 :
1203 34413 : tree lhs = make_ssa_name (type);
1204 34413 : gimple *g = gimple_build_assign (lhs, NOP_EXPR, val);
1205 34413 : insert_before (g);
1206 34413 : return lhs;
1207 : }
1208 :
1209 : /* Helper of handle_stmt method, handle PLUS_EXPR or MINUS_EXPR. */
1210 :
1211 : tree
1212 13495 : bitint_large_huge::handle_plus_minus (tree_code code, tree rhs1, tree rhs2,
1213 : tree idx)
1214 : {
1215 13495 : tree lhs, data_out, ctype;
1216 13495 : tree rhs1_type = TREE_TYPE (rhs1);
1217 13495 : gimple *g;
1218 13495 : tree data_in = prepare_data_in_out (build_zero_cst (m_limb_type), idx,
1219 : &data_out);
1220 :
1221 19281 : if (optab_handler (code == PLUS_EXPR ? uaddc5_optab : usubc5_optab,
1222 13495 : TYPE_MODE (m_limb_type)) != CODE_FOR_nothing)
1223 : {
1224 13495 : ctype = build_complex_type (m_limb_type);
1225 13495 : if (!types_compatible_p (rhs1_type, m_limb_type))
1226 : {
1227 1118 : if (!TYPE_UNSIGNED (rhs1_type))
1228 : {
1229 360 : tree type = unsigned_type_for (rhs1_type);
1230 360 : rhs1 = add_cast (type, rhs1);
1231 360 : rhs2 = add_cast (type, rhs2);
1232 : }
1233 1118 : rhs1 = add_cast (m_limb_type, rhs1);
1234 1118 : rhs2 = add_cast (m_limb_type, rhs2);
1235 : }
1236 13495 : lhs = make_ssa_name (ctype);
1237 19281 : g = gimple_build_call_internal (code == PLUS_EXPR
1238 : ? IFN_UADDC : IFN_USUBC,
1239 : 3, rhs1, rhs2, data_in);
1240 13495 : gimple_call_set_lhs (g, lhs);
1241 13495 : insert_before (g);
1242 13495 : if (data_out == NULL_TREE)
1243 11249 : data_out = make_ssa_name (m_limb_type);
1244 13495 : g = gimple_build_assign (data_out, IMAGPART_EXPR,
1245 : build1 (IMAGPART_EXPR, m_limb_type, lhs));
1246 13495 : insert_before (g);
1247 : }
1248 0 : else if (types_compatible_p (rhs1_type, m_limb_type))
1249 : {
1250 0 : ctype = build_complex_type (m_limb_type);
1251 0 : lhs = make_ssa_name (ctype);
1252 0 : g = gimple_build_call_internal (code == PLUS_EXPR
1253 : ? IFN_ADD_OVERFLOW : IFN_SUB_OVERFLOW,
1254 : 2, rhs1, rhs2);
1255 0 : gimple_call_set_lhs (g, lhs);
1256 0 : insert_before (g);
1257 0 : if (data_out == NULL_TREE)
1258 0 : data_out = make_ssa_name (m_limb_type);
1259 0 : if (!integer_zerop (data_in))
1260 : {
1261 0 : rhs1 = make_ssa_name (m_limb_type);
1262 0 : g = gimple_build_assign (rhs1, REALPART_EXPR,
1263 : build1 (REALPART_EXPR, m_limb_type, lhs));
1264 0 : insert_before (g);
1265 0 : rhs2 = make_ssa_name (m_limb_type);
1266 0 : g = gimple_build_assign (rhs2, IMAGPART_EXPR,
1267 : build1 (IMAGPART_EXPR, m_limb_type, lhs));
1268 0 : insert_before (g);
1269 0 : lhs = make_ssa_name (ctype);
1270 0 : g = gimple_build_call_internal (code == PLUS_EXPR
1271 : ? IFN_ADD_OVERFLOW
1272 : : IFN_SUB_OVERFLOW,
1273 : 2, rhs1, data_in);
1274 0 : gimple_call_set_lhs (g, lhs);
1275 0 : insert_before (g);
1276 0 : data_in = make_ssa_name (m_limb_type);
1277 0 : g = gimple_build_assign (data_in, IMAGPART_EXPR,
1278 : build1 (IMAGPART_EXPR, m_limb_type, lhs));
1279 0 : insert_before (g);
1280 0 : g = gimple_build_assign (data_out, PLUS_EXPR, rhs2, data_in);
1281 0 : insert_before (g);
1282 : }
1283 : else
1284 : {
1285 0 : g = gimple_build_assign (data_out, IMAGPART_EXPR,
1286 : build1 (IMAGPART_EXPR, m_limb_type, lhs));
1287 0 : insert_before (g);
1288 : }
1289 : }
1290 : else
1291 : {
1292 : /* Always perform the two additions or subtractions in
1293 : unsigned type, avoid introducing a temporary UB. While
1294 : the end result of the 2 additions or 2 subtractions in
1295 : a valid program should not overflow, temporarily it can.
1296 : See PR126503. */
1297 0 : tree utype = unsigned_type_for (rhs1_type);
1298 0 : tree in = add_cast (utype, data_in);
1299 0 : lhs = make_ssa_name (utype);
1300 0 : if (utype != rhs1_type)
1301 : {
1302 0 : rhs1 = add_cast (utype, rhs1);
1303 0 : rhs2 = add_cast (utype, rhs2);
1304 : }
1305 0 : g = gimple_build_assign (lhs, code, rhs1, rhs2);
1306 0 : insert_before (g);
1307 0 : rhs1 = make_ssa_name (utype);
1308 0 : g = gimple_build_assign (rhs1, code, lhs, in);
1309 0 : insert_before (g);
1310 0 : m_data[m_data_cnt] = NULL_TREE;
1311 0 : m_data_cnt += 2;
1312 0 : return utype != rhs1_type ? add_cast (rhs1_type, rhs1) : rhs1;
1313 : }
1314 13495 : rhs1 = make_ssa_name (m_limb_type);
1315 13495 : g = gimple_build_assign (rhs1, REALPART_EXPR,
1316 : build1 (REALPART_EXPR, m_limb_type, lhs));
1317 13495 : insert_before (g);
1318 13495 : if (!types_compatible_p (rhs1_type, m_limb_type))
1319 1118 : rhs1 = add_cast (rhs1_type, rhs1);
1320 13495 : m_data[m_data_cnt] = data_out;
1321 13495 : m_data_cnt += 2;
1322 13495 : return rhs1;
1323 : }
1324 :
1325 : /* Helper function for handle_stmt method, handle LSHIFT_EXPR by
1326 : count in [0, limb_prec - 1] range. */
1327 :
1328 : tree
1329 152 : bitint_large_huge::handle_lshift (tree rhs1, tree rhs2, tree idx)
1330 : {
1331 152 : unsigned HOST_WIDE_INT cnt = tree_to_uhwi (rhs2);
1332 152 : gcc_checking_assert (cnt < (unsigned) limb_prec);
1333 152 : if (cnt == 0)
1334 : return rhs1;
1335 :
1336 152 : tree lhs, data_out, rhs1_type = TREE_TYPE (rhs1);
1337 152 : gimple *g;
1338 152 : tree data_in = prepare_data_in_out (build_zero_cst (m_limb_type), idx,
1339 : &data_out);
1340 :
1341 152 : if (!integer_zerop (data_in))
1342 : {
1343 136 : lhs = make_ssa_name (m_limb_type);
1344 136 : g = gimple_build_assign (lhs, RSHIFT_EXPR, data_in,
1345 : build_int_cst (unsigned_type_node,
1346 136 : limb_prec - cnt));
1347 136 : insert_before (g);
1348 136 : if (!types_compatible_p (rhs1_type, m_limb_type))
1349 35 : lhs = add_cast (rhs1_type, lhs);
1350 : data_in = lhs;
1351 : }
1352 152 : if (types_compatible_p (rhs1_type, m_limb_type))
1353 : {
1354 117 : if (data_out == NULL_TREE)
1355 82 : data_out = make_ssa_name (m_limb_type);
1356 117 : g = gimple_build_assign (data_out, rhs1);
1357 117 : insert_before (g);
1358 : }
1359 152 : if (cnt < (unsigned) TYPE_PRECISION (rhs1_type))
1360 : {
1361 137 : lhs = make_ssa_name (rhs1_type);
1362 137 : g = gimple_build_assign (lhs, LSHIFT_EXPR, rhs1, rhs2);
1363 137 : insert_before (g);
1364 137 : if (!integer_zerop (data_in))
1365 : {
1366 121 : rhs1 = lhs;
1367 121 : lhs = make_ssa_name (rhs1_type);
1368 121 : g = gimple_build_assign (lhs, BIT_IOR_EXPR, rhs1, data_in);
1369 121 : insert_before (g);
1370 : }
1371 : }
1372 : else
1373 : lhs = data_in;
1374 152 : m_data[m_data_cnt] = data_out;
1375 152 : m_data_cnt += 2;
1376 152 : return lhs;
1377 : }
1378 :
1379 : /* Helper function for handle_stmt method, handle an integral
1380 : to integral conversion. */
1381 :
1382 : tree
1383 7615 : bitint_large_huge::handle_cast (tree lhs_type, tree rhs1, tree idx)
1384 : {
1385 7615 : tree rhs_type = TREE_TYPE (rhs1);
1386 7615 : gimple *g;
1387 7615 : if ((TREE_CODE (rhs1) == SSA_NAME || TREE_CODE (rhs1) == INTEGER_CST)
1388 7615 : && BITINT_TYPE_P (lhs_type)
1389 7615 : && BITINT_TYPE_P (rhs_type)
1390 6568 : && bitint_precision_kind (lhs_type) >= bitint_prec_large
1391 14183 : && bitint_precision_kind (rhs_type) >= bitint_prec_large)
1392 : {
1393 5929 : if (TYPE_PRECISION (rhs_type) >= TYPE_PRECISION (lhs_type)
1394 : /* If lhs has bigger precision than rhs, we can use
1395 : the simple case only if there is a guarantee that
1396 : the most significant limb is handled in straight
1397 : line code. If m_var_msb (on left shifts) or
1398 : if m_upwards_2limb * limb_prec is equal to
1399 : lhs precision or if not m_upwards_2limb and lhs_type
1400 : has precision which is multiple of limb_prec that is
1401 : not the case. */
1402 5929 : || (!m_var_msb
1403 1524 : && (CEIL (TYPE_PRECISION (lhs_type), limb_prec)
1404 1524 : == CEIL (TYPE_PRECISION (rhs_type), limb_prec))
1405 370 : && ((!m_upwards_2limb
1406 182 : && (TYPE_PRECISION (lhs_type) % limb_prec != 0))
1407 267 : || (m_upwards_2limb
1408 376 : && (m_upwards_2limb * limb_prec
1409 188 : < TYPE_PRECISION (lhs_type))))))
1410 : {
1411 4670 : tree ridx = idx;
1412 4670 : if (bitint_big_endian
1413 4670 : && (CEIL (TYPE_PRECISION (lhs_type), limb_prec)
1414 0 : != CEIL (TYPE_PRECISION (rhs_type), limb_prec)))
1415 : {
1416 0 : HOST_WIDE_INT diff = CEIL (TYPE_PRECISION (rhs_type), limb_prec);
1417 0 : diff -= CEIL (TYPE_PRECISION (lhs_type), limb_prec);
1418 0 : if (tree_fits_uhwi_p (idx))
1419 0 : ridx = size_int (tree_to_uhwi (idx) + diff);
1420 : else
1421 : {
1422 0 : tree t = make_ssa_name (sizetype);
1423 0 : g = gimple_build_assign (t, PLUS_EXPR, idx, size_int (diff));
1424 0 : insert_before (g);
1425 0 : ridx = t;
1426 : }
1427 : }
1428 4670 : rhs1 = handle_operand (rhs1, ridx);
1429 4670 : if (tree_fits_uhwi_p (idx))
1430 : {
1431 2389 : tree type = limb_access_type (lhs_type, idx);
1432 2389 : if (!types_compatible_p (type, TREE_TYPE (rhs1)))
1433 1250 : rhs1 = add_cast (type, rhs1);
1434 : }
1435 : return rhs1;
1436 : }
1437 1259 : tree t;
1438 : /* Indexes lower than this don't need any special processing. */
1439 1259 : unsigned low = ((unsigned) TYPE_PRECISION (rhs_type)
1440 1259 : - !TYPE_UNSIGNED (rhs_type)) / limb_prec;
1441 : /* Indexes >= than this always contain an extension. */
1442 1259 : unsigned high = CEIL ((unsigned) TYPE_PRECISION (rhs_type), limb_prec);
1443 1259 : unsigned lcnt = CEIL ((unsigned) TYPE_PRECISION (lhs_type), limb_prec);
1444 1259 : unsigned lowe = bitint_big_endian ? lcnt - 1 - low : low;
1445 1259 : bool save_first = m_first;
1446 1259 : if (m_first)
1447 : {
1448 414 : m_data.safe_push (NULL_TREE);
1449 414 : m_data.safe_push (NULL_TREE);
1450 414 : m_data.safe_push (NULL_TREE);
1451 414 : if (TYPE_UNSIGNED (rhs_type))
1452 : /* No need to keep state between iterations. */
1453 : ;
1454 193 : else if (m_upwards && !m_upwards_2limb)
1455 : /* We need to keep state between iterations, but
1456 : not within any loop, everything is straight line
1457 : code with only increasing indexes. */
1458 : ;
1459 153 : else if (!m_upwards_2limb)
1460 : {
1461 3 : unsigned save_data_cnt = m_data_cnt;
1462 3 : gimple_stmt_iterator save_gsi = m_gsi;
1463 3 : m_gsi = m_init_gsi;
1464 3 : if (gsi_end_p (m_gsi))
1465 0 : m_gsi = gsi_after_labels (gsi_bb (m_gsi));
1466 : else
1467 3 : gsi_next (&m_gsi);
1468 3 : m_data_cnt = save_data_cnt + 3;
1469 3 : t = handle_operand (rhs1, size_int (bitint_big_endian
1470 : ? high - 1 - low : low));
1471 3 : m_first = false;
1472 3 : m_data[save_data_cnt + 2]
1473 3 : = build_int_cst (integer_type_node, m_data_cnt);
1474 3 : m_data_cnt = save_data_cnt;
1475 3 : t = add_cast (signed_type_for (m_limb_type), t);
1476 3 : tree lpm1 = build_int_cst (unsigned_type_node, limb_prec - 1);
1477 3 : tree n = make_ssa_name (TREE_TYPE (t));
1478 3 : g = gimple_build_assign (n, RSHIFT_EXPR, t, lpm1);
1479 3 : insert_before (g);
1480 3 : m_data[save_data_cnt + 1] = add_cast (m_limb_type, n);
1481 3 : m_init_gsi = m_gsi;
1482 3 : if (gsi_end_p (m_init_gsi))
1483 0 : m_init_gsi = gsi_last_bb (gsi_bb (m_init_gsi));
1484 : else
1485 3 : gsi_prev (&m_init_gsi);
1486 3 : m_gsi = save_gsi;
1487 : }
1488 150 : else if (m_upwards_2limb * limb_prec < TYPE_PRECISION (rhs_type))
1489 : /* We need to keep state between iterations, but
1490 : fortunately not within the loop, only afterwards. */
1491 : ;
1492 : else
1493 : {
1494 146 : tree out;
1495 146 : m_data.truncate (m_data_cnt);
1496 146 : prepare_data_in_out (build_zero_cst (m_limb_type), idx, &out);
1497 146 : m_data.safe_push (NULL_TREE);
1498 : }
1499 : }
1500 :
1501 1259 : unsigned save_data_cnt = m_data_cnt;
1502 1259 : m_data_cnt += 3;
1503 1259 : if (!tree_fits_uhwi_p (idx))
1504 : {
1505 688 : if (m_upwards_2limb
1506 668 : && low >= m_upwards_2limb - m_first)
1507 : {
1508 158 : if (bitint_big_endian
1509 158 : && (CEIL (TYPE_PRECISION (lhs_type), limb_prec)
1510 0 : != CEIL (TYPE_PRECISION (rhs_type), limb_prec)))
1511 : {
1512 0 : HOST_WIDE_INT diff
1513 0 : = CEIL (TYPE_PRECISION (rhs_type), limb_prec);
1514 0 : diff -= CEIL (TYPE_PRECISION (lhs_type), limb_prec);
1515 0 : tree t = make_ssa_name (sizetype);
1516 0 : g = gimple_build_assign (t, PLUS_EXPR, idx, size_int (diff));
1517 0 : insert_before (g);
1518 0 : idx = t;
1519 : }
1520 158 : rhs1 = handle_operand (rhs1, idx);
1521 158 : if (m_first)
1522 131 : m_data[save_data_cnt + 2]
1523 262 : = build_int_cst (integer_type_node, m_data_cnt);
1524 158 : m_first = save_first;
1525 158 : return rhs1;
1526 : }
1527 1326 : bool single_comparison
1528 530 : = low == high || (m_upwards_2limb && (low & 1) == m_first);
1529 266 : tree idxc = idx;
1530 266 : if (!single_comparison
1531 266 : && m_upwards_2limb
1532 246 : && !m_first
1533 112 : && low + 1 == m_upwards_2limb)
1534 : /* In this case we know that idx <= low always,
1535 : so effectively we just needs a single comparison,
1536 : idx < low or idx == low, but we'd need to emit different
1537 : code for the 2 branches than single_comparison normally
1538 : emits. So, instead of special-casing that, emit a
1539 : low <= low comparison which cfg cleanup will clean up
1540 : at the end of the pass. */
1541 89 : idxc = size_int (lowe);
1542 530 : if (bitint_big_endian)
1543 0 : g = gimple_build_cond (single_comparison ? GT_EXPR : GE_EXPR,
1544 0 : idxc, size_int (lowe),
1545 : NULL_TREE, NULL_TREE);
1546 : else
1547 796 : g = gimple_build_cond (single_comparison ? LT_EXPR : LE_EXPR,
1548 530 : idxc, size_int (low), NULL_TREE, NULL_TREE);
1549 530 : edge edge_true_true, edge_true_false, edge_false;
1550 796 : if_then_if_then_else (g, (single_comparison ? NULL
1551 266 : : gimple_build_cond (EQ_EXPR, idx,
1552 266 : size_int (lowe),
1553 : NULL_TREE,
1554 : NULL_TREE)),
1555 : profile_probability::likely (),
1556 : profile_probability::unlikely (),
1557 : edge_true_true, edge_true_false, edge_false);
1558 530 : bool save_cast_conditional = m_cast_conditional;
1559 530 : m_cast_conditional = true;
1560 530 : m_bitfld_load = 0;
1561 530 : tree t1 = idx, t2 = NULL_TREE;
1562 530 : if (bitint_big_endian
1563 530 : && (CEIL (TYPE_PRECISION (lhs_type), limb_prec)
1564 0 : != CEIL (TYPE_PRECISION (rhs_type), limb_prec)))
1565 : {
1566 0 : HOST_WIDE_INT diff = CEIL (TYPE_PRECISION (rhs_type), limb_prec);
1567 0 : diff -= CEIL (TYPE_PRECISION (lhs_type), limb_prec);
1568 0 : t1 = make_ssa_name (sizetype);
1569 0 : g = gimple_build_assign (t1, PLUS_EXPR, idx, size_int (diff));
1570 0 : insert_before (g);
1571 : }
1572 530 : t1 = handle_operand (rhs1, t1);
1573 530 : if (m_first)
1574 208 : m_data[save_data_cnt + 2]
1575 416 : = build_int_cst (integer_type_node, m_data_cnt);
1576 530 : tree ext = NULL_TREE;
1577 530 : tree bitfld = NULL_TREE;
1578 530 : if (!single_comparison)
1579 : {
1580 266 : m_gsi = gsi_after_labels (edge_true_true->src);
1581 266 : m_first = false;
1582 266 : m_data_cnt = save_data_cnt + 3;
1583 266 : if (m_bitfld_load)
1584 : {
1585 4 : bitfld = m_data[m_bitfld_load];
1586 4 : m_data[m_bitfld_load] = m_data[m_bitfld_load + 2];
1587 4 : m_bitfld_load = 0;
1588 : }
1589 266 : t2 = handle_operand (rhs1, size_int (bitint_big_endian
1590 : ? high - 1 - low : low));
1591 266 : if (!useless_type_conversion_p (m_limb_type, TREE_TYPE (t2)))
1592 221 : t2 = add_cast (m_limb_type, t2);
1593 266 : if (!TYPE_UNSIGNED (rhs_type) && m_upwards_2limb)
1594 : {
1595 146 : ext = add_cast (signed_type_for (m_limb_type), t2);
1596 292 : tree lpm1 = build_int_cst (unsigned_type_node,
1597 146 : limb_prec - 1);
1598 146 : tree n = make_ssa_name (TREE_TYPE (ext));
1599 146 : g = gimple_build_assign (n, RSHIFT_EXPR, ext, lpm1);
1600 146 : insert_before (g);
1601 146 : ext = add_cast (m_limb_type, n);
1602 : }
1603 : }
1604 530 : tree t3;
1605 530 : if (TYPE_UNSIGNED (rhs_type))
1606 278 : t3 = build_zero_cst (m_limb_type);
1607 252 : else if (m_upwards_2limb && (save_first || ext != NULL_TREE))
1608 168 : t3 = m_data[save_data_cnt];
1609 : else
1610 84 : t3 = m_data[save_data_cnt + 1];
1611 530 : m_gsi = gsi_after_labels (edge_true_false->dest);
1612 530 : t = make_ssa_name (m_limb_type);
1613 530 : gphi *phi = create_phi_node (t, edge_true_false->dest);
1614 530 : add_phi_arg (phi, t1, edge_true_false, UNKNOWN_LOCATION);
1615 530 : add_phi_arg (phi, t3, edge_false, UNKNOWN_LOCATION);
1616 530 : if (edge_true_true)
1617 266 : add_phi_arg (phi, t2, edge_true_true, UNKNOWN_LOCATION);
1618 530 : if (ext)
1619 : {
1620 146 : tree t4 = make_ssa_name (m_limb_type);
1621 146 : phi = create_phi_node (t4, edge_true_false->dest);
1622 146 : add_phi_arg (phi, build_zero_cst (m_limb_type), edge_true_false,
1623 : UNKNOWN_LOCATION);
1624 146 : add_phi_arg (phi, m_data[save_data_cnt], edge_false,
1625 : UNKNOWN_LOCATION);
1626 146 : add_phi_arg (phi, ext, edge_true_true, UNKNOWN_LOCATION);
1627 146 : if (!save_cast_conditional)
1628 : {
1629 136 : g = gimple_build_assign (m_data[save_data_cnt + 1], t4);
1630 136 : insert_before (g);
1631 : }
1632 : else
1633 10 : for (basic_block bb = gsi_bb (m_gsi);;)
1634 : {
1635 10 : edge e1 = single_succ_edge (bb);
1636 10 : edge e2 = find_edge (e1->dest, m_bb), e3;
1637 10 : tree t5 = (e2 ? m_data[save_data_cnt + 1]
1638 10 : : make_ssa_name (m_limb_type));
1639 10 : phi = create_phi_node (t5, e1->dest);
1640 10 : edge_iterator ei;
1641 30 : FOR_EACH_EDGE (e3, ei, e1->dest->preds)
1642 30 : add_phi_arg (phi, (e3 == e1 ? t4
1643 10 : : build_zero_cst (m_limb_type)),
1644 : e3, UNKNOWN_LOCATION);
1645 10 : if (e2)
1646 : break;
1647 0 : t4 = t5;
1648 0 : bb = e1->dest;
1649 0 : }
1650 : }
1651 530 : if (m_bitfld_load)
1652 : {
1653 8 : tree t4;
1654 8 : if (!save_first && !save_cast_conditional)
1655 2 : t4 = m_data[m_bitfld_load + 1];
1656 : else
1657 6 : t4 = make_ssa_name (m_limb_type);
1658 8 : phi = create_phi_node (t4, edge_true_false->dest);
1659 12 : add_phi_arg (phi,
1660 4 : edge_true_true ? bitfld : m_data[m_bitfld_load],
1661 : edge_true_false, UNKNOWN_LOCATION);
1662 8 : add_phi_arg (phi, m_data[m_bitfld_load + 2],
1663 : edge_false, UNKNOWN_LOCATION);
1664 8 : if (edge_true_true)
1665 4 : add_phi_arg (phi, m_data[m_bitfld_load], edge_true_true,
1666 : UNKNOWN_LOCATION);
1667 8 : if (save_cast_conditional)
1668 4 : for (basic_block bb = gsi_bb (m_gsi);;)
1669 : {
1670 4 : edge e1 = single_succ_edge (bb);
1671 4 : edge e2 = find_edge (e1->dest, m_bb), e3;
1672 4 : tree t5 = ((e2 && !save_first) ? m_data[m_bitfld_load + 1]
1673 4 : : make_ssa_name (m_limb_type));
1674 4 : phi = create_phi_node (t5, e1->dest);
1675 4 : edge_iterator ei;
1676 14 : FOR_EACH_EDGE (e3, ei, e1->dest->preds)
1677 16 : add_phi_arg (phi, (e3 == e1 ? t4
1678 6 : : build_zero_cst (m_limb_type)),
1679 : e3, UNKNOWN_LOCATION);
1680 4 : t4 = t5;
1681 4 : if (e2)
1682 : break;
1683 0 : bb = e1->dest;
1684 0 : }
1685 8 : m_data[m_bitfld_load] = t4;
1686 8 : m_data[m_bitfld_load + 2] = t4;
1687 8 : m_bitfld_load = 0;
1688 : }
1689 530 : m_cast_conditional = save_cast_conditional;
1690 530 : m_first = save_first;
1691 530 : return t;
1692 : }
1693 : else
1694 : {
1695 571 : unsigned tidx = tree_to_uhwi (idx);
1696 571 : if (bitint_big_endian)
1697 0 : tidx = lcnt - 1 - tidx;
1698 571 : if (tidx < low)
1699 : {
1700 152 : t = handle_operand (rhs1, (bitint_big_endian
1701 0 : ? size_int (high - 1 - tidx) : idx));
1702 152 : if (m_first)
1703 71 : m_data[save_data_cnt + 2]
1704 142 : = build_int_cst (integer_type_node, m_data_cnt);
1705 : }
1706 419 : else if (tidx < high)
1707 : {
1708 68 : t = handle_operand (rhs1, size_int (bitint_big_endian
1709 : ? high - 1 - low : low));
1710 68 : if (m_first)
1711 1 : m_data[save_data_cnt + 2]
1712 2 : = build_int_cst (integer_type_node, m_data_cnt);
1713 68 : if (!useless_type_conversion_p (m_limb_type, TREE_TYPE (t)))
1714 60 : t = add_cast (m_limb_type, t);
1715 68 : tree ext = NULL_TREE;
1716 68 : if (!TYPE_UNSIGNED (rhs_type) && m_upwards)
1717 : {
1718 44 : ext = add_cast (signed_type_for (m_limb_type), t);
1719 88 : tree lpm1 = build_int_cst (unsigned_type_node,
1720 44 : limb_prec - 1);
1721 44 : tree n = make_ssa_name (TREE_TYPE (ext));
1722 44 : g = gimple_build_assign (n, RSHIFT_EXPR, ext, lpm1);
1723 44 : insert_before (g);
1724 44 : ext = add_cast (m_limb_type, n);
1725 44 : m_data[save_data_cnt + 1] = ext;
1726 : }
1727 : }
1728 : else
1729 : {
1730 351 : if (TYPE_UNSIGNED (rhs_type) && m_first)
1731 : {
1732 0 : handle_operand (rhs1, (bitint_big_endian
1733 0 : ? size_int (high - 1)
1734 : : size_zero_node));
1735 0 : m_data[save_data_cnt + 2]
1736 0 : = build_int_cst (integer_type_node, m_data_cnt);
1737 : }
1738 : else
1739 351 : m_data_cnt = tree_to_uhwi (m_data[save_data_cnt + 2]);
1740 351 : if (TYPE_UNSIGNED (rhs_type))
1741 184 : t = build_zero_cst (m_limb_type);
1742 167 : else if (m_bb
1743 16 : && m_data[save_data_cnt]
1744 180 : && ((tidx & 1) == 0 || tidx != low + 1))
1745 : t = m_data[save_data_cnt];
1746 : else
1747 161 : t = m_data[save_data_cnt + 1];
1748 : }
1749 571 : tree type = limb_access_type (lhs_type, idx);
1750 571 : if (!useless_type_conversion_p (type, m_limb_type))
1751 292 : t = add_cast (type, t);
1752 571 : m_first = save_first;
1753 571 : return t;
1754 : }
1755 : }
1756 0 : else if (BITINT_TYPE_P (lhs_type)
1757 1686 : && bitint_precision_kind (lhs_type) >= bitint_prec_large
1758 3372 : && INTEGRAL_TYPE_P (rhs_type))
1759 : {
1760 : /* Add support for 3 or more limbs filled in from normal integral
1761 : type if this assert fails. If no target chooses limb mode smaller
1762 : than half of largest supported normal integral type, this will not
1763 : be needed. */
1764 1686 : gcc_assert (TYPE_PRECISION (rhs_type) <= 2 * limb_prec);
1765 1686 : tree r1 = NULL_TREE, r2 = NULL_TREE, rext = NULL_TREE;
1766 1686 : if (m_first)
1767 : {
1768 612 : gimple_stmt_iterator save_gsi = m_gsi;
1769 612 : m_gsi = m_init_gsi;
1770 612 : if (gsi_end_p (m_gsi))
1771 67 : m_gsi = gsi_after_labels (gsi_bb (m_gsi));
1772 : else
1773 545 : gsi_next (&m_gsi);
1774 393 : if (BITINT_TYPE_P (rhs_type)
1775 612 : && bitint_precision_kind (rhs_type) == bitint_prec_middle)
1776 : {
1777 63 : tree type = NULL_TREE;
1778 63 : rhs1 = maybe_cast_middle_bitint (&m_gsi, rhs1, type);
1779 63 : rhs_type = TREE_TYPE (rhs1);
1780 : }
1781 612 : r1 = rhs1;
1782 612 : if (!useless_type_conversion_p (m_limb_type, TREE_TYPE (rhs1)))
1783 544 : r1 = add_cast (m_limb_type, rhs1);
1784 612 : if (TYPE_PRECISION (rhs_type) > limb_prec)
1785 : {
1786 109 : g = gimple_build_assign (make_ssa_name (rhs_type),
1787 : RSHIFT_EXPR, rhs1,
1788 : build_int_cst (unsigned_type_node,
1789 109 : limb_prec));
1790 109 : insert_before (g);
1791 109 : r2 = add_cast (m_limb_type, gimple_assign_lhs (g));
1792 : }
1793 612 : if (TYPE_UNSIGNED (rhs_type))
1794 305 : rext = build_zero_cst (m_limb_type);
1795 : else
1796 : {
1797 307 : rext = add_cast (signed_type_for (m_limb_type), r2 ? r2 : r1);
1798 307 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (rext)),
1799 : RSHIFT_EXPR, rext,
1800 : build_int_cst (unsigned_type_node,
1801 307 : limb_prec - 1));
1802 307 : insert_before (g);
1803 307 : rext = add_cast (m_limb_type, gimple_assign_lhs (g));
1804 : }
1805 612 : m_init_gsi = m_gsi;
1806 612 : if (gsi_end_p (m_init_gsi))
1807 616 : m_init_gsi = gsi_last_bb (gsi_bb (m_init_gsi));
1808 : else
1809 304 : gsi_prev (&m_init_gsi);
1810 612 : m_gsi = save_gsi;
1811 : }
1812 1686 : tree t;
1813 1686 : if (m_upwards_2limb)
1814 : {
1815 803 : if (m_first)
1816 : {
1817 306 : tree out1, out2;
1818 306 : prepare_data_in_out (r1, idx, &out1, rext);
1819 306 : if (TYPE_PRECISION (rhs_type) > limb_prec)
1820 : {
1821 70 : prepare_data_in_out (r2, idx, &out2, rext);
1822 70 : m_data.pop ();
1823 70 : t = m_data.pop ();
1824 70 : m_data[m_data_cnt + 1] = t;
1825 : }
1826 : else
1827 236 : m_data[m_data_cnt + 1] = rext;
1828 306 : m_data.safe_push (rext);
1829 306 : t = m_data[m_data_cnt];
1830 : }
1831 497 : else if (!tree_fits_uhwi_p (idx))
1832 306 : t = m_data[m_data_cnt + 1];
1833 : else
1834 : {
1835 191 : tree type = limb_access_type (lhs_type, idx);
1836 191 : t = m_data[m_data_cnt + 2];
1837 191 : if (!useless_type_conversion_p (type, m_limb_type))
1838 162 : t = add_cast (type, t);
1839 : }
1840 803 : m_data_cnt += 3;
1841 803 : return t;
1842 : }
1843 883 : else if (m_first)
1844 : {
1845 306 : m_data.safe_push (r1);
1846 306 : m_data.safe_push (r2);
1847 306 : m_data.safe_push (rext);
1848 : }
1849 883 : unsigned lcnt = CEIL ((unsigned) TYPE_PRECISION (lhs_type), limb_prec);
1850 883 : if (tree_fits_uhwi_p (idx))
1851 : {
1852 839 : tree type = limb_access_type (lhs_type, idx);
1853 839 : if (bitint_big_endian
1854 839 : ? tree_to_uhwi (idx) == lcnt - 1 : integer_zerop (idx))
1855 278 : t = m_data[m_data_cnt];
1856 561 : else if (TYPE_PRECISION (rhs_type) > limb_prec
1857 561 : && (bitint_big_endian
1858 72 : ? tree_to_uhwi (idx) == lcnt - 2
1859 72 : : integer_onep (idx)))
1860 33 : t = m_data[m_data_cnt + 1];
1861 : else
1862 528 : t = m_data[m_data_cnt + 2];
1863 839 : if (!useless_type_conversion_p (type, m_limb_type))
1864 259 : t = add_cast (type, t);
1865 839 : m_data_cnt += 3;
1866 839 : return t;
1867 : }
1868 44 : g = gimple_build_cond (NE_EXPR, idx,
1869 : bitint_big_endian
1870 0 : ? size_int (lcnt - 1) : size_zero_node,
1871 : NULL_TREE, NULL_TREE);
1872 44 : edge e2, e3, e4 = NULL;
1873 44 : if_then (g, profile_probability::likely (), e2, e3);
1874 44 : if (m_data[m_data_cnt + 1])
1875 : {
1876 14 : g = gimple_build_cond (EQ_EXPR, idx,
1877 : bitint_big_endian
1878 0 : ? size_int (lcnt - 2) : size_one_node,
1879 : NULL_TREE, NULL_TREE);
1880 14 : insert_before (g);
1881 14 : edge e5 = split_block (gsi_bb (m_gsi), g);
1882 14 : e4 = make_edge (e5->src, e2->dest, EDGE_TRUE_VALUE);
1883 14 : e2 = find_edge (e5->dest, e2->dest);
1884 14 : e4->probability = profile_probability::unlikely ();
1885 14 : e5->flags = EDGE_FALSE_VALUE;
1886 14 : e5->probability = e4->probability.invert ();
1887 : }
1888 44 : m_gsi = gsi_after_labels (e2->dest);
1889 44 : t = make_ssa_name (m_limb_type);
1890 44 : gphi *phi = create_phi_node (t, e2->dest);
1891 44 : add_phi_arg (phi, m_data[m_data_cnt + 2], e2, UNKNOWN_LOCATION);
1892 44 : add_phi_arg (phi, m_data[m_data_cnt], e3, UNKNOWN_LOCATION);
1893 44 : if (e4)
1894 14 : add_phi_arg (phi, m_data[m_data_cnt + 1], e4, UNKNOWN_LOCATION);
1895 44 : m_data_cnt += 3;
1896 44 : return t;
1897 : }
1898 : return NULL_TREE;
1899 : }
1900 :
1901 : /* Helper function for handle_stmt method, handle a BIT_FIELD_REF. */
1902 :
1903 : tree
1904 23 : bitint_large_huge::handle_bit_field_ref (tree op, tree idx)
1905 : {
1906 23 : if (tree_fits_uhwi_p (idx))
1907 : {
1908 13 : if (m_first)
1909 4 : m_data.safe_push (NULL);
1910 13 : ++m_data_cnt;
1911 13 : unsigned HOST_WIDE_INT sz = tree_to_uhwi (TYPE_SIZE (m_limb_type));
1912 13 : unsigned i = tree_to_uhwi (idx);
1913 13 : if (bitint_big_endian)
1914 0 : i = CEIL (TYPE_PRECISION (TREE_TYPE (op)), limb_prec) - 1 - i;
1915 26 : tree bfr = build3 (BIT_FIELD_REF, m_limb_type,
1916 13 : TREE_OPERAND (op, 0),
1917 13 : TYPE_SIZE (m_limb_type),
1918 13 : size_binop (PLUS_EXPR, TREE_OPERAND (op, 2),
1919 : bitsize_int (i * sz)));
1920 13 : tree r = make_ssa_name (m_limb_type);
1921 13 : gimple *g = gimple_build_assign (r, bfr);
1922 13 : insert_before (g);
1923 13 : tree type = limb_access_type (TREE_TYPE (op), idx);
1924 13 : if (!useless_type_conversion_p (type, m_limb_type))
1925 0 : r = add_cast (type, r);
1926 : return r;
1927 : }
1928 10 : tree var;
1929 10 : if (m_first)
1930 : {
1931 5 : unsigned HOST_WIDE_INT sz = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (op)));
1932 5 : machine_mode mode;
1933 5 : tree type, bfr;
1934 5 : if (bitwise_mode_for_size (sz).exists (&mode)
1935 0 : && known_eq (GET_MODE_BITSIZE (mode), sz))
1936 0 : type = bitwise_type_for_mode (mode);
1937 : else
1938 : {
1939 5 : mode = VOIDmode;
1940 5 : type = TYPE_MAIN_VARIANT (TREE_TYPE (TREE_OPERAND (op, 0)));
1941 : }
1942 5 : if (TYPE_ALIGN (type) < TYPE_ALIGN (TREE_TYPE (op)))
1943 0 : type = build_aligned_type (type, TYPE_ALIGN (TREE_TYPE (op)));
1944 5 : var = create_tmp_var (type);
1945 5 : TREE_ADDRESSABLE (var) = 1;
1946 5 : gimple *g;
1947 5 : if (mode != VOIDmode)
1948 : {
1949 0 : bfr = build3 (BIT_FIELD_REF, type, TREE_OPERAND (op, 0),
1950 0 : TYPE_SIZE (type), TREE_OPERAND (op, 2));
1951 0 : g = gimple_build_assign (make_ssa_name (type),
1952 : BIT_FIELD_REF, bfr);
1953 0 : gimple_set_location (g, m_loc);
1954 0 : gsi_insert_after (&m_init_gsi, g, GSI_NEW_STMT);
1955 0 : bfr = gimple_assign_lhs (g);
1956 : }
1957 : else
1958 5 : bfr = TREE_OPERAND (op, 0);
1959 5 : g = gimple_build_assign (var, bfr);
1960 5 : gimple_set_location (g, m_loc);
1961 5 : gsi_insert_after (&m_init_gsi, g, GSI_NEW_STMT);
1962 5 : if (mode == VOIDmode)
1963 : {
1964 5 : unsigned HOST_WIDE_INT nelts
1965 5 : = CEIL (tree_to_uhwi (TYPE_SIZE (TREE_TYPE (op))), limb_prec);
1966 5 : tree atype = build_array_type_nelts (m_limb_type, nelts);
1967 5 : var = build2 (MEM_REF, atype, build_fold_addr_expr (var),
1968 : build_int_cst (build_pointer_type (type),
1969 5 : tree_to_uhwi (TREE_OPERAND (op, 2))
1970 5 : / BITS_PER_UNIT));
1971 : }
1972 5 : m_data.safe_push (var);
1973 : }
1974 : else
1975 5 : var = unshare_expr (m_data[m_data_cnt]);
1976 10 : ++m_data_cnt;
1977 10 : var = limb_access (TREE_TYPE (op), var, idx, false);
1978 10 : tree r = make_ssa_name (m_limb_type);
1979 10 : gimple *g = gimple_build_assign (r, var);
1980 10 : insert_before (g);
1981 10 : return r;
1982 : }
1983 :
1984 : /* Add a new EH edge from SRC to EH_EDGE->dest, where EH_EDGE
1985 : is an older EH edge, and except for virtual PHIs duplicate the
1986 : PHI argument from the EH_EDGE to the new EH edge. */
1987 :
1988 : static void
1989 20 : add_eh_edge (basic_block src, edge eh_edge)
1990 : {
1991 20 : edge e = make_edge (src, eh_edge->dest, EDGE_EH);
1992 20 : e->probability = profile_probability::very_unlikely ();
1993 20 : for (gphi_iterator gsi = gsi_start_phis (eh_edge->dest);
1994 27 : !gsi_end_p (gsi); gsi_next (&gsi))
1995 : {
1996 7 : gphi *phi = gsi.phi ();
1997 7 : tree lhs = gimple_phi_result (phi);
1998 14 : if (virtual_operand_p (lhs))
1999 4 : continue;
2000 3 : const phi_arg_d *arg = gimple_phi_arg (phi, eh_edge->dest_idx);
2001 3 : add_phi_arg (phi, arg->def, e, arg->locus);
2002 : }
2003 20 : }
2004 :
2005 : /* Helper function for handle_stmt method, handle a load from memory. */
2006 :
2007 : tree
2008 21430 : bitint_large_huge::handle_load (gimple *stmt, tree idx)
2009 : {
2010 21430 : tree rhs1 = gimple_assign_rhs1 (stmt);
2011 21430 : tree rhs_type = TREE_TYPE (rhs1);
2012 21430 : bool eh = stmt_ends_bb_p (stmt);
2013 21430 : bool load_bitfield_p = false;
2014 21430 : edge eh_edge = NULL;
2015 21430 : gimple *g;
2016 :
2017 21430 : if (TREE_CODE (rhs1) == BIT_FIELD_REF
2018 21430 : && integer_zerop (TREE_OPERAND (rhs1, 2)))
2019 2 : rhs1 = TREE_OPERAND (rhs1, 0);
2020 :
2021 21430 : if (eh)
2022 : {
2023 10 : edge_iterator ei;
2024 10 : basic_block bb = gimple_bb (stmt);
2025 :
2026 10 : FOR_EACH_EDGE (eh_edge, ei, bb->succs)
2027 10 : if (eh_edge->flags & EDGE_EH)
2028 : break;
2029 : }
2030 :
2031 21430 : if (TREE_CODE (rhs1) == COMPONENT_REF
2032 21430 : && DECL_BIT_FIELD_TYPE (TREE_OPERAND (rhs1, 1)))
2033 : {
2034 1267 : tree fld = TREE_OPERAND (rhs1, 1);
2035 : /* For little-endian, we can allow as inputs bit-fields
2036 : which start at a limb boundary. */
2037 1267 : gcc_assert (tree_fits_uhwi_p (DECL_FIELD_BIT_OFFSET (fld)));
2038 1267 : if (!bitint_big_endian
2039 1267 : && DECL_OFFSET_ALIGN (fld) >= TYPE_ALIGN (TREE_TYPE (rhs1))
2040 2534 : && (tree_to_uhwi (DECL_FIELD_BIT_OFFSET (fld)) % limb_prec) == 0)
2041 : {
2042 739 : load_bitfield_p = true;
2043 751 : goto normal_load;
2044 : }
2045 : /* Even if DECL_FIELD_BIT_OFFSET (fld) is a multiple of BITS_PER_UNIT,
2046 : handle it normally for now. */
2047 528 : if (!bitint_big_endian
2048 528 : && (tree_to_uhwi (DECL_FIELD_BIT_OFFSET (fld)) % BITS_PER_UNIT) == 0)
2049 : {
2050 12 : load_bitfield_p = true;
2051 12 : goto normal_load;
2052 : }
2053 516 : tree repr = DECL_BIT_FIELD_REPRESENTATIVE (fld);
2054 516 : poly_int64 bitoffset;
2055 516 : poly_uint64 field_offset, repr_offset;
2056 516 : bool var_field_off = false;
2057 516 : if (poly_int_tree_p (DECL_FIELD_OFFSET (fld), &field_offset)
2058 1032 : && poly_int_tree_p (DECL_FIELD_OFFSET (repr), &repr_offset))
2059 516 : bitoffset = (field_offset - repr_offset) * BITS_PER_UNIT;
2060 : else
2061 : {
2062 : bitoffset = 0;
2063 : var_field_off = true;
2064 : }
2065 516 : bitoffset += (tree_to_uhwi (DECL_FIELD_BIT_OFFSET (fld))
2066 516 : - tree_to_uhwi (DECL_FIELD_BIT_OFFSET (repr)));
2067 1032 : tree nrhs1 = build3 (COMPONENT_REF, TREE_TYPE (repr),
2068 516 : TREE_OPERAND (rhs1, 0), repr,
2069 0 : var_field_off ? TREE_OPERAND (rhs1, 2) : NULL_TREE);
2070 516 : HOST_WIDE_INT bo = bitoffset.to_constant ();
2071 516 : unsigned bo_idx = (unsigned HOST_WIDE_INT) bo / limb_prec;
2072 516 : unsigned bo_bit = (unsigned HOST_WIDE_INT) bo % limb_prec;
2073 516 : unsigned bo_last = 0;
2074 516 : unsigned bo_shift = bo_bit;
2075 516 : unsigned nelts = CEIL (TYPE_PRECISION (rhs_type), limb_prec);
2076 516 : if (bitint_big_endian)
2077 : {
2078 0 : bo_last = CEIL (TYPE_PRECISION (rhs_type) + bo_bit, limb_prec) - 1;
2079 0 : bo_shift = (TYPE_PRECISION (rhs_type) + bo_bit) % limb_prec;
2080 0 : if (bo_shift)
2081 0 : bo_shift = limb_prec - bo_shift;
2082 : }
2083 516 : if (m_first)
2084 : {
2085 153 : if (m_upwards && bo_shift)
2086 : {
2087 150 : gimple_stmt_iterator save_gsi = m_gsi;
2088 150 : m_gsi = m_init_gsi;
2089 150 : if (gsi_end_p (m_gsi))
2090 61 : m_gsi = gsi_after_labels (gsi_bb (m_gsi));
2091 : else
2092 89 : gsi_next (&m_gsi);
2093 150 : tree t = limb_access (NULL_TREE, nrhs1,
2094 150 : size_int (bo_idx + bo_last), true);
2095 150 : tree iv = make_ssa_name (m_limb_type);
2096 150 : g = gimple_build_assign (iv, t);
2097 150 : insert_before (g);
2098 150 : if (eh)
2099 : {
2100 2 : maybe_duplicate_eh_stmt (g, stmt);
2101 2 : if (eh_edge)
2102 : {
2103 2 : edge e = split_block (gsi_bb (m_gsi), g);
2104 2 : add_eh_edge (e->src, eh_edge);
2105 2 : m_gsi = gsi_after_labels (e->dest);
2106 2 : if (gsi_bb (save_gsi) == e->src)
2107 : {
2108 1 : if (gsi_end_p (save_gsi))
2109 0 : save_gsi = gsi_end_bb (e->dest);
2110 : else
2111 1 : save_gsi = gsi_for_stmt (gsi_stmt (save_gsi));
2112 : }
2113 2 : if (m_preheader_bb == e->src)
2114 1 : m_preheader_bb = e->dest;
2115 : }
2116 : }
2117 150 : m_init_gsi = m_gsi;
2118 150 : if (gsi_end_p (m_init_gsi))
2119 246 : m_init_gsi = gsi_last_bb (gsi_bb (m_init_gsi));
2120 : else
2121 28 : gsi_prev (&m_init_gsi);
2122 150 : m_gsi = save_gsi;
2123 150 : tree out;
2124 150 : prepare_data_in_out (iv, idx, &out);
2125 150 : out = m_data[m_data_cnt];
2126 150 : m_data.safe_push (out);
2127 150 : }
2128 : else
2129 : {
2130 3 : m_data.safe_push (NULL_TREE);
2131 3 : m_data.safe_push (NULL_TREE);
2132 3 : m_data.safe_push (NULL_TREE);
2133 : }
2134 : }
2135 :
2136 516 : tree nidx0 = NULL_TREE, nidx1 = NULL_TREE;
2137 516 : tree iv = m_data[m_data_cnt];
2138 516 : if (m_cast_conditional && iv)
2139 : {
2140 12 : gcc_assert (!m_bitfld_load);
2141 12 : m_bitfld_load = m_data_cnt;
2142 : }
2143 516 : if (tree_fits_uhwi_p (idx))
2144 : {
2145 290 : unsigned prec = TYPE_PRECISION (rhs_type);
2146 290 : unsigned HOST_WIDE_INT i = tree_to_uhwi (idx);
2147 290 : if (bitint_big_endian)
2148 0 : i = nelts - 1 - i;
2149 290 : gcc_assert (i * limb_prec < prec);
2150 290 : if (bo_shift)
2151 290 : nidx1 = size_int (bo_idx + (bitint_big_endian
2152 : ? bo_last - i - 1 : i + 1));
2153 290 : if ((i + 1) * limb_prec > prec)
2154 : {
2155 109 : prec %= limb_prec;
2156 109 : if (prec + bo_bit <= (unsigned) limb_prec)
2157 290 : nidx1 = NULL_TREE;
2158 : }
2159 290 : if (!iv)
2160 4 : nidx0 = size_int (bo_idx + (bitint_big_endian ? bo_last - i : i));
2161 : }
2162 : else
2163 : {
2164 226 : HOST_WIDE_INT adj = bo_idx;
2165 226 : if (bitint_big_endian)
2166 0 : adj += (HOST_WIDE_INT) bo_last + 1 - nelts;
2167 226 : if (!iv)
2168 : {
2169 4 : if (adj == 0)
2170 : nidx0 = idx;
2171 : else
2172 : {
2173 0 : nidx0 = make_ssa_name (sizetype);
2174 0 : g = gimple_build_assign (nidx0, PLUS_EXPR, idx,
2175 0 : size_int (adj));
2176 0 : insert_before (g);
2177 : }
2178 : }
2179 226 : if (bo_shift)
2180 : {
2181 226 : if (bitint_big_endian && adj == 1)
2182 : nidx1 = idx;
2183 : else
2184 : {
2185 226 : nidx1 = make_ssa_name (sizetype);
2186 226 : g = gimple_build_assign (nidx1, PLUS_EXPR, idx,
2187 452 : size_int (adj + (bitint_big_endian
2188 : ? -1 : 1)));
2189 226 : insert_before (g);
2190 : }
2191 : }
2192 : }
2193 :
2194 746 : tree iv2 = NULL_TREE;
2195 230 : if (nidx0)
2196 : {
2197 8 : tree t = limb_access (NULL_TREE, nrhs1, nidx0, true);
2198 8 : iv = make_ssa_name (m_limb_type);
2199 8 : g = gimple_build_assign (iv, t);
2200 8 : insert_before (g);
2201 8 : if (eh)
2202 : {
2203 0 : maybe_duplicate_eh_stmt (g, stmt);
2204 0 : if (eh_edge)
2205 : {
2206 0 : edge e = split_block (gsi_bb (m_gsi), g);
2207 0 : m_gsi = gsi_after_labels (e->dest);
2208 0 : add_eh_edge (e->src, eh_edge);
2209 : }
2210 : }
2211 : }
2212 516 : if (nidx1)
2213 : {
2214 427 : bool conditional = m_var_msb && !tree_fits_uhwi_p (idx);
2215 427 : unsigned prec = TYPE_PRECISION (rhs_type);
2216 427 : if (conditional)
2217 : {
2218 3 : if ((prec % limb_prec) == 0
2219 3 : || ((prec % limb_prec) + bo_bit > (unsigned) limb_prec))
2220 424 : conditional = false;
2221 : }
2222 427 : edge edge_true = NULL, edge_false = NULL;
2223 427 : if (conditional)
2224 : {
2225 6 : g = gimple_build_cond (NE_EXPR, idx,
2226 : bitint_big_endian
2227 : ? size_zero_node
2228 3 : : size_int (prec / limb_prec),
2229 : NULL_TREE, NULL_TREE);
2230 3 : if_then (g, profile_probability::likely (),
2231 : edge_true, edge_false);
2232 : }
2233 427 : tree t = limb_access (NULL_TREE, nrhs1, nidx1, true);
2234 427 : if (m_upwards_2limb
2235 325 : && !m_first
2236 214 : && !m_bitfld_load
2237 208 : && !tree_fits_uhwi_p (idx))
2238 107 : iv2 = m_data[m_data_cnt + 1];
2239 : else
2240 320 : iv2 = make_ssa_name (m_limb_type);
2241 427 : g = gimple_build_assign (iv2, t);
2242 427 : insert_before (g);
2243 427 : if (eh)
2244 : {
2245 5 : maybe_duplicate_eh_stmt (g, stmt);
2246 5 : if (eh_edge)
2247 : {
2248 5 : edge e = split_block (gsi_bb (m_gsi), g);
2249 5 : m_gsi = gsi_after_labels (e->dest);
2250 5 : add_eh_edge (e->src, eh_edge);
2251 : }
2252 : }
2253 427 : if (conditional)
2254 : {
2255 3 : tree iv3 = make_ssa_name (m_limb_type);
2256 3 : if (eh)
2257 0 : edge_true = find_edge (gsi_bb (m_gsi), edge_false->dest);
2258 3 : gphi *phi = create_phi_node (iv3, edge_true->dest);
2259 3 : add_phi_arg (phi, iv2, edge_true, UNKNOWN_LOCATION);
2260 3 : add_phi_arg (phi, build_zero_cst (m_limb_type),
2261 : edge_false, UNKNOWN_LOCATION);
2262 3 : m_gsi = gsi_after_labels (edge_true->dest);
2263 3 : iv2 = iv3;
2264 : }
2265 : }
2266 516 : if (bo_shift)
2267 : {
2268 516 : g = gimple_build_assign (make_ssa_name (m_limb_type), RSHIFT_EXPR,
2269 : iv, build_int_cst (unsigned_type_node,
2270 516 : bo_shift));
2271 516 : insert_before (g);
2272 516 : iv = gimple_assign_lhs (g);
2273 : }
2274 516 : if (iv2)
2275 : {
2276 427 : g = gimple_build_assign (make_ssa_name (m_limb_type), LSHIFT_EXPR,
2277 : iv2, build_int_cst (unsigned_type_node,
2278 427 : limb_prec - bo_shift));
2279 427 : insert_before (g);
2280 427 : g = gimple_build_assign (make_ssa_name (m_limb_type), BIT_IOR_EXPR,
2281 : gimple_assign_lhs (g), iv);
2282 427 : insert_before (g);
2283 427 : iv = gimple_assign_lhs (g);
2284 427 : if (m_data[m_data_cnt])
2285 421 : m_data[m_data_cnt] = iv2;
2286 : }
2287 516 : if (tree_fits_uhwi_p (idx))
2288 : {
2289 290 : tree atype = limb_access_type (rhs_type, idx);
2290 290 : if (!useless_type_conversion_p (atype, TREE_TYPE (iv)))
2291 109 : iv = add_cast (atype, iv);
2292 : }
2293 516 : m_data_cnt += 3;
2294 516 : return iv;
2295 : }
2296 :
2297 20914 : normal_load:
2298 : /* Use write_p = true for loads with EH edges to make
2299 : sure limb_access doesn't add a cast as separate
2300 : statement after it. */
2301 20914 : rhs1 = limb_access (rhs_type, rhs1, idx, eh, !load_bitfield_p);
2302 20914 : tree ret = make_ssa_name (TREE_TYPE (rhs1));
2303 20914 : g = gimple_build_assign (ret, rhs1);
2304 20914 : insert_before (g);
2305 20914 : if (eh)
2306 : {
2307 3 : maybe_duplicate_eh_stmt (g, stmt);
2308 3 : if (eh_edge)
2309 : {
2310 3 : edge e = split_block (gsi_bb (m_gsi), g);
2311 3 : m_gsi = gsi_after_labels (e->dest);
2312 3 : add_eh_edge (e->src, eh_edge);
2313 : }
2314 3 : if (tree_fits_uhwi_p (idx))
2315 : {
2316 1 : tree atype = limb_access_type (rhs_type, idx);
2317 1 : if (!useless_type_conversion_p (atype, TREE_TYPE (rhs1)))
2318 1 : ret = add_cast (atype, ret);
2319 : }
2320 : }
2321 : return ret;
2322 : }
2323 :
2324 : /* Return a limb IDX from a mergeable statement STMT. */
2325 :
2326 : tree
2327 39504 : bitint_large_huge::handle_stmt (gimple *stmt, tree idx)
2328 : {
2329 39504 : tree lhs, rhs1, rhs2 = NULL_TREE;
2330 39504 : gimple *g;
2331 39504 : switch (gimple_code (stmt))
2332 : {
2333 39504 : case GIMPLE_ASSIGN:
2334 39504 : if (gimple_assign_load_p (stmt))
2335 21430 : return handle_load (stmt, idx);
2336 18074 : switch (gimple_assign_rhs_code (stmt))
2337 : {
2338 1004 : case BIT_AND_EXPR:
2339 1004 : case BIT_IOR_EXPR:
2340 1004 : case BIT_XOR_EXPR:
2341 1004 : rhs2 = handle_operand (gimple_assign_rhs2 (stmt), idx);
2342 : /* FALLTHRU */
2343 1424 : case BIT_NOT_EXPR:
2344 1424 : rhs1 = handle_operand (gimple_assign_rhs1 (stmt), idx);
2345 1424 : lhs = make_ssa_name (TREE_TYPE (rhs1));
2346 1424 : g = gimple_build_assign (lhs, gimple_assign_rhs_code (stmt),
2347 : rhs1, rhs2);
2348 1424 : insert_before (g);
2349 1424 : return lhs;
2350 4161 : case PLUS_EXPR:
2351 4161 : case MINUS_EXPR:
2352 4161 : rhs1 = handle_operand (gimple_assign_rhs1 (stmt), idx);
2353 4161 : rhs2 = handle_operand (gimple_assign_rhs2 (stmt), idx);
2354 4161 : return handle_plus_minus (gimple_assign_rhs_code (stmt),
2355 4161 : rhs1, rhs2, idx);
2356 163 : case NEGATE_EXPR:
2357 163 : rhs2 = handle_operand (gimple_assign_rhs1 (stmt), idx);
2358 163 : rhs1 = build_zero_cst (TREE_TYPE (rhs2));
2359 163 : return handle_plus_minus (MINUS_EXPR, rhs1, rhs2, idx);
2360 152 : case LSHIFT_EXPR:
2361 152 : return handle_lshift (handle_operand (gimple_assign_rhs1 (stmt),
2362 : idx),
2363 152 : gimple_assign_rhs2 (stmt), idx);
2364 5120 : case SSA_NAME:
2365 5120 : case PAREN_EXPR:
2366 5120 : case INTEGER_CST:
2367 5120 : return handle_operand (gimple_assign_rhs1 (stmt), idx);
2368 7023 : CASE_CONVERT:
2369 7023 : return handle_cast (TREE_TYPE (gimple_assign_lhs (stmt)),
2370 7023 : gimple_assign_rhs1 (stmt), idx);
2371 8 : case VIEW_CONVERT_EXPR:
2372 8 : return handle_cast (TREE_TYPE (gimple_assign_lhs (stmt)),
2373 8 : TREE_OPERAND (gimple_assign_rhs1 (stmt), 0),
2374 8 : idx);
2375 23 : case BIT_FIELD_REF:
2376 23 : return handle_bit_field_ref (gimple_assign_rhs1 (stmt), idx);
2377 : default:
2378 : break;
2379 : }
2380 : break;
2381 : default:
2382 : break;
2383 : }
2384 0 : gcc_unreachable ();
2385 : }
2386 :
2387 : /* Return minimum precision of OP at STMT.
2388 : Positive value is minimum precision above which all bits
2389 : are zero, negative means all bits above negation of the
2390 : value are copies of the sign bit. */
2391 :
2392 : static int
2393 8450 : range_to_prec (tree op, gimple *stmt)
2394 : {
2395 8450 : int_range_max r;
2396 8450 : wide_int w;
2397 8450 : tree type = TREE_TYPE (op);
2398 8450 : unsigned int prec = TYPE_PRECISION (type);
2399 :
2400 8450 : if (!optimize
2401 7518 : || !get_range_query (cfun)->range_of_expr (r, op, stmt)
2402 12209 : || r.undefined_p ())
2403 : {
2404 4692 : if (TYPE_UNSIGNED (type))
2405 1902 : return prec;
2406 : else
2407 2790 : return MIN ((int) -prec, -2);
2408 : }
2409 :
2410 3758 : if (!TYPE_UNSIGNED (TREE_TYPE (op)))
2411 : {
2412 2343 : w = r.lower_bound ();
2413 2343 : if (wi::neg_p (w))
2414 : {
2415 1938 : int min_prec1 = wi::min_precision (w, SIGNED);
2416 1938 : w = r.upper_bound ();
2417 1938 : int min_prec2 = wi::min_precision (w, SIGNED);
2418 1938 : int min_prec = MAX (min_prec1, min_prec2);
2419 1938 : return MIN (-min_prec, -2);
2420 : }
2421 : }
2422 :
2423 1820 : w = r.upper_bound ();
2424 1820 : int min_prec = wi::min_precision (w, UNSIGNED);
2425 1820 : return MAX (min_prec, 1);
2426 8450 : }
2427 :
2428 : /* Return address of the first limb of OP and write into *PREC
2429 : its precision. If positive, the operand is zero extended
2430 : from that precision, if it is negative, the operand is sign-extended
2431 : from -*PREC. If PREC_STORED is NULL, it is the toplevel call,
2432 : otherwise *PREC_STORED is prec from the innermost call without
2433 : range optimizations (0 for uninitialized SSA_NAME). */
2434 :
2435 : tree
2436 3868 : bitint_large_huge::handle_operand_addr (tree op, gimple *stmt,
2437 : int *prec_stored, int *prec)
2438 : {
2439 3868 : wide_int w;
2440 3868 : location_t loc_save = m_loc;
2441 3868 : tree ret = NULL_TREE;
2442 3868 : int precs = 0;
2443 3903 : if ((!BITINT_TYPE_P (TREE_TYPE (op))
2444 3839 : || bitint_precision_kind (TREE_TYPE (op)) < bitint_prec_large)
2445 3992 : && TREE_CODE (op) != INTEGER_CST)
2446 : {
2447 126 : do_int:
2448 126 : *prec = range_to_prec (op, stmt);
2449 126 : bitint_prec_kind kind = bitint_prec_small;
2450 126 : gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (op)));
2451 126 : if (BITINT_TYPE_P (TREE_TYPE (op)))
2452 106 : kind = bitint_precision_kind (TREE_TYPE (op));
2453 106 : if (kind == bitint_prec_middle)
2454 : {
2455 12 : tree type = NULL_TREE;
2456 12 : op = maybe_cast_middle_bitint (&m_gsi, op, type);
2457 : }
2458 126 : tree op_type = TREE_TYPE (op);
2459 126 : unsigned HOST_WIDE_INT nelts
2460 126 : = CEIL (TYPE_PRECISION (op_type), limb_prec);
2461 : /* Add support for 3 or more limbs filled in from normal
2462 : integral type if this assert fails. If no target chooses
2463 : limb mode smaller than half of largest supported normal
2464 : integral type, this will not be needed. */
2465 126 : gcc_assert (nelts <= 2);
2466 126 : precs = (TYPE_UNSIGNED (op_type)
2467 126 : ? TYPE_PRECISION (op_type) : -TYPE_PRECISION (op_type));
2468 126 : if (*prec <= limb_prec && *prec >= -limb_prec)
2469 : {
2470 113 : nelts = 1;
2471 113 : if (TYPE_UNSIGNED (op_type))
2472 : {
2473 26 : if (precs > limb_prec)
2474 126 : precs = limb_prec;
2475 : }
2476 87 : else if (precs < -limb_prec)
2477 126 : precs = -limb_prec;
2478 : }
2479 126 : if (prec_stored)
2480 0 : *prec_stored = precs;
2481 126 : tree atype = build_array_type_nelts (m_limb_type, nelts);
2482 126 : tree var = create_tmp_var (atype);
2483 126 : tree t1 = op;
2484 126 : if (!useless_type_conversion_p (m_limb_type, op_type))
2485 126 : t1 = add_cast (m_limb_type, t1);
2486 126 : tree v = build4 (ARRAY_REF, m_limb_type, var,
2487 0 : bitint_big_endian && nelts > 1
2488 : ? size_one_node : size_zero_node,
2489 : NULL_TREE, NULL_TREE);
2490 126 : gimple *g = gimple_build_assign (v, t1);
2491 126 : insert_before (g);
2492 126 : if (nelts > 1)
2493 : {
2494 13 : tree lp = build_int_cst (unsigned_type_node, limb_prec);
2495 13 : g = gimple_build_assign (make_ssa_name (op_type),
2496 : RSHIFT_EXPR, op, lp);
2497 13 : insert_before (g);
2498 13 : tree t2 = gimple_assign_lhs (g);
2499 13 : t2 = add_cast (m_limb_type, t2);
2500 13 : v = build4 (ARRAY_REF, m_limb_type, var,
2501 : bitint_big_endian ? size_zero_node : size_one_node,
2502 : NULL_TREE, NULL_TREE);
2503 13 : g = gimple_build_assign (v, t2);
2504 13 : insert_before (g);
2505 : }
2506 126 : ret = build_fold_addr_expr (var);
2507 126 : if (!stmt_ends_bb_p (gsi_stmt (m_gsi)))
2508 : {
2509 125 : tree clobber = build_clobber (atype, CLOBBER_STORAGE_END);
2510 125 : g = gimple_build_assign (var, clobber);
2511 125 : gsi_insert_after (&m_gsi, g, GSI_SAME_STMT);
2512 : }
2513 126 : m_loc = loc_save;
2514 126 : goto do_ret;
2515 : }
2516 3792 : switch (TREE_CODE (op))
2517 : {
2518 2926 : case SSA_NAME:
2519 2926 : if (m_names == NULL
2520 2926 : || !bitmap_bit_p (m_names, SSA_NAME_VERSION (op)))
2521 : {
2522 104 : gimple *g = SSA_NAME_DEF_STMT (op);
2523 104 : m_loc = gimple_location (g);
2524 104 : if (gimple_assign_load_p (g))
2525 : {
2526 36 : *prec = range_to_prec (op, NULL);
2527 36 : precs = (TYPE_UNSIGNED (TREE_TYPE (op))
2528 36 : ? TYPE_PRECISION (TREE_TYPE (op))
2529 25 : : -TYPE_PRECISION (TREE_TYPE (op)));
2530 36 : if (prec_stored)
2531 6 : *prec_stored = precs;
2532 36 : ret = build_fold_addr_expr (gimple_assign_rhs1 (g));
2533 36 : ret = force_gimple_operand_gsi (&m_gsi, ret, true,
2534 : NULL_TREE, true, GSI_SAME_STMT);
2535 : }
2536 68 : else if (gimple_code (g) == GIMPLE_NOP)
2537 : {
2538 2 : *prec = TYPE_UNSIGNED (TREE_TYPE (op)) ? limb_prec : -limb_prec;
2539 2 : precs = *prec;
2540 2 : if (prec_stored)
2541 1 : *prec_stored = 0;
2542 2 : tree var = create_tmp_var (m_limb_type);
2543 2 : TREE_ADDRESSABLE (var) = 1;
2544 2 : ret = build_fold_addr_expr (var);
2545 2 : if (!stmt_ends_bb_p (gsi_stmt (m_gsi)))
2546 : {
2547 2 : tree clobber = build_clobber (m_limb_type,
2548 : CLOBBER_STORAGE_END);
2549 2 : g = gimple_build_assign (var, clobber);
2550 2 : gsi_insert_after (&m_gsi, g, GSI_SAME_STMT);
2551 : }
2552 : }
2553 : else
2554 : {
2555 66 : gcc_assert (gimple_assign_cast_p (g));
2556 66 : tree rhs1 = gimple_assign_rhs1 (g);
2557 66 : bitint_prec_kind kind = bitint_prec_small;
2558 66 : if (TREE_CODE (rhs1) == VIEW_CONVERT_EXPR)
2559 1 : rhs1 = TREE_OPERAND (rhs1, 0);
2560 66 : gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (rhs1)));
2561 66 : if (BITINT_TYPE_P (TREE_TYPE (rhs1)))
2562 51 : kind = bitint_precision_kind (TREE_TYPE (rhs1));
2563 51 : if (kind >= bitint_prec_large)
2564 : {
2565 16 : tree lhs_type = TREE_TYPE (op);
2566 16 : tree rhs_type = TREE_TYPE (rhs1);
2567 16 : int prec_stored_val = 0;
2568 16 : ret = handle_operand_addr (rhs1, g, &prec_stored_val, prec);
2569 16 : precs = prec_stored_val;
2570 16 : if (prec_stored)
2571 0 : *prec_stored = prec_stored_val;
2572 16 : if (precs == 0)
2573 : {
2574 1 : gcc_assert (*prec == limb_prec || *prec == -limb_prec);
2575 : precs = *prec;
2576 : }
2577 16 : if (TYPE_PRECISION (lhs_type) > TYPE_PRECISION (rhs_type))
2578 : {
2579 4 : if (TYPE_UNSIGNED (lhs_type)
2580 4 : && !TYPE_UNSIGNED (rhs_type))
2581 1 : gcc_assert (*prec >= 0 || prec_stored == NULL);
2582 : }
2583 : else
2584 : {
2585 12 : if (prec_stored_val == 0)
2586 : /* Non-widening cast of uninitialized value. */;
2587 11 : else if (*prec > 0 && *prec < TYPE_PRECISION (lhs_type))
2588 : ;
2589 11 : else if (TYPE_UNSIGNED (lhs_type))
2590 : {
2591 8 : gcc_assert (*prec > 0
2592 : || prec_stored_val > 0
2593 : || (-prec_stored_val
2594 : >= TYPE_PRECISION (lhs_type)));
2595 8 : *prec = TYPE_PRECISION (lhs_type);
2596 : }
2597 3 : else if (*prec < 0 && -*prec < TYPE_PRECISION (lhs_type))
2598 : ;
2599 : else
2600 3 : *prec = -TYPE_PRECISION (lhs_type);
2601 : }
2602 : }
2603 : else
2604 : {
2605 50 : op = rhs1;
2606 50 : stmt = g;
2607 50 : goto do_int;
2608 : }
2609 : }
2610 54 : m_loc = loc_save;
2611 54 : goto do_ret;
2612 : }
2613 : else
2614 : {
2615 2822 : int p = var_to_partition (m_map, op);
2616 2822 : gcc_assert (m_vars[p] != NULL_TREE);
2617 2822 : *prec = range_to_prec (op, stmt);
2618 2822 : precs = (TYPE_UNSIGNED (TREE_TYPE (op))
2619 2822 : ? TYPE_PRECISION (TREE_TYPE (op))
2620 1671 : : -TYPE_PRECISION (TREE_TYPE (op)));
2621 2822 : if (prec_stored)
2622 9 : *prec_stored = precs;
2623 2822 : ret = build_fold_addr_expr (m_vars[p]);
2624 2822 : goto do_ret;
2625 : }
2626 866 : case INTEGER_CST:
2627 866 : unsigned int min_prec, mp;
2628 866 : tree type;
2629 866 : w = wi::to_wide (op);
2630 866 : if (tree_int_cst_sgn (op) >= 0)
2631 : {
2632 651 : min_prec = wi::min_precision (w, UNSIGNED);
2633 651 : *prec = MAX (min_prec, 1);
2634 : }
2635 : else
2636 : {
2637 215 : min_prec = wi::min_precision (w, SIGNED);
2638 215 : *prec = MIN ((int) -min_prec, -2);
2639 : }
2640 866 : mp = CEIL (min_prec, limb_prec) * limb_prec;
2641 866 : if (mp == 0)
2642 : mp = 1;
2643 866 : if (mp >= (unsigned) TYPE_PRECISION (TREE_TYPE (op))
2644 866 : && (BITINT_TYPE_P (TREE_TYPE (op))
2645 24 : || TYPE_PRECISION (TREE_TYPE (op)) <= limb_prec))
2646 316 : type = TREE_TYPE (op);
2647 : else
2648 550 : type = build_bitint_type (mp, 1);
2649 20 : if (!BITINT_TYPE_P (type)
2650 866 : || bitint_precision_kind (type) == bitint_prec_small)
2651 : {
2652 587 : if (TYPE_PRECISION (type) <= limb_prec)
2653 587 : type = m_limb_type;
2654 : else
2655 : {
2656 0 : while (bitint_precision_kind (mp) == bitint_prec_small)
2657 0 : mp += limb_prec;
2658 : /* This case is for targets which e.g. have 64-bit
2659 : limb but categorize up to 128-bits _BitInts as
2660 : small. We could use type of m_limb_type[2] and
2661 : similar instead to save space. */
2662 0 : type = build_bitint_type (mp, 1);
2663 : }
2664 : }
2665 866 : if (tree_int_cst_sgn (op) >= 0)
2666 651 : precs = MAX (TYPE_PRECISION (type), 1);
2667 : else
2668 215 : precs = MIN ((int) -TYPE_PRECISION (type), -2);
2669 866 : if (prec_stored)
2670 0 : *prec_stored = precs;
2671 866 : op = tree_output_constant_def (fold_convert (type, op));
2672 866 : ret = build_fold_addr_expr (op);
2673 866 : goto do_ret;
2674 0 : default:
2675 0 : gcc_unreachable ();
2676 : }
2677 3868 : do_ret:
2678 3868 : if (bitint_big_endian && prec_stored == NULL)
2679 : {
2680 0 : int p1 = *prec < 0 ? -*prec : *prec;
2681 0 : int p2 = precs < 0 ? -precs : precs;
2682 0 : int c1 = CEIL (p1, limb_prec);
2683 0 : int c2 = CEIL (p2, limb_prec);
2684 0 : gcc_assert (c1 <= c2);
2685 0 : if (c1 != c2)
2686 : {
2687 0 : gimple *g
2688 0 : = gimple_build_assign (make_ssa_name (TREE_TYPE (ret)),
2689 : POINTER_PLUS_EXPR, ret,
2690 0 : size_int ((c2 - c1) * m_limb_size));
2691 0 : insert_before (g);
2692 0 : ret = gimple_assign_lhs (g);
2693 : }
2694 : }
2695 3868 : return ret;
2696 3868 : }
2697 :
2698 : /* Helper function, create a loop before the current location,
2699 : start with sizetype INIT value from the preheader edge. Return
2700 : a PHI result and set *IDX_NEXT to SSA_NAME it creates and uses
2701 : from the latch edge. */
2702 :
2703 : tree
2704 14962 : bitint_large_huge::create_loop (tree init, tree *idx_next)
2705 : {
2706 14962 : if (!gsi_end_p (m_gsi))
2707 16000 : gsi_prev (&m_gsi);
2708 : else
2709 4528 : m_gsi = gsi_last_bb (gsi_bb (m_gsi));
2710 14962 : edge e1 = split_block (gsi_bb (m_gsi), gsi_stmt (m_gsi));
2711 14962 : edge e2 = split_block (e1->dest, (gimple *) NULL);
2712 14962 : edge e3 = make_edge (e1->dest, e1->dest, EDGE_TRUE_VALUE);
2713 14962 : e3->probability = profile_probability::very_unlikely ();
2714 14962 : e2->flags = EDGE_FALSE_VALUE;
2715 14962 : e2->probability = e3->probability.invert ();
2716 14962 : tree idx = make_ssa_name (sizetype);
2717 14962 : gphi *phi = create_phi_node (idx, e1->dest);
2718 14962 : add_phi_arg (phi, init, e1, UNKNOWN_LOCATION);
2719 14962 : *idx_next = make_ssa_name (sizetype);
2720 14962 : add_phi_arg (phi, *idx_next, e3, UNKNOWN_LOCATION);
2721 14962 : m_gsi = gsi_after_labels (e1->dest);
2722 14962 : m_bb = e1->dest;
2723 14962 : m_preheader_bb = e1->src;
2724 14962 : class loop *loop = alloc_loop ();
2725 14962 : loop->header = e1->dest;
2726 14962 : add_loop (loop, e1->src->loop_father);
2727 14962 : return idx;
2728 : }
2729 :
2730 : /* Lower large/huge _BitInt statement mergeable or similar STMT which can be
2731 : lowered using iteration from the least significant limb up to the most
2732 : significant limb. For large _BitInt it is emitted as straight line code
2733 : before current location, for huge _BitInt as a loop handling two limbs
2734 : at once, followed by handling up to limbs in straight line code (at most
2735 : one full and one partial limb). It can also handle EQ_EXPR/NE_EXPR
2736 : comparisons, in that case CMP_CODE should be the comparison code and
2737 : CMP_OP1/CMP_OP2 the comparison operands. */
2738 :
2739 : tree
2740 23609 : bitint_large_huge::lower_mergeable_stmt (gimple *stmt, tree_code &cmp_code,
2741 : tree cmp_op1, tree cmp_op2)
2742 : {
2743 23609 : bool eq_p = cmp_code != ERROR_MARK;
2744 23609 : tree type;
2745 23609 : if (eq_p)
2746 6665 : type = TREE_TYPE (cmp_op1);
2747 : else
2748 16944 : type = TREE_TYPE (gimple_assign_lhs (stmt));
2749 23609 : gcc_assert (BITINT_TYPE_P (type));
2750 23609 : bitint_prec_kind kind = bitint_precision_kind (type);
2751 23609 : gcc_assert (kind >= bitint_prec_large);
2752 23609 : gimple *g;
2753 23609 : tree lhs = gimple_get_lhs (stmt);
2754 23609 : tree rhs1, lhs_type = lhs ? TREE_TYPE (lhs) : NULL_TREE;
2755 17384 : if (lhs
2756 17384 : && TREE_CODE (lhs) == SSA_NAME
2757 8815 : && BITINT_TYPE_P (TREE_TYPE (lhs))
2758 8375 : && bitint_precision_kind (TREE_TYPE (lhs)) >= bitint_prec_large)
2759 : {
2760 8375 : int p = var_to_partition (m_map, lhs);
2761 8375 : gcc_assert (m_vars[p] != NULL_TREE);
2762 8375 : m_lhs = lhs = m_vars[p];
2763 : }
2764 23609 : unsigned cnt, rem = 0, end = 0, prec = TYPE_PRECISION (type);
2765 23609 : bool sext = false;
2766 23609 : tree ext = NULL_TREE, store_operand = NULL_TREE;
2767 23609 : bool eh = false;
2768 23609 : basic_block eh_pad = NULL;
2769 23609 : tree nlhs = NULL_TREE;
2770 23609 : unsigned HOST_WIDE_INT bo_idx = 0;
2771 23609 : unsigned HOST_WIDE_INT bo_bit = 0;
2772 23609 : unsigned bo_shift = 0;
2773 23609 : unsigned bo_last = 0;
2774 23609 : bool bo_be_p = false;
2775 23609 : tree bf_cur = NULL_TREE, bf_next = NULL_TREE;
2776 23609 : if (gimple_store_p (stmt))
2777 : {
2778 8569 : store_operand = gimple_assign_rhs1 (stmt);
2779 8569 : eh = stmt_ends_bb_p (stmt);
2780 8569 : if (eh)
2781 : {
2782 2 : edge e;
2783 2 : edge_iterator ei;
2784 2 : basic_block bb = gimple_bb (stmt);
2785 :
2786 2 : FOR_EACH_EDGE (e, ei, bb->succs)
2787 2 : if (e->flags & EDGE_EH)
2788 : {
2789 2 : eh_pad = e->dest;
2790 2 : break;
2791 : }
2792 : }
2793 8569 : if (TREE_CODE (lhs) == COMPONENT_REF
2794 8569 : && DECL_BIT_FIELD_TYPE (TREE_OPERAND (lhs, 1)))
2795 : {
2796 162 : tree fld = TREE_OPERAND (lhs, 1);
2797 162 : gcc_assert (tree_fits_uhwi_p (DECL_FIELD_BIT_OFFSET (fld)));
2798 162 : tree repr = DECL_BIT_FIELD_REPRESENTATIVE (fld);
2799 162 : poly_int64 bitoffset;
2800 162 : poly_uint64 field_offset, repr_offset;
2801 162 : if (!bitint_big_endian
2802 162 : && (tree_to_uhwi (DECL_FIELD_BIT_OFFSET (fld))
2803 162 : % BITS_PER_UNIT) == 0)
2804 : nlhs = lhs;
2805 : else
2806 : {
2807 143 : bool var_field_off = false;
2808 143 : if (poly_int_tree_p (DECL_FIELD_OFFSET (fld), &field_offset)
2809 286 : && poly_int_tree_p (DECL_FIELD_OFFSET (repr), &repr_offset))
2810 143 : bitoffset = (field_offset - repr_offset) * BITS_PER_UNIT;
2811 : else
2812 : {
2813 : bitoffset = 0;
2814 : var_field_off = true;
2815 : }
2816 143 : bitoffset += (tree_to_uhwi (DECL_FIELD_BIT_OFFSET (fld))
2817 143 : - tree_to_uhwi (DECL_FIELD_BIT_OFFSET (repr)));
2818 286 : nlhs = build3 (COMPONENT_REF, TREE_TYPE (repr),
2819 143 : TREE_OPERAND (lhs, 0), repr,
2820 : var_field_off
2821 0 : ? TREE_OPERAND (lhs, 2) : NULL_TREE);
2822 143 : HOST_WIDE_INT bo = bitoffset.to_constant ();
2823 143 : bo_idx = (unsigned HOST_WIDE_INT) bo / limb_prec;
2824 143 : bo_bit = (unsigned HOST_WIDE_INT) bo % limb_prec;
2825 143 : bo_shift = bo_bit;
2826 143 : if (bitint_big_endian)
2827 : {
2828 0 : bo_last = CEIL (prec + bo_bit, limb_prec) - 1;
2829 0 : bo_shift = (prec + bo_bit) % limb_prec;
2830 0 : bo_be_p = true;
2831 0 : if (bo_shift)
2832 0 : bo_shift = limb_prec - bo_shift;
2833 : }
2834 : }
2835 : }
2836 : }
2837 8569 : if ((store_operand
2838 8569 : && TREE_CODE (store_operand) == SSA_NAME
2839 7581 : && (m_names == NULL
2840 7551 : || !bitmap_bit_p (m_names, SSA_NAME_VERSION (store_operand)))
2841 1164 : && gimple_assign_cast_p (SSA_NAME_DEF_STMT (store_operand)))
2842 31622 : || gimple_assign_cast_p (stmt))
2843 : {
2844 2169 : rhs1 = gimple_assign_rhs1 (store_operand
2845 556 : ? SSA_NAME_DEF_STMT (store_operand)
2846 : : stmt);
2847 1613 : if (TREE_CODE (rhs1) == VIEW_CONVERT_EXPR)
2848 2 : rhs1 = TREE_OPERAND (rhs1, 0);
2849 : /* Optimize mergeable ops ending with widening cast to _BitInt
2850 : (or followed by store). We can lower just the limbs of the
2851 : cast operand and widen afterwards. */
2852 1613 : if (TREE_CODE (rhs1) == SSA_NAME
2853 1613 : && (m_names == NULL
2854 1607 : || !bitmap_bit_p (m_names, SSA_NAME_VERSION (rhs1)))
2855 629 : && BITINT_TYPE_P (TREE_TYPE (rhs1))
2856 480 : && bitint_precision_kind (TREE_TYPE (rhs1)) >= bitint_prec_large
2857 2044 : && (CEIL ((unsigned) TYPE_PRECISION (TREE_TYPE (rhs1)),
2858 431 : limb_prec) < CEIL (prec, limb_prec)
2859 306 : || (kind == bitint_prec_huge
2860 262 : && TYPE_PRECISION (TREE_TYPE (rhs1)) < prec)))
2861 : {
2862 131 : store_operand = rhs1;
2863 131 : prec = TYPE_PRECISION (TREE_TYPE (rhs1));
2864 131 : kind = bitint_precision_kind (TREE_TYPE (rhs1));
2865 131 : if (!TYPE_UNSIGNED (TREE_TYPE (rhs1)))
2866 54 : sext = true;
2867 : }
2868 : }
2869 23609 : tree idx = NULL_TREE, idx_first = NULL_TREE, idx_next = NULL_TREE;
2870 23609 : if (kind == bitint_prec_large)
2871 12301 : cnt = CEIL (prec, limb_prec);
2872 : else
2873 : {
2874 11308 : rem = (prec % (2 * limb_prec));
2875 11308 : end = (prec - rem) / limb_prec;
2876 11308 : cnt = 2 + CEIL (rem, limb_prec);
2877 11308 : idx = idx_first = create_loop (bitint_big_endian
2878 11308 : ? size_int (cnt - 2 + end - 1)
2879 : : size_zero_node, &idx_next);
2880 : }
2881 :
2882 23609 : basic_block edge_bb = NULL;
2883 23609 : if (eq_p)
2884 : {
2885 6665 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
2886 6665 : gsi_prev (&gsi);
2887 6665 : edge e = split_block (gsi_bb (gsi), gsi_stmt (gsi));
2888 6665 : edge_bb = e->src;
2889 6665 : if (kind == bitint_prec_large)
2890 7264 : m_gsi = gsi_end_bb (edge_bb);
2891 : }
2892 : else
2893 16944 : m_after_stmt = stmt;
2894 23609 : if (kind != bitint_prec_large)
2895 11308 : m_upwards_2limb = end;
2896 23609 : m_upwards = true;
2897 :
2898 23609 : bool separate_ext
2899 23609 : = (prec != (unsigned) TYPE_PRECISION (type)
2900 23609 : && (CEIL ((unsigned) TYPE_PRECISION (type), limb_prec)
2901 131 : > CEIL (prec, limb_prec)));
2902 23609 : bool zero_ms_limb = false;
2903 23609 : if (bitint_extended == bitint_ext_full
2904 0 : && !eq_p
2905 0 : && !nlhs
2906 0 : && abi_limb_prec > limb_prec
2907 23609 : && ((CEIL ((unsigned) TYPE_PRECISION (type), abi_limb_prec)
2908 0 : * abi_limb_prec / limb_prec)
2909 0 : > CEIL ((unsigned) TYPE_PRECISION (type), limb_prec)))
2910 : {
2911 0 : if (prec == (unsigned) TYPE_PRECISION (type))
2912 : {
2913 0 : sext = !TYPE_UNSIGNED (type);
2914 0 : separate_ext = true;
2915 : }
2916 0 : else if (TYPE_UNSIGNED (type) && sext)
2917 : zero_ms_limb = true;
2918 : else
2919 : separate_ext = true;
2920 : }
2921 23734 : unsigned dst_idx_off = 0;
2922 23609 : if (separate_ext && bitint_big_endian)
2923 0 : dst_idx_off = (CEIL ((unsigned) TYPE_PRECISION (type), limb_prec)
2924 0 : - CEIL (prec, limb_prec));
2925 :
2926 94635 : for (unsigned i = 0; i < cnt; i++)
2927 : {
2928 71026 : m_data_cnt = 0;
2929 71026 : if (kind == bitint_prec_large)
2930 37517 : idx = size_int (bitint_big_endian ? cnt - 1 - i : i);
2931 33509 : else if (i >= 2)
2932 10893 : idx = size_int (bitint_big_endian ? cnt - 1 - i : end + (i > 2));
2933 71026 : if (eq_p)
2934 : {
2935 20164 : rhs1 = handle_operand (cmp_op1, idx);
2936 20164 : tree rhs2 = handle_operand (cmp_op2, idx);
2937 20164 : g = gimple_build_cond (NE_EXPR, rhs1, rhs2, NULL_TREE, NULL_TREE);
2938 20164 : insert_before (g);
2939 20164 : edge e1 = split_block (gsi_bb (m_gsi), g);
2940 20164 : e1->flags = EDGE_FALSE_VALUE;
2941 20164 : edge e2 = make_edge (e1->src, gimple_bb (stmt), EDGE_TRUE_VALUE);
2942 20164 : e1->probability = profile_probability::unlikely ();
2943 20164 : e2->probability = e1->probability.invert ();
2944 20164 : if (i == 0)
2945 6665 : set_immediate_dominator (CDI_DOMINATORS, e2->dest, e2->src);
2946 20164 : m_gsi = gsi_after_labels (e1->dest);
2947 : }
2948 : else
2949 : {
2950 50862 : if (store_operand)
2951 25930 : rhs1 = handle_operand (store_operand, idx);
2952 : else
2953 24932 : rhs1 = handle_stmt (stmt, idx);
2954 50862 : if (!useless_type_conversion_p (m_limb_type, TREE_TYPE (rhs1)))
2955 12155 : rhs1 = add_cast (m_limb_type, rhs1);
2956 50862 : if (sext && i == cnt - 1)
2957 50862 : ext = rhs1;
2958 50862 : tree nidx = idx;
2959 50862 : HOST_WIDE_INT adj = bo_idx;
2960 50862 : if (bo_be_p)
2961 0 : adj += bo_last - (CEIL (prec, limb_prec) - 1);
2962 : else
2963 50862 : adj += dst_idx_off;
2964 50862 : if (adj)
2965 : {
2966 220 : if (tree_fits_uhwi_p (idx))
2967 116 : nidx = size_int (tree_to_uhwi (idx) + adj);
2968 : else
2969 : {
2970 104 : nidx = make_ssa_name (sizetype);
2971 104 : g = gimple_build_assign (nidx, PLUS_EXPR, idx,
2972 104 : size_int (adj));
2973 104 : insert_before (g);
2974 : }
2975 : }
2976 50862 : bool done = false;
2977 50862 : basic_block new_bb = NULL;
2978 : /* Handle stores into bit-fields. */
2979 50862 : if (bo_shift)
2980 : {
2981 455 : if (i == 0)
2982 : {
2983 143 : edge e2 = NULL;
2984 143 : if (kind != bitint_prec_large)
2985 : {
2986 112 : prepare_data_in_out (build_zero_cst (m_limb_type),
2987 : idx, &bf_next);
2988 112 : bf_next = m_data.pop ();
2989 112 : bf_cur = m_data.pop ();
2990 112 : g = gimple_build_cond (EQ_EXPR, idx,
2991 : bitint_big_endian
2992 0 : ? size_int (CEIL (prec,
2993 : limb_prec) - 1)
2994 : : size_zero_node,
2995 : NULL_TREE, NULL_TREE);
2996 112 : edge edge_true;
2997 112 : if_then_else (g, profile_probability::unlikely (),
2998 : edge_true, e2);
2999 112 : new_bb = e2->dest;
3000 : }
3001 143 : tree ftype
3002 143 : = build_nonstandard_integer_type (limb_prec - bo_shift, 1);
3003 286 : tree bfr = build_bit_field_ref (ftype, unshare_expr (nlhs),
3004 143 : limb_prec - bo_shift,
3005 : bitint_big_endian
3006 0 : ? (bo_idx + bo_last)
3007 0 : * limb_prec
3008 143 : : bo_idx * limb_prec
3009 : + bo_bit);
3010 143 : tree t = add_cast (ftype, rhs1);
3011 143 : g = gimple_build_assign (bfr, t);
3012 143 : insert_before (g);
3013 143 : if (eh)
3014 : {
3015 0 : maybe_duplicate_eh_stmt (g, stmt);
3016 0 : if (eh_pad)
3017 : {
3018 0 : edge e = split_block (gsi_bb (m_gsi), g);
3019 0 : m_gsi = gsi_after_labels (e->dest);
3020 0 : add_eh_edge (e->src,
3021 : find_edge (gimple_bb (stmt), eh_pad));
3022 : }
3023 : }
3024 143 : if (kind == bitint_prec_large)
3025 : {
3026 : bf_cur = rhs1;
3027 : done = true;
3028 : }
3029 112 : else if (e2)
3030 112 : m_gsi = gsi_after_labels (e2->src);
3031 : }
3032 112 : if (!done)
3033 : {
3034 424 : tree t1 = make_ssa_name (m_limb_type);
3035 424 : tree t2 = make_ssa_name (m_limb_type);
3036 424 : tree t3 = make_ssa_name (m_limb_type);
3037 424 : g = gimple_build_assign (t1, RSHIFT_EXPR, bf_cur,
3038 : build_int_cst (unsigned_type_node,
3039 424 : limb_prec
3040 424 : - bo_shift));
3041 424 : insert_before (g);
3042 424 : g = gimple_build_assign (t2, LSHIFT_EXPR, rhs1,
3043 : build_int_cst (unsigned_type_node,
3044 424 : bo_shift));
3045 424 : insert_before (g);
3046 424 : bf_cur = rhs1;
3047 424 : g = gimple_build_assign (t3, BIT_IOR_EXPR, t1, t2);
3048 424 : insert_before (g);
3049 424 : rhs1 = t3;
3050 424 : if (bf_next && i == 1)
3051 : {
3052 112 : g = gimple_build_assign (bf_next, bf_cur);
3053 112 : insert_before (g);
3054 : }
3055 : }
3056 : }
3057 31 : if (!done)
3058 : {
3059 : /* Handle bit-field access to partial last limb if needed. */
3060 50831 : if (nlhs
3061 480 : && i == cnt - 1
3062 162 : && !separate_ext
3063 138 : && tree_fits_uhwi_p (idx))
3064 : {
3065 123 : unsigned int tprec = TYPE_PRECISION (type);
3066 123 : unsigned int rprec = (tprec - 1) % limb_prec + 1;
3067 123 : if (rprec + bo_shift < (unsigned) limb_prec)
3068 : {
3069 46 : tree ftype
3070 46 : = build_nonstandard_integer_type (rprec + bo_shift, 1);
3071 46 : tree bfr
3072 92 : = build_bit_field_ref (ftype, unshare_expr (nlhs),
3073 : rprec + bo_shift,
3074 : bitint_big_endian
3075 0 : ? bo_idx * limb_prec + bo_bit
3076 46 : : (bo_idx + tprec / limb_prec)
3077 46 : * limb_prec);
3078 46 : tree t = add_cast (ftype, rhs1);
3079 46 : g = gimple_build_assign (bfr, t);
3080 46 : done = true;
3081 46 : bf_cur = NULL_TREE;
3082 : }
3083 77 : else if (rprec + bo_shift == (unsigned) limb_prec)
3084 50785 : bf_cur = NULL_TREE;
3085 : }
3086 : /* Otherwise, stores to any other lhs. */
3087 46 : if (!done)
3088 : {
3089 51219 : tree l = limb_access (nlhs ? NULL_TREE : lhs_type,
3090 : nlhs ? nlhs : lhs, nidx, true);
3091 50785 : g = gimple_build_assign (l, rhs1);
3092 : }
3093 50831 : insert_before (g);
3094 50831 : if (eh)
3095 : {
3096 6 : maybe_duplicate_eh_stmt (g, stmt);
3097 6 : if (eh_pad)
3098 : {
3099 6 : edge e = split_block (gsi_bb (m_gsi), g);
3100 6 : m_gsi = gsi_after_labels (e->dest);
3101 6 : add_eh_edge (e->src,
3102 : find_edge (gimple_bb (stmt), eh_pad));
3103 : }
3104 : }
3105 50831 : if (new_bb)
3106 112 : m_gsi = gsi_after_labels (new_bb);
3107 : }
3108 : }
3109 71026 : m_first = false;
3110 71026 : if (kind == bitint_prec_huge && i <= 1)
3111 : {
3112 22616 : if (i == 0)
3113 : {
3114 11308 : idx = make_ssa_name (sizetype);
3115 11308 : g = gimple_build_assign (idx, PLUS_EXPR, idx_first,
3116 : bitint_big_endian
3117 0 : ? size_int (-1) : size_one_node);
3118 11308 : insert_before (g);
3119 : }
3120 : else
3121 : {
3122 11308 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx_first,
3123 22616 : size_int (bitint_big_endian ? -2 : 2));
3124 11308 : insert_before (g);
3125 11308 : if (bitint_big_endian)
3126 0 : g = gimple_build_cond (NE_EXPR, idx_first, size_int (cnt - 1),
3127 : NULL_TREE, NULL_TREE);
3128 : else
3129 11308 : g = gimple_build_cond (NE_EXPR, idx_next, size_int (end),
3130 : NULL_TREE, NULL_TREE);
3131 11308 : insert_before (g);
3132 11308 : if (eq_p)
3133 3033 : m_gsi = gsi_after_labels (edge_bb);
3134 : else
3135 8275 : m_gsi = gsi_for_stmt (stmt);
3136 11308 : m_bb = NULL;
3137 : }
3138 : }
3139 : }
3140 :
3141 23609 : if (separate_ext)
3142 : {
3143 125 : if (sext)
3144 : {
3145 51 : ext = add_cast (signed_type_for (m_limb_type), ext);
3146 102 : tree lpm1 = build_int_cst (unsigned_type_node,
3147 51 : limb_prec - 1);
3148 51 : tree n = make_ssa_name (TREE_TYPE (ext));
3149 51 : g = gimple_build_assign (n, RSHIFT_EXPR, ext, lpm1);
3150 51 : insert_before (g);
3151 51 : ext = add_cast (m_limb_type, n);
3152 : }
3153 : else
3154 74 : ext = build_zero_cst (m_limb_type);
3155 125 : kind = bitint_precision_kind (type);
3156 125 : unsigned start = CEIL (prec, limb_prec);
3157 125 : prec = TYPE_PRECISION (type);
3158 125 : if (bitint_extended == bitint_ext_full
3159 0 : && !nlhs
3160 0 : && !zero_ms_limb
3161 0 : && abi_limb_prec > limb_prec)
3162 : {
3163 0 : prec = CEIL (prec, abi_limb_prec) * abi_limb_prec;
3164 0 : kind = bitint_precision_kind (prec);
3165 : }
3166 125 : unsigned total = CEIL (prec, limb_prec);
3167 125 : idx = idx_first = idx_next = NULL_TREE;
3168 125 : if (prec <= (start + 2 + (bo_shift != 0)) * limb_prec)
3169 : kind = bitint_prec_large;
3170 80 : if (kind == bitint_prec_large)
3171 45 : cnt = total - start;
3172 : else
3173 : {
3174 80 : rem = prec % limb_prec;
3175 80 : end = (prec - rem) / limb_prec;
3176 152 : cnt = (bo_shift != 0) + 1 + (rem != 0);
3177 : }
3178 125 : if (bitint_big_endian && bo_shift != 0 && (prec % limb_prec) == 0)
3179 0 : ++total;
3180 321 : for (unsigned i = 0; i < cnt; i++)
3181 : {
3182 196 : if (kind == bitint_prec_large || (i == 0 && bo_shift != 0))
3183 62 : idx = size_int (bo_idx
3184 : + (bitint_big_endian
3185 : ? total - 1 - start - i : start + i));
3186 134 : else if (i == cnt - 1 && rem != 0)
3187 108 : idx = size_int (bo_idx + (bitint_big_endian ? 0 : end));
3188 80 : else if (i == (bo_shift != 0))
3189 80 : idx = create_loop (size_int (bo_idx
3190 : + (bitint_big_endian
3191 : ? total - 1 - start - i
3192 : : start + i)), &idx_next);
3193 196 : rhs1 = ext;
3194 196 : if (bf_cur != NULL_TREE && bf_cur != ext)
3195 : {
3196 20 : tree t1 = make_ssa_name (m_limb_type);
3197 20 : g = gimple_build_assign (t1, RSHIFT_EXPR, bf_cur,
3198 : build_int_cst (unsigned_type_node,
3199 20 : limb_prec - bo_shift));
3200 20 : insert_before (g);
3201 20 : if (integer_zerop (ext))
3202 : rhs1 = t1;
3203 : else
3204 : {
3205 10 : tree t2 = make_ssa_name (m_limb_type);
3206 10 : rhs1 = make_ssa_name (m_limb_type);
3207 10 : g = gimple_build_assign (t2, LSHIFT_EXPR, ext,
3208 : build_int_cst (unsigned_type_node,
3209 10 : bo_shift));
3210 10 : insert_before (g);
3211 10 : g = gimple_build_assign (rhs1, BIT_IOR_EXPR, t1, t2);
3212 10 : insert_before (g);
3213 : }
3214 : bf_cur = ext;
3215 : }
3216 196 : bool done = false;
3217 : /* Handle bit-field access to partial last limb if needed. */
3218 196 : if (nlhs && i == cnt - 1)
3219 : {
3220 24 : unsigned int tprec = TYPE_PRECISION (type);
3221 24 : unsigned int rprec = (tprec - 1) % limb_prec + 1;
3222 24 : if (rprec + bo_shift < (unsigned) limb_prec)
3223 : {
3224 12 : tree ftype
3225 12 : = build_nonstandard_integer_type (rprec + bo_shift, 1);
3226 12 : tree bfr
3227 24 : = build_bit_field_ref (ftype, unshare_expr (nlhs),
3228 : rprec + bo_shift,
3229 : bitint_big_endian
3230 0 : ? bo_idx * limb_prec + bo_bit
3231 12 : : (bo_idx + tprec / limb_prec)
3232 12 : * limb_prec);
3233 12 : tree t = add_cast (ftype, rhs1);
3234 12 : g = gimple_build_assign (bfr, t);
3235 12 : done = true;
3236 12 : bf_cur = NULL_TREE;
3237 : }
3238 12 : else if (rprec + bo_shift == (unsigned) limb_prec)
3239 : bf_cur = NULL_TREE;
3240 : }
3241 : /* Otherwise, stores to any other lhs. */
3242 12 : if (!done)
3243 : {
3244 214 : tree l = limb_access (nlhs ? NULL_TREE : lhs_type,
3245 : nlhs ? nlhs : lhs, idx, true);
3246 :
3247 184 : if (bitint_extended
3248 0 : && sext
3249 0 : && TYPE_UNSIGNED (lhs_type)
3250 0 : && tree_fits_uhwi_p (idx)
3251 184 : && !nlhs)
3252 : {
3253 0 : rhs1 = add_cast (limb_access_type (lhs_type, idx), rhs1);
3254 0 : rhs1 = add_cast (TREE_TYPE (l), rhs1);
3255 : }
3256 :
3257 184 : g = gimple_build_assign (l, rhs1);
3258 : }
3259 196 : insert_before (g);
3260 196 : if (eh)
3261 : {
3262 0 : maybe_duplicate_eh_stmt (g, stmt);
3263 0 : if (eh_pad)
3264 : {
3265 0 : edge e = split_block (gsi_bb (m_gsi), g);
3266 0 : m_gsi = gsi_after_labels (e->dest);
3267 0 : add_eh_edge (e->src, find_edge (gimple_bb (stmt), eh_pad));
3268 : }
3269 : }
3270 196 : if (kind == bitint_prec_huge && i == (bo_shift != 0))
3271 : {
3272 80 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
3273 : bitint_big_endian
3274 0 : ? size_int (-1) : size_one_node);
3275 80 : insert_before (g);
3276 : /* For big-endian, if bo_idx + total - 1 - end is all ones, then
3277 : compare idx (which is equal to idx_next + 1) against 0
3278 : instead. */
3279 80 : if (bitint_big_endian && bo_idx + total - end == 0)
3280 0 : g = gimple_build_cond (NE_EXPR, idx, size_zero_node,
3281 : NULL_TREE, NULL_TREE);
3282 : else
3283 80 : g = gimple_build_cond (NE_EXPR, idx_next,
3284 80 : size_int (bo_idx
3285 : + (bitint_big_endian
3286 : ? total - 1 - end
3287 : : end)),
3288 : NULL_TREE, NULL_TREE);
3289 80 : insert_before (g);
3290 80 : m_gsi = gsi_for_stmt (stmt);
3291 80 : m_bb = NULL;
3292 : }
3293 : }
3294 : }
3295 23609 : if (bf_cur != NULL_TREE)
3296 : {
3297 72 : unsigned int tprec = TYPE_PRECISION (type);
3298 72 : unsigned int rprec = (tprec + bo_shift) % limb_prec;
3299 72 : tree ftype = build_nonstandard_integer_type (rprec, 1);
3300 144 : tree bfr = build_bit_field_ref (ftype, unshare_expr (nlhs),
3301 : rprec,
3302 : bitint_big_endian
3303 0 : ? bo_idx * limb_prec + bo_bit
3304 72 : : (bo_idx + (tprec + bo_bit) / limb_prec)
3305 : * limb_prec);
3306 72 : rhs1 = bf_cur;
3307 72 : if (bf_cur != ext)
3308 : {
3309 64 : rhs1 = make_ssa_name (TREE_TYPE (rhs1));
3310 64 : g = gimple_build_assign (rhs1, RSHIFT_EXPR, bf_cur,
3311 : build_int_cst (unsigned_type_node,
3312 64 : limb_prec - bo_shift));
3313 64 : insert_before (g);
3314 : }
3315 72 : rhs1 = add_cast (ftype, rhs1);
3316 72 : g = gimple_build_assign (bfr, rhs1);
3317 72 : insert_before (g);
3318 72 : if (eh)
3319 : {
3320 0 : maybe_duplicate_eh_stmt (g, stmt);
3321 0 : if (eh_pad)
3322 : {
3323 0 : edge e = split_block (gsi_bb (m_gsi), g);
3324 0 : m_gsi = gsi_after_labels (e->dest);
3325 0 : add_eh_edge (e->src, find_edge (gimple_bb (stmt), eh_pad));
3326 : }
3327 : }
3328 : }
3329 23609 : if (zero_ms_limb)
3330 : {
3331 0 : tree p2 = build_int_cst (sizetype,
3332 0 : CEIL ((unsigned) TYPE_PRECISION (type),
3333 : abi_limb_prec)
3334 0 : * abi_limb_prec / limb_prec - 1);
3335 0 : tree l = limb_access (lhs_type, lhs, p2, true);
3336 0 : g = gimple_build_assign (l, build_zero_cst (m_limb_type));
3337 0 : insert_before (g);
3338 0 : if (eh)
3339 : {
3340 0 : maybe_duplicate_eh_stmt (g, stmt);
3341 0 : if (eh_pad)
3342 : {
3343 0 : edge e = split_block (gsi_bb (m_gsi), g);
3344 0 : m_gsi = gsi_after_labels (e->dest);
3345 0 : add_eh_edge (e->src, find_edge (gimple_bb (stmt), eh_pad));
3346 : }
3347 : }
3348 : }
3349 :
3350 23609 : if (gimple_store_p (stmt))
3351 : {
3352 8569 : unlink_stmt_vdef (stmt);
3353 17138 : release_ssa_name (gimple_vdef (stmt));
3354 8569 : gsi_remove (&m_gsi, true);
3355 : }
3356 23609 : if (eq_p)
3357 : {
3358 6665 : lhs = make_ssa_name (boolean_type_node);
3359 6665 : basic_block bb = gimple_bb (stmt);
3360 6665 : gphi *phi = create_phi_node (lhs, bb);
3361 6665 : edge e = find_edge (gsi_bb (m_gsi), bb);
3362 6665 : unsigned int n = EDGE_COUNT (bb->preds);
3363 33494 : for (unsigned int i = 0; i < n; i++)
3364 : {
3365 26829 : edge e2 = EDGE_PRED (bb, i);
3366 26829 : add_phi_arg (phi, e == e2 ? boolean_true_node : boolean_false_node,
3367 : e2, UNKNOWN_LOCATION);
3368 : }
3369 6665 : cmp_code = cmp_code == EQ_EXPR ? NE_EXPR : EQ_EXPR;
3370 6665 : return lhs;
3371 : }
3372 : else
3373 : return NULL_TREE;
3374 : }
3375 :
3376 : /* Handle a large/huge _BitInt comparison statement STMT other than
3377 : EQ_EXPR/NE_EXPR. CMP_CODE, CMP_OP1 and CMP_OP2 meaning is like in
3378 : lower_mergeable_stmt. The {GT,GE,LT,LE}_EXPR comparisons are
3379 : lowered by iteration from the most significant limb downwards to
3380 : the least significant one, for large _BitInt in straight line code,
3381 : otherwise with most significant limb handled in
3382 : straight line code followed by a loop handling one limb at a time.
3383 : Comparisons with unsigned huge _BitInt with precisions which are
3384 : multiples of limb precision can use just the loop and don't need to
3385 : handle most significant limb before the loop. The loop or straight
3386 : line code jumps to final basic block if a particular pair of limbs
3387 : is not equal. */
3388 :
3389 : tree
3390 722 : bitint_large_huge::lower_comparison_stmt (gimple *stmt, tree_code &cmp_code,
3391 : tree cmp_op1, tree cmp_op2)
3392 : {
3393 722 : tree type = TREE_TYPE (cmp_op1);
3394 722 : gcc_assert (BITINT_TYPE_P (type));
3395 722 : bitint_prec_kind kind = bitint_precision_kind (type);
3396 722 : gcc_assert (kind >= bitint_prec_large);
3397 722 : gimple *g;
3398 722 : if (!TYPE_UNSIGNED (type)
3399 441 : && integer_zerop (cmp_op2)
3400 750 : && (cmp_code == GE_EXPR || cmp_code == LT_EXPR))
3401 : {
3402 28 : unsigned end = CEIL ((unsigned) TYPE_PRECISION (type), limb_prec) - 1;
3403 56 : tree idx = size_int (bitint_big_endian ? 0 : end);
3404 28 : m_data_cnt = 0;
3405 28 : tree rhs1 = handle_operand (cmp_op1, idx);
3406 28 : if (TYPE_UNSIGNED (TREE_TYPE (rhs1)))
3407 : {
3408 24 : tree stype = signed_type_for (TREE_TYPE (rhs1));
3409 24 : rhs1 = add_cast (stype, rhs1);
3410 : }
3411 28 : tree lhs = make_ssa_name (boolean_type_node);
3412 28 : g = gimple_build_assign (lhs, cmp_code, rhs1,
3413 28 : build_zero_cst (TREE_TYPE (rhs1)));
3414 28 : insert_before (g);
3415 28 : cmp_code = NE_EXPR;
3416 28 : return lhs;
3417 : }
3418 :
3419 694 : unsigned cnt, rem = 0, end = 0;
3420 694 : tree idx = NULL_TREE, idx_next = NULL_TREE;
3421 694 : if (kind == bitint_prec_large)
3422 377 : cnt = CEIL ((unsigned) TYPE_PRECISION (type), limb_prec);
3423 : else
3424 : {
3425 317 : rem = ((unsigned) TYPE_PRECISION (type) % limb_prec);
3426 317 : if (rem == 0 && !TYPE_UNSIGNED (type))
3427 : rem = limb_prec;
3428 317 : end = ((unsigned) TYPE_PRECISION (type) - rem) / limb_prec;
3429 317 : cnt = 1 + (rem != 0);
3430 : }
3431 :
3432 694 : basic_block edge_bb = NULL;
3433 694 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
3434 694 : gsi_prev (&gsi);
3435 694 : edge e = split_block (gsi_bb (gsi), gsi_stmt (gsi));
3436 694 : edge_bb = e->src;
3437 694 : m_gsi = gsi_end_bb (edge_bb);
3438 :
3439 694 : edge *edges = XALLOCAVEC (edge, cnt * 2);
3440 2519 : for (unsigned i = 0; i < cnt; i++)
3441 : {
3442 1825 : m_data_cnt = 0;
3443 1825 : if (kind == bitint_prec_large)
3444 1246 : idx = size_int (bitint_big_endian ? i : cnt - i - 1);
3445 579 : else if (i == cnt - 1)
3446 317 : idx = create_loop (size_int (bitint_big_endian ? cnt - 1 : end - 1),
3447 : &idx_next);
3448 : else
3449 524 : idx = size_int (bitint_big_endian ? 0 : end);
3450 1825 : tree rhs1 = handle_operand (cmp_op1, idx);
3451 1825 : tree rhs2 = handle_operand (cmp_op2, idx);
3452 1825 : if (i == 0
3453 694 : && !TYPE_UNSIGNED (type)
3454 2238 : && TYPE_UNSIGNED (TREE_TYPE (rhs1)))
3455 : {
3456 112 : tree stype = signed_type_for (TREE_TYPE (rhs1));
3457 112 : rhs1 = add_cast (stype, rhs1);
3458 112 : rhs2 = add_cast (stype, rhs2);
3459 : }
3460 1825 : g = gimple_build_cond (GT_EXPR, rhs1, rhs2, NULL_TREE, NULL_TREE);
3461 1825 : insert_before (g);
3462 1825 : edge e1 = split_block (gsi_bb (m_gsi), g);
3463 1825 : e1->flags = EDGE_FALSE_VALUE;
3464 1825 : edge e2 = make_edge (e1->src, gimple_bb (stmt), EDGE_TRUE_VALUE);
3465 1825 : e1->probability = profile_probability::likely ();
3466 1825 : e2->probability = e1->probability.invert ();
3467 1825 : if (i == 0)
3468 694 : set_immediate_dominator (CDI_DOMINATORS, e2->dest, e2->src);
3469 1825 : m_gsi = gsi_after_labels (e1->dest);
3470 1825 : edges[2 * i] = e2;
3471 1825 : g = gimple_build_cond (LT_EXPR, rhs1, rhs2, NULL_TREE, NULL_TREE);
3472 1825 : insert_before (g);
3473 1825 : e1 = split_block (gsi_bb (m_gsi), g);
3474 1825 : e1->flags = EDGE_FALSE_VALUE;
3475 1825 : e2 = make_edge (e1->src, gimple_bb (stmt), EDGE_TRUE_VALUE);
3476 1825 : e1->probability = profile_probability::unlikely ();
3477 1825 : e2->probability = e1->probability.invert ();
3478 1825 : m_gsi = gsi_after_labels (e1->dest);
3479 1825 : edges[2 * i + 1] = e2;
3480 1825 : m_first = false;
3481 1825 : if (kind == bitint_prec_huge && i == cnt - 1)
3482 : {
3483 634 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
3484 : bitint_big_endian ? size_one_node
3485 317 : : size_int (-1));
3486 317 : insert_before (g);
3487 317 : g = gimple_build_cond (NE_EXPR, idx,
3488 : bitint_big_endian
3489 0 : ? size_int (end - 1 + (cnt != 1))
3490 : : size_zero_node,
3491 : NULL_TREE, NULL_TREE);
3492 317 : insert_before (g);
3493 317 : edge true_edge, false_edge;
3494 317 : extract_true_false_edges_from_block (gsi_bb (m_gsi),
3495 : &true_edge, &false_edge);
3496 317 : m_gsi = gsi_after_labels (false_edge->dest);
3497 317 : m_bb = NULL;
3498 : }
3499 : }
3500 :
3501 694 : tree lhs = make_ssa_name (boolean_type_node);
3502 694 : basic_block bb = gimple_bb (stmt);
3503 694 : gphi *phi = create_phi_node (lhs, bb);
3504 5038 : for (unsigned int i = 0; i < cnt * 2; i++)
3505 : {
3506 3650 : tree val = ((cmp_code == GT_EXPR || cmp_code == GE_EXPR)
3507 3650 : ^ (i & 1)) ? boolean_true_node : boolean_false_node;
3508 3650 : add_phi_arg (phi, val, edges[i], UNKNOWN_LOCATION);
3509 : }
3510 694 : add_phi_arg (phi, (cmp_code == GE_EXPR || cmp_code == LE_EXPR)
3511 : ? boolean_true_node : boolean_false_node,
3512 : find_edge (gsi_bb (m_gsi), bb), UNKNOWN_LOCATION);
3513 694 : cmp_code = NE_EXPR;
3514 694 : return lhs;
3515 : }
3516 :
3517 : /* Lower large/huge _BitInt left and right shift except for left
3518 : shift by < limb_prec constant. */
3519 :
3520 : void
3521 602 : bitint_large_huge::lower_shift_stmt (tree obj, gimple *stmt)
3522 : {
3523 602 : tree rhs1 = gimple_assign_rhs1 (stmt);
3524 602 : tree lhs = gimple_assign_lhs (stmt);
3525 602 : tree_code rhs_code = gimple_assign_rhs_code (stmt);
3526 602 : tree type = TREE_TYPE (rhs1);
3527 602 : gimple *final_stmt = gsi_stmt (m_gsi);
3528 602 : gcc_assert (BITINT_TYPE_P (type)
3529 : && bitint_precision_kind (type) >= bitint_prec_large);
3530 602 : int prec = TYPE_PRECISION (type);
3531 602 : tree n = gimple_assign_rhs2 (stmt), n1, n2, n3, n4;
3532 602 : gimple *g;
3533 602 : if (obj == NULL_TREE)
3534 : {
3535 483 : int part = var_to_partition (m_map, lhs);
3536 483 : gcc_assert (m_vars[part] != NULL_TREE);
3537 : obj = m_vars[part];
3538 : }
3539 : /* Preparation code common for both left and right shifts.
3540 : unsigned n1 = n % limb_prec;
3541 : size_t n2 = n / limb_prec;
3542 : size_t n3 = n1 != 0;
3543 : unsigned n4 = (limb_prec - n1) % limb_prec;
3544 : (for power of 2 limb_prec n4 can be -n1 & limb_prec). */
3545 602 : if (TREE_CODE (n) == INTEGER_CST)
3546 : {
3547 267 : tree lp = build_int_cst (TREE_TYPE (n), limb_prec);
3548 267 : n1 = int_const_binop (TRUNC_MOD_EXPR, n, lp);
3549 267 : n2 = fold_convert (sizetype, int_const_binop (TRUNC_DIV_EXPR, n, lp));
3550 267 : n3 = size_int (!integer_zerop (n1));
3551 267 : n4 = int_const_binop (TRUNC_MOD_EXPR,
3552 267 : int_const_binop (MINUS_EXPR, lp, n1), lp);
3553 : }
3554 : else
3555 : {
3556 335 : n1 = make_ssa_name (TREE_TYPE (n));
3557 335 : n2 = make_ssa_name (sizetype);
3558 335 : n3 = make_ssa_name (sizetype);
3559 335 : n4 = make_ssa_name (TREE_TYPE (n));
3560 335 : if (pow2p_hwi (limb_prec))
3561 : {
3562 335 : tree lpm1 = build_int_cst (TREE_TYPE (n), limb_prec - 1);
3563 335 : g = gimple_build_assign (n1, BIT_AND_EXPR, n, lpm1);
3564 335 : insert_before (g);
3565 989 : g = gimple_build_assign (useless_type_conversion_p (sizetype,
3566 335 : TREE_TYPE (n))
3567 319 : ? n2 : make_ssa_name (TREE_TYPE (n)),
3568 : RSHIFT_EXPR, n,
3569 335 : build_int_cst (TREE_TYPE (n),
3570 670 : exact_log2 (limb_prec)));
3571 335 : insert_before (g);
3572 335 : if (gimple_assign_lhs (g) != n2)
3573 : {
3574 319 : g = gimple_build_assign (n2, NOP_EXPR, gimple_assign_lhs (g));
3575 319 : insert_before (g);
3576 : }
3577 335 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (n)),
3578 : NEGATE_EXPR, n1);
3579 335 : insert_before (g);
3580 335 : g = gimple_build_assign (n4, BIT_AND_EXPR, gimple_assign_lhs (g),
3581 : lpm1);
3582 335 : insert_before (g);
3583 : }
3584 : else
3585 : {
3586 0 : tree lp = build_int_cst (TREE_TYPE (n), limb_prec);
3587 0 : g = gimple_build_assign (n1, TRUNC_MOD_EXPR, n, lp);
3588 0 : insert_before (g);
3589 0 : g = gimple_build_assign (useless_type_conversion_p (sizetype,
3590 0 : TREE_TYPE (n))
3591 0 : ? n2 : make_ssa_name (TREE_TYPE (n)),
3592 : TRUNC_DIV_EXPR, n, lp);
3593 0 : insert_before (g);
3594 0 : if (gimple_assign_lhs (g) != n2)
3595 : {
3596 0 : g = gimple_build_assign (n2, NOP_EXPR, gimple_assign_lhs (g));
3597 0 : insert_before (g);
3598 : }
3599 0 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (n)),
3600 : MINUS_EXPR, lp, n1);
3601 0 : insert_before (g);
3602 0 : g = gimple_build_assign (n4, TRUNC_MOD_EXPR, gimple_assign_lhs (g),
3603 : lp);
3604 0 : insert_before (g);
3605 : }
3606 335 : g = gimple_build_assign (make_ssa_name (boolean_type_node), NE_EXPR, n1,
3607 335 : build_zero_cst (TREE_TYPE (n)));
3608 335 : insert_before (g);
3609 335 : g = gimple_build_assign (n3, NOP_EXPR, gimple_assign_lhs (g));
3610 335 : insert_before (g);
3611 : }
3612 1204 : tree p = build_int_cst (sizetype,
3613 602 : prec / limb_prec - (prec % limb_prec == 0));
3614 602 : if (rhs_code == RSHIFT_EXPR)
3615 : {
3616 : /* Lower
3617 : dst = src >> n;
3618 : as
3619 : unsigned n1 = n % limb_prec;
3620 : size_t n2 = n / limb_prec;
3621 : size_t n3 = n1 != 0;
3622 : unsigned n4 = (limb_prec - n1) % limb_prec;
3623 : size_t idx;
3624 : size_t p = prec / limb_prec - (prec % limb_prec == 0);
3625 : int signed_p = (typeof (src) -1) < 0;
3626 : for (idx = n2; idx < ((!signed_p && (prec % limb_prec == 0))
3627 : ? p : p - n3); ++idx)
3628 : dst[idx - n2] = (src[idx] >> n1) | (src[idx + n3] << n4);
3629 : limb_type ext;
3630 : if (prec % limb_prec == 0)
3631 : ext = src[p];
3632 : else if (signed_p)
3633 : ext = ((signed limb_type) (src[p] << (limb_prec
3634 : - (prec % limb_prec))))
3635 : >> (limb_prec - (prec % limb_prec));
3636 : else
3637 : ext = src[p] & (((limb_type) 1 << (prec % limb_prec)) - 1);
3638 : if (!signed_p && (prec % limb_prec == 0))
3639 : ;
3640 : else if (idx < prec / 64)
3641 : {
3642 : dst[idx - n2] = (src[idx] >> n1) | (ext << n4);
3643 : ++idx;
3644 : }
3645 : idx -= n2;
3646 : if (signed_p)
3647 : {
3648 : dst[idx] = ((signed limb_type) ext) >> n1;
3649 : ext = ((signed limb_type) ext) >> (limb_prec - 1);
3650 : }
3651 : else
3652 : {
3653 : dst[idx] = ext >> n1;
3654 : ext = 0;
3655 : }
3656 : for (++idx; idx <= p; ++idx)
3657 : dst[idx] = ext; */
3658 405 : tree pmn3;
3659 405 : if (TYPE_UNSIGNED (type) && prec % limb_prec == 0)
3660 104 : pmn3 = bitint_big_endian ? size_zero_node : p;
3661 301 : else if (bitint_big_endian)
3662 : pmn3 = n3;
3663 301 : else if (TREE_CODE (n3) == INTEGER_CST)
3664 128 : pmn3 = int_const_binop (MINUS_EXPR, p, n3);
3665 : else
3666 : {
3667 173 : pmn3 = make_ssa_name (sizetype);
3668 173 : g = gimple_build_assign (pmn3, MINUS_EXPR, p, n3);
3669 173 : insert_before (g);
3670 : }
3671 405 : tree pmn2 = NULL_TREE;
3672 405 : if (bitint_big_endian)
3673 : {
3674 0 : if (TREE_CODE (n2) == INTEGER_CST)
3675 0 : pmn2 = int_const_binop (MINUS_EXPR, p, n2);
3676 : else
3677 : {
3678 0 : pmn2 = make_ssa_name (sizetype);
3679 0 : g = gimple_build_assign (pmn2, MINUS_EXPR, p, n2);
3680 0 : insert_before (g);
3681 : }
3682 0 : g = gimple_build_cond (GT_EXPR, pmn2, pmn3, NULL_TREE, NULL_TREE);
3683 : }
3684 : else
3685 405 : g = gimple_build_cond (LT_EXPR, n2, pmn3, NULL_TREE, NULL_TREE);
3686 405 : edge edge_true, edge_false;
3687 405 : if_then (g, profile_probability::likely (), edge_true, edge_false);
3688 405 : tree idx_next;
3689 810 : tree idx = create_loop (bitint_big_endian ? pmn2 : n2, &idx_next);
3690 405 : tree idxmn2 = make_ssa_name (sizetype);
3691 405 : tree idxpn3 = make_ssa_name (sizetype);
3692 810 : g = gimple_build_assign (idxmn2,
3693 : bitint_big_endian ? PLUS_EXPR : MINUS_EXPR,
3694 : idx, n2);
3695 405 : insert_before (g);
3696 810 : g = gimple_build_assign (idxpn3,
3697 : bitint_big_endian ? MINUS_EXPR : PLUS_EXPR,
3698 : idx, n3);
3699 405 : insert_before (g);
3700 405 : m_data_cnt = 0;
3701 405 : tree t1 = handle_operand (rhs1, idx);
3702 405 : m_first = false;
3703 405 : g = gimple_build_assign (make_ssa_name (m_limb_type),
3704 : RSHIFT_EXPR, t1, n1);
3705 405 : insert_before (g);
3706 405 : t1 = gimple_assign_lhs (g);
3707 405 : if (!integer_zerop (n3))
3708 : {
3709 317 : m_data_cnt = 0;
3710 317 : tree t2 = handle_operand (rhs1, idxpn3);
3711 317 : g = gimple_build_assign (make_ssa_name (m_limb_type),
3712 : LSHIFT_EXPR, t2, n4);
3713 317 : insert_before (g);
3714 317 : t2 = gimple_assign_lhs (g);
3715 317 : g = gimple_build_assign (make_ssa_name (m_limb_type),
3716 : BIT_IOR_EXPR, t1, t2);
3717 317 : insert_before (g);
3718 317 : t1 = gimple_assign_lhs (g);
3719 : }
3720 405 : tree l = limb_access (TREE_TYPE (lhs), obj, idxmn2, true);
3721 405 : g = gimple_build_assign (l, t1);
3722 405 : insert_before (g);
3723 405 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
3724 0 : bitint_big_endian ? size_int (-1)
3725 : : size_one_node);
3726 405 : insert_before (g);
3727 810 : g = gimple_build_cond (bitint_big_endian ? GT_EXPR : LT_EXPR,
3728 : idx_next, pmn3, NULL_TREE, NULL_TREE);
3729 405 : insert_before (g);
3730 405 : idx = make_ssa_name (sizetype);
3731 405 : m_gsi = gsi_for_stmt (final_stmt);
3732 405 : gphi *phi = create_phi_node (idx, gsi_bb (m_gsi));
3733 405 : edge_false = find_edge (edge_false->src, gsi_bb (m_gsi));
3734 405 : edge_true = EDGE_PRED (gsi_bb (m_gsi),
3735 : EDGE_PRED (gsi_bb (m_gsi), 0) == edge_false);
3736 810 : add_phi_arg (phi, bitint_big_endian ? pmn2 : n2, edge_false,
3737 : UNKNOWN_LOCATION);
3738 405 : add_phi_arg (phi, idx_next, edge_true, UNKNOWN_LOCATION);
3739 405 : m_data_cnt = 0;
3740 405 : tree ms = handle_operand (rhs1, bitint_big_endian ? size_zero_node : p);
3741 405 : tree ext = ms;
3742 405 : if (!types_compatible_p (TREE_TYPE (ms), m_limb_type))
3743 244 : ext = add_cast (m_limb_type, ms);
3744 623 : if (!(TYPE_UNSIGNED (type) && prec % limb_prec == 0)
3745 519 : && !integer_zerop (n3))
3746 : {
3747 263 : if (bitint_big_endian)
3748 0 : g = gimple_build_cond (GT_EXPR, idx, size_zero_node,
3749 : NULL_TREE, NULL_TREE);
3750 : else
3751 263 : g = gimple_build_cond (LT_EXPR, idx, p, NULL_TREE, NULL_TREE);
3752 263 : if_then (g, profile_probability::likely (), edge_true, edge_false);
3753 263 : m_data_cnt = 0;
3754 263 : t1 = handle_operand (rhs1, idx);
3755 263 : g = gimple_build_assign (make_ssa_name (m_limb_type),
3756 : RSHIFT_EXPR, t1, n1);
3757 263 : insert_before (g);
3758 263 : t1 = gimple_assign_lhs (g);
3759 263 : g = gimple_build_assign (make_ssa_name (m_limb_type),
3760 : LSHIFT_EXPR, ext, n4);
3761 263 : insert_before (g);
3762 263 : tree t2 = gimple_assign_lhs (g);
3763 263 : g = gimple_build_assign (make_ssa_name (m_limb_type),
3764 : BIT_IOR_EXPR, t1, t2);
3765 263 : insert_before (g);
3766 263 : t1 = gimple_assign_lhs (g);
3767 263 : idxmn2 = make_ssa_name (sizetype);
3768 526 : g = gimple_build_assign (idxmn2, bitint_big_endian
3769 : ? PLUS_EXPR : MINUS_EXPR, idx, n2);
3770 263 : insert_before (g);
3771 263 : l = limb_access (TREE_TYPE (lhs), obj, idxmn2, true);
3772 263 : g = gimple_build_assign (l, t1);
3773 263 : insert_before (g);
3774 263 : idx_next = make_ssa_name (sizetype);
3775 263 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
3776 : bitint_big_endian
3777 0 : ? size_int (-1) : size_one_node);
3778 263 : insert_before (g);
3779 263 : m_gsi = gsi_for_stmt (final_stmt);
3780 263 : tree nidx = make_ssa_name (sizetype);
3781 263 : phi = create_phi_node (nidx, gsi_bb (m_gsi));
3782 263 : edge_false = find_edge (edge_false->src, gsi_bb (m_gsi));
3783 263 : edge_true = EDGE_PRED (gsi_bb (m_gsi),
3784 : EDGE_PRED (gsi_bb (m_gsi), 0) == edge_false);
3785 263 : add_phi_arg (phi, idx, edge_false, UNKNOWN_LOCATION);
3786 263 : add_phi_arg (phi, idx_next, edge_true, UNKNOWN_LOCATION);
3787 263 : idx = nidx;
3788 : }
3789 810 : g = gimple_build_assign (make_ssa_name (sizetype),
3790 : bitint_big_endian ? PLUS_EXPR : MINUS_EXPR,
3791 : idx, n2);
3792 405 : insert_before (g);
3793 405 : idx = gimple_assign_lhs (g);
3794 405 : tree sext = ext;
3795 405 : if (!TYPE_UNSIGNED (type))
3796 187 : sext = add_cast (signed_type_for (m_limb_type), ext);
3797 405 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (sext)),
3798 : RSHIFT_EXPR, sext, n1);
3799 405 : insert_before (g);
3800 405 : t1 = gimple_assign_lhs (g);
3801 405 : if (!TYPE_UNSIGNED (type))
3802 : {
3803 187 : t1 = add_cast (m_limb_type, t1);
3804 187 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (sext)),
3805 : RSHIFT_EXPR, sext,
3806 187 : build_int_cst (TREE_TYPE (n),
3807 187 : limb_prec - 1));
3808 187 : insert_before (g);
3809 187 : ext = add_cast (m_limb_type, gimple_assign_lhs (g));
3810 : }
3811 : else
3812 218 : ext = build_zero_cst (m_limb_type);
3813 405 : l = limb_access (TREE_TYPE (lhs), obj, idx, true);
3814 405 : g = gimple_build_assign (l, t1);
3815 405 : insert_before (g);
3816 405 : g = gimple_build_assign (make_ssa_name (sizetype), PLUS_EXPR, idx,
3817 : bitint_big_endian
3818 0 : ? size_int (-1) : size_one_node);
3819 405 : insert_before (g);
3820 405 : tree p2 = p;
3821 405 : if (bitint_big_endian)
3822 : {
3823 0 : tree new_idx = gimple_assign_lhs (g);
3824 0 : g = gimple_build_cond (NE_EXPR, idx, size_zero_node,
3825 : NULL_TREE, NULL_TREE);
3826 0 : idx = new_idx;
3827 : }
3828 : else
3829 : {
3830 405 : if (bitint_extended == bitint_ext_full
3831 0 : && abi_limb_prec > limb_prec)
3832 0 : p2 = build_int_cst (sizetype,
3833 0 : CEIL (prec, abi_limb_prec)
3834 0 : * abi_limb_prec / limb_prec - 1);
3835 405 : idx = gimple_assign_lhs (g);
3836 405 : g = gimple_build_cond (LE_EXPR, idx, p2, NULL_TREE, NULL_TREE);
3837 : }
3838 405 : if_then (g, profile_probability::likely (), edge_true, edge_false);
3839 405 : idx = create_loop (idx, &idx_next);
3840 405 : l = limb_access (TREE_TYPE (lhs), obj, idx, true);
3841 405 : g = gimple_build_assign (l, ext);
3842 405 : insert_before (g);
3843 405 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
3844 : bitint_big_endian
3845 0 : ? size_int (-1) : size_one_node);
3846 405 : insert_before (g);
3847 405 : if (bitint_big_endian)
3848 0 : g = gimple_build_cond (NE_EXPR, idx, size_zero_node,
3849 : NULL_TREE, NULL_TREE);
3850 : else
3851 405 : g = gimple_build_cond (LE_EXPR, idx_next, p2, NULL_TREE, NULL_TREE);
3852 405 : insert_before (g);
3853 : }
3854 : else
3855 : {
3856 : /* Lower
3857 : dst = src << n;
3858 : as
3859 : unsigned n1 = n % limb_prec;
3860 : size_t n2 = n / limb_prec;
3861 : size_t n3 = n1 != 0;
3862 : unsigned n4 = (limb_prec - n1) % limb_prec;
3863 : size_t idx;
3864 : size_t p = prec / limb_prec - (prec % limb_prec == 0);
3865 : for (idx = p; (ssize_t) idx >= (ssize_t) (n2 + n3); --idx)
3866 : dst[idx] = (src[idx - n2] << n1) | (src[idx - n2 - n3] >> n4);
3867 : if (n1)
3868 : {
3869 : dst[idx] = src[idx - n2] << n1;
3870 : --idx;
3871 : }
3872 : for (; (ssize_t) idx >= 0; --idx)
3873 : dst[idx] = 0; */
3874 197 : tree n2pn3;
3875 197 : if (TREE_CODE (n2) == INTEGER_CST && TREE_CODE (n3) == INTEGER_CST)
3876 71 : n2pn3 = int_const_binop (PLUS_EXPR, n2, n3);
3877 : else
3878 : {
3879 126 : n2pn3 = make_ssa_name (sizetype);
3880 126 : g = gimple_build_assign (n2pn3, PLUS_EXPR, n2, n3);
3881 126 : insert_before (g);
3882 : }
3883 197 : if (bitint_big_endian)
3884 : {
3885 0 : if (TREE_CODE (n2pn3) == INTEGER_CST)
3886 0 : n2pn3 = int_const_binop (MINUS_EXPR, p, n2pn3);
3887 : else
3888 : {
3889 0 : g = gimple_build_assign (make_ssa_name (sizetype),
3890 : MINUS_EXPR, p, n2pn3);
3891 0 : insert_before (g);
3892 0 : n2pn3 = gimple_assign_lhs (g);
3893 : }
3894 : }
3895 : /* For LSHIFT_EXPR, we can use handle_operand with non-INTEGER_CST
3896 : idx even to access the most significant partial limb. */
3897 197 : m_var_msb = true;
3898 197 : if (integer_zerop (n3))
3899 : /* For n3 == 0 p >= n2 + n3 is always true for all valid shift
3900 : counts. Emit if (true) condition that can be optimized later. */
3901 45 : g = gimple_build_cond (NE_EXPR, boolean_true_node, boolean_false_node,
3902 : NULL_TREE, NULL_TREE);
3903 152 : else if (bitint_big_endian)
3904 0 : g = gimple_build_cond (NE_EXPR, n2pn3, size_int (-1), NULL_TREE,
3905 : NULL_TREE);
3906 : else
3907 152 : g = gimple_build_cond (LE_EXPR, n2pn3, p, NULL_TREE, NULL_TREE);
3908 197 : edge edge_true, edge_false;
3909 197 : if_then (g, profile_probability::likely (), edge_true, edge_false);
3910 197 : tree idx_next;
3911 197 : tree idx = create_loop (bitint_big_endian ? size_zero_node : p,
3912 : &idx_next);
3913 197 : tree idxmn2 = make_ssa_name (sizetype);
3914 197 : tree idxmn2mn3 = make_ssa_name (sizetype);
3915 394 : g = gimple_build_assign (idxmn2,
3916 : bitint_big_endian ? PLUS_EXPR : MINUS_EXPR,
3917 : idx, n2);
3918 197 : insert_before (g);
3919 394 : g = gimple_build_assign (idxmn2mn3,
3920 : bitint_big_endian ? PLUS_EXPR : MINUS_EXPR,
3921 : idxmn2, n3);
3922 197 : insert_before (g);
3923 197 : m_data_cnt = 0;
3924 197 : tree t1 = handle_operand (rhs1, idxmn2);
3925 197 : m_first = false;
3926 197 : g = gimple_build_assign (make_ssa_name (m_limb_type),
3927 : LSHIFT_EXPR, t1, n1);
3928 197 : insert_before (g);
3929 197 : t1 = gimple_assign_lhs (g);
3930 197 : if (!integer_zerop (n3))
3931 : {
3932 152 : m_data_cnt = 0;
3933 152 : tree t2 = handle_operand (rhs1, idxmn2mn3);
3934 152 : g = gimple_build_assign (make_ssa_name (m_limb_type),
3935 : RSHIFT_EXPR, t2, n4);
3936 152 : insert_before (g);
3937 152 : t2 = gimple_assign_lhs (g);
3938 152 : g = gimple_build_assign (make_ssa_name (m_limb_type),
3939 : BIT_IOR_EXPR, t1, t2);
3940 152 : insert_before (g);
3941 152 : t1 = gimple_assign_lhs (g);
3942 : }
3943 197 : tree l = limb_access (TREE_TYPE (lhs), obj, idx, true);
3944 197 : g = gimple_build_assign (l, t1);
3945 197 : insert_before (g);
3946 394 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
3947 : bitint_big_endian
3948 197 : ? size_one_node : size_int (-1));
3949 197 : insert_before (g);
3950 197 : tree sn2pn3 = add_cast (ssizetype, n2pn3);
3951 394 : g = gimple_build_cond (bitint_big_endian ? LE_EXPR : GE_EXPR,
3952 : add_cast (ssizetype, idx_next), sn2pn3,
3953 : NULL_TREE, NULL_TREE);
3954 197 : insert_before (g);
3955 197 : idx = make_ssa_name (sizetype);
3956 197 : m_gsi = gsi_for_stmt (final_stmt);
3957 197 : gphi *phi = create_phi_node (idx, gsi_bb (m_gsi));
3958 197 : edge_false = find_edge (edge_false->src, gsi_bb (m_gsi));
3959 197 : edge_true = EDGE_PRED (gsi_bb (m_gsi),
3960 : EDGE_PRED (gsi_bb (m_gsi), 0) == edge_false);
3961 197 : add_phi_arg (phi, bitint_big_endian ? size_zero_node : p,
3962 : edge_false, UNKNOWN_LOCATION);
3963 197 : add_phi_arg (phi, idx_next, edge_true, UNKNOWN_LOCATION);
3964 197 : m_data_cnt = 0;
3965 197 : if (!integer_zerop (n3))
3966 : {
3967 152 : g = gimple_build_cond (NE_EXPR, n3, size_zero_node,
3968 : NULL_TREE, NULL_TREE);
3969 152 : if_then (g, profile_probability::likely (), edge_true, edge_false);
3970 152 : idxmn2 = make_ssa_name (sizetype);
3971 304 : g = gimple_build_assign (idxmn2,
3972 : bitint_big_endian ? PLUS_EXPR : MINUS_EXPR,
3973 : idx, n2);
3974 152 : insert_before (g);
3975 152 : m_data_cnt = 0;
3976 152 : t1 = handle_operand (rhs1, idxmn2);
3977 152 : g = gimple_build_assign (make_ssa_name (m_limb_type),
3978 : LSHIFT_EXPR, t1, n1);
3979 152 : insert_before (g);
3980 152 : t1 = gimple_assign_lhs (g);
3981 152 : l = limb_access (TREE_TYPE (lhs), obj, idx, true);
3982 152 : g = gimple_build_assign (l, t1);
3983 152 : insert_before (g);
3984 152 : idx_next = make_ssa_name (sizetype);
3985 304 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
3986 : bitint_big_endian
3987 152 : ? size_one_node : size_int (-1));
3988 152 : insert_before (g);
3989 152 : m_gsi = gsi_for_stmt (final_stmt);
3990 152 : tree nidx = make_ssa_name (sizetype);
3991 152 : phi = create_phi_node (nidx, gsi_bb (m_gsi));
3992 152 : edge_false = find_edge (edge_false->src, gsi_bb (m_gsi));
3993 152 : edge_true = EDGE_PRED (gsi_bb (m_gsi),
3994 : EDGE_PRED (gsi_bb (m_gsi), 0) == edge_false);
3995 152 : add_phi_arg (phi, idx, edge_false, UNKNOWN_LOCATION);
3996 152 : add_phi_arg (phi, idx_next, edge_true, UNKNOWN_LOCATION);
3997 152 : idx = nidx;
3998 : }
3999 197 : if (bitint_big_endian)
4000 0 : g = gimple_build_cond (LE_EXPR, idx, p, NULL_TREE, NULL_TREE);
4001 : else
4002 197 : g = gimple_build_cond (GE_EXPR, add_cast (ssizetype, idx),
4003 : ssize_int (0), NULL_TREE, NULL_TREE);
4004 197 : if_then (g, profile_probability::likely (), edge_true, edge_false);
4005 197 : idx = create_loop (idx, &idx_next);
4006 197 : l = limb_access (TREE_TYPE (lhs), obj, idx, true);
4007 197 : g = gimple_build_assign (l, build_zero_cst (m_limb_type));
4008 197 : insert_before (g);
4009 394 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
4010 : bitint_big_endian
4011 197 : ? size_one_node : size_int (-1));
4012 197 : insert_before (g);
4013 197 : if (bitint_big_endian)
4014 0 : g = gimple_build_cond (LE_EXPR, idx_next, p, NULL_TREE, NULL_TREE);
4015 : else
4016 197 : g = gimple_build_cond (GE_EXPR, add_cast (ssizetype, idx_next),
4017 : ssize_int (0), NULL_TREE, NULL_TREE);
4018 197 : insert_before (g);
4019 197 : if (bitint_extended && prec % limb_prec != 0)
4020 : {
4021 : /* The most significant limb has been updated either in the
4022 : loop or in the if after it. To simplify the code, just
4023 : read it back from memory and extend. */
4024 0 : m_gsi = gsi_after_labels (edge_false->dest);
4025 0 : idx = bitint_big_endian ? size_zero_node : p;
4026 0 : tree l = limb_access (TREE_TYPE (lhs), obj, idx, true);
4027 0 : tree type = limb_access_type (TREE_TYPE (lhs), idx);
4028 0 : tree v = make_ssa_name (m_limb_type);
4029 0 : g = gimple_build_assign (v, l);
4030 0 : insert_before (g);
4031 0 : v = add_cast (type, v);
4032 0 : l = limb_access (TREE_TYPE (lhs), obj, idx, true);
4033 0 : v = add_cast (m_limb_type, v);
4034 0 : g = gimple_build_assign (l, v);
4035 0 : insert_before (g);
4036 0 : if (bitint_extended == bitint_ext_full
4037 0 : && abi_limb_prec > limb_prec
4038 0 : && (CEIL (prec, abi_limb_prec) * abi_limb_prec
4039 0 : > CEIL (prec, limb_prec) * limb_prec))
4040 : {
4041 0 : tree p2 = build_int_cst (sizetype,
4042 : CEIL (prec, abi_limb_prec)
4043 0 : * abi_limb_prec / limb_prec - 1);
4044 0 : if (TYPE_UNSIGNED (TREE_TYPE (lhs)))
4045 0 : v = build_zero_cst (m_limb_type);
4046 : else
4047 : {
4048 0 : v = add_cast (signed_type_for (m_limb_type), v);
4049 0 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (v)),
4050 : RSHIFT_EXPR, v,
4051 : build_int_cst (unsigned_type_node,
4052 0 : limb_prec - 1));
4053 0 : insert_before (g);
4054 0 : v = add_cast (m_limb_type, gimple_assign_lhs (g));
4055 : }
4056 0 : l = limb_access (TREE_TYPE (lhs), obj, p2, true);
4057 0 : g = gimple_build_assign (l, v);
4058 0 : insert_before (g);
4059 : }
4060 : }
4061 197 : else if (bitint_extended == bitint_ext_full
4062 0 : && abi_limb_prec > limb_prec
4063 0 : && (CEIL (prec, abi_limb_prec) * abi_limb_prec
4064 0 : > CEIL (prec, limb_prec) * limb_prec))
4065 : {
4066 0 : m_gsi = gsi_after_labels (edge_false->dest);
4067 0 : tree p2 = build_int_cst (sizetype,
4068 : CEIL (prec, abi_limb_prec)
4069 0 : * abi_limb_prec / limb_prec - 1);
4070 0 : tree v;
4071 0 : if (TYPE_UNSIGNED (TREE_TYPE (lhs)))
4072 0 : v = build_zero_cst (m_limb_type);
4073 : else
4074 : {
4075 0 : tree l = limb_access (TREE_TYPE (lhs), obj, p, true);
4076 0 : v = make_ssa_name (m_limb_type);
4077 0 : g = gimple_build_assign (v, l);
4078 0 : insert_before (g);
4079 0 : v = add_cast (signed_type_for (m_limb_type), v);
4080 0 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (v)),
4081 : RSHIFT_EXPR, v,
4082 : build_int_cst (unsigned_type_node,
4083 0 : limb_prec - 1));
4084 0 : insert_before (g);
4085 0 : v = add_cast (m_limb_type, gimple_assign_lhs (g));
4086 : }
4087 0 : tree l = limb_access (TREE_TYPE (lhs), obj, p2, true);
4088 0 : g = gimple_build_assign (l, v);
4089 0 : insert_before (g);
4090 : }
4091 : }
4092 602 : }
4093 :
4094 : /* Lower large/huge _BitInt multiplication or division. */
4095 :
4096 : void
4097 373 : bitint_large_huge::lower_muldiv_stmt (tree obj, gimple *stmt)
4098 : {
4099 373 : tree rhs1 = gimple_assign_rhs1 (stmt);
4100 373 : tree rhs2 = gimple_assign_rhs2 (stmt);
4101 373 : tree lhs = gimple_assign_lhs (stmt);
4102 373 : tree_code rhs_code = gimple_assign_rhs_code (stmt);
4103 373 : tree type = TREE_TYPE (rhs1);
4104 373 : gcc_assert (BITINT_TYPE_P (type)
4105 : && bitint_precision_kind (type) >= bitint_prec_large);
4106 373 : int prec = TYPE_PRECISION (type), prec1, prec2;
4107 373 : bool ext_ms_limb = false;
4108 373 : bool do_ext = false;
4109 373 : rhs1 = handle_operand_addr (rhs1, stmt, NULL, &prec1);
4110 373 : rhs2 = handle_operand_addr (rhs2, stmt, NULL, &prec2);
4111 373 : if (obj == NULL_TREE)
4112 : {
4113 166 : int part = var_to_partition (m_map, lhs);
4114 166 : gcc_assert (m_vars[part] != NULL_TREE);
4115 166 : obj = m_vars[part];
4116 166 : lhs = build_fold_addr_expr (obj);
4117 : }
4118 : else
4119 : {
4120 207 : lhs = build_fold_addr_expr (obj);
4121 207 : lhs = force_gimple_operand_gsi (&m_gsi, lhs, true,
4122 : NULL_TREE, true, GSI_SAME_STMT);
4123 : }
4124 373 : if (bitint_extended && TYPE_OVERFLOW_WRAPS (type))
4125 : {
4126 0 : if (rhs_code == MULT_EXPR)
4127 : do_ext = true;
4128 : /* For signed division with -fwrapv, minimum negative / -1 needs
4129 : is minimum negative and the padding bits above it should be all
4130 : set. */
4131 0 : else if (!TYPE_UNSIGNED (type)
4132 0 : && (rhs_code == TRUNC_DIV_EXPR || rhs_code == EXACT_DIV_EXPR))
4133 373 : do_ext = true;
4134 : }
4135 373 : if (bitint_extended == bitint_ext_full
4136 0 : && abi_limb_prec > limb_prec
4137 0 : && (CEIL (prec, abi_limb_prec) * abi_limb_prec
4138 0 : > CEIL (prec, limb_prec) * limb_prec))
4139 : {
4140 : /* unsigned multiplication needs to wrap around, so we can't
4141 : increase prec. Similarly for -fwrapv. */
4142 0 : if (do_ext)
4143 : ext_ms_limb = true;
4144 : else
4145 0 : prec = CEIL (prec, abi_limb_prec) * abi_limb_prec;
4146 : }
4147 373 : tree sitype = lang_hooks.types.type_for_mode (SImode, 0);
4148 373 : gimple *g;
4149 373 : switch (rhs_code)
4150 : {
4151 210 : case MULT_EXPR:
4152 210 : g = gimple_build_call_internal (IFN_MULBITINT, 6,
4153 210 : lhs, build_int_cst (sitype, prec),
4154 210 : rhs1, build_int_cst (sitype, prec1),
4155 210 : rhs2, build_int_cst (sitype, prec2));
4156 210 : insert_before (g);
4157 210 : break;
4158 103 : case TRUNC_DIV_EXPR:
4159 103 : case EXACT_DIV_EXPR:
4160 103 : g = gimple_build_call_internal (IFN_DIVMODBITINT, 8,
4161 103 : lhs, build_int_cst (sitype, prec),
4162 : null_pointer_node,
4163 : build_int_cst (sitype, 0),
4164 103 : rhs1, build_int_cst (sitype, prec1),
4165 103 : rhs2, build_int_cst (sitype, prec2));
4166 103 : if (!stmt_ends_bb_p (stmt))
4167 102 : gimple_call_set_nothrow (as_a <gcall *> (g), true);
4168 103 : insert_before (g);
4169 103 : break;
4170 60 : case TRUNC_MOD_EXPR:
4171 60 : g = gimple_build_call_internal (IFN_DIVMODBITINT, 8, null_pointer_node,
4172 : build_int_cst (sitype, 0),
4173 60 : lhs, build_int_cst (sitype, prec),
4174 60 : rhs1, build_int_cst (sitype, prec1),
4175 60 : rhs2, build_int_cst (sitype, prec2));
4176 60 : if (!stmt_ends_bb_p (stmt))
4177 57 : gimple_call_set_nothrow (as_a <gcall *> (g), true);
4178 60 : insert_before (g);
4179 60 : break;
4180 0 : default:
4181 0 : gcc_unreachable ();
4182 : }
4183 373 : if (stmt_ends_bb_p (stmt))
4184 : {
4185 4 : maybe_duplicate_eh_stmt (g, stmt);
4186 4 : edge e1;
4187 4 : edge_iterator ei;
4188 4 : basic_block bb = gimple_bb (stmt);
4189 :
4190 4 : FOR_EACH_EDGE (e1, ei, bb->succs)
4191 4 : if (e1->flags & EDGE_EH)
4192 : break;
4193 4 : if (e1)
4194 : {
4195 4 : edge e2 = split_block (gsi_bb (m_gsi), g);
4196 4 : m_gsi = gsi_after_labels (e2->dest);
4197 4 : add_eh_edge (e2->src, e1);
4198 : }
4199 : }
4200 373 : if (do_ext
4201 373 : && ((prec % limb_prec) != 0 || (ext_ms_limb && !TYPE_UNSIGNED (type))))
4202 : {
4203 : /* Unsigned multiplication wraps, but libgcc function will return the
4204 : bits beyond prec within the top limb as another limb of the full
4205 : multiplication. So, clear the padding bits here. */
4206 0 : tree idx = size_int (bitint_big_endian ? 0 : prec / limb_prec);
4207 0 : tree l = limb_access (type, obj, idx, true);
4208 0 : tree ctype = limb_access_type (type, idx);
4209 0 : tree v = make_ssa_name (m_limb_type);
4210 0 : g = gimple_build_assign (v, l);
4211 0 : insert_before (g);
4212 0 : tree v2 = v;
4213 0 : if ((prec % limb_prec) != 0)
4214 : {
4215 0 : v = add_cast (ctype, v);
4216 0 : l = limb_access (type, obj, idx, true);
4217 0 : v = add_cast (m_limb_type, v);
4218 0 : v2 = v;
4219 0 : g = gimple_build_assign (l, v);
4220 0 : insert_before (g);
4221 : }
4222 0 : if (ext_ms_limb && !TYPE_UNSIGNED (type))
4223 : {
4224 0 : v2 = add_cast (signed_type_for (m_limb_type), v2);
4225 0 : tree lpm1 = build_int_cst (unsigned_type_node, limb_prec - 1);
4226 0 : v = make_ssa_name (TREE_TYPE (v2));
4227 0 : g = gimple_build_assign (v, RSHIFT_EXPR, v2, lpm1);
4228 0 : insert_before (g);
4229 0 : unsigned int i
4230 0 : = CEIL (prec, abi_limb_prec) * abi_limb_prec / limb_prec;
4231 0 : v = add_cast (m_limb_type, v);
4232 0 : g = gimple_build_assign (limb_access (type, obj, size_int (i - 1),
4233 : true), v);
4234 0 : insert_before (g);
4235 0 : ext_ms_limb = false;
4236 : }
4237 : }
4238 373 : if (ext_ms_limb)
4239 : {
4240 0 : unsigned int i = CEIL (prec, abi_limb_prec) * abi_limb_prec / limb_prec;
4241 0 : g = gimple_build_assign (limb_access (type, obj, size_int (i - 1), true),
4242 : build_zero_cst (m_limb_type));
4243 0 : insert_before (g);
4244 : }
4245 373 : }
4246 :
4247 : /* Lower large/huge _BitInt conversion to/from floating point. */
4248 :
4249 : void
4250 327 : bitint_large_huge::lower_float_conv_stmt (tree obj, gimple *stmt)
4251 : {
4252 327 : tree rhs1 = gimple_assign_rhs1 (stmt);
4253 327 : tree lhs = gimple_assign_lhs (stmt);
4254 327 : tree_code rhs_code = gimple_assign_rhs_code (stmt);
4255 327 : tree sitype = lang_hooks.types.type_for_mode (SImode, 0);
4256 327 : gimple *g;
4257 327 : if (rhs_code == FIX_TRUNC_EXPR)
4258 : {
4259 179 : tree type = TREE_TYPE (lhs);
4260 179 : int prec = TYPE_PRECISION (type);
4261 179 : bool extend_ms_limb = false;
4262 179 : if (bitint_extended == bitint_ext_full
4263 0 : && abi_limb_prec > limb_prec
4264 0 : && (CEIL (prec, abi_limb_prec) * abi_limb_prec
4265 0 : > CEIL (prec, limb_prec) * limb_prec))
4266 179 : extend_ms_limb = true;
4267 179 : if (!TYPE_UNSIGNED (type))
4268 93 : prec = -prec;
4269 179 : if (obj == NULL_TREE)
4270 : {
4271 135 : int part = var_to_partition (m_map, lhs);
4272 135 : gcc_assert (m_vars[part] != NULL_TREE);
4273 135 : obj = m_vars[part];
4274 135 : lhs = build_fold_addr_expr (obj);
4275 : }
4276 : else
4277 : {
4278 44 : lhs = build_fold_addr_expr (obj);
4279 44 : lhs = force_gimple_operand_gsi (&m_gsi, lhs, true,
4280 : NULL_TREE, true, GSI_SAME_STMT);
4281 : }
4282 179 : scalar_mode from_mode
4283 179 : = as_a <scalar_mode> (TYPE_MODE (TREE_TYPE (rhs1)));
4284 : #ifdef HAVE_SFmode
4285 : /* IEEE single is a full superset of both IEEE half and
4286 : bfloat formats, convert to float first and then to _BitInt
4287 : to avoid the need of another 2 library routines. */
4288 179 : if ((REAL_MODE_FORMAT (from_mode) == &arm_bfloat_half_format
4289 179 : || REAL_MODE_FORMAT (from_mode) == &ieee_half_format)
4290 191 : && REAL_MODE_FORMAT (SFmode) == &ieee_single_format)
4291 : {
4292 12 : tree type = lang_hooks.types.type_for_mode (SFmode, 0);
4293 12 : if (type)
4294 12 : rhs1 = add_cast (type, rhs1);
4295 : }
4296 : #endif
4297 179 : g = gimple_build_call_internal (IFN_FLOATTOBITINT, 3,
4298 179 : lhs, build_int_cst (sitype, prec),
4299 : rhs1);
4300 179 : insert_before (g);
4301 179 : if (extend_ms_limb)
4302 : {
4303 0 : unsigned int i
4304 0 : = (CEIL (prec < 0 ? -prec : prec, abi_limb_prec)
4305 0 : * abi_limb_prec / limb_prec);
4306 0 : tree val;
4307 0 : if (prec < 0)
4308 : {
4309 0 : g = gimple_build_assign (make_ssa_name (m_limb_type),
4310 : limb_access (type, obj,
4311 0 : size_int (i - 2),
4312 : true));
4313 0 : insert_before (g);
4314 0 : val = add_cast (signed_type_for (m_limb_type),
4315 : gimple_assign_lhs (g));
4316 0 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (val)),
4317 : RSHIFT_EXPR, val,
4318 : build_int_cst (unsigned_type_node,
4319 0 : limb_prec - 1));
4320 0 : insert_before (g);
4321 0 : val = add_cast (m_limb_type, gimple_assign_lhs (g));
4322 : }
4323 : else
4324 0 : val = build_zero_cst (m_limb_type);
4325 0 : g = gimple_build_assign (limb_access (type, obj, size_int (i - 1),
4326 : true), val);
4327 0 : insert_before (g);
4328 : }
4329 : }
4330 : else
4331 : {
4332 148 : int prec;
4333 148 : rhs1 = handle_operand_addr (rhs1, stmt, NULL, &prec);
4334 148 : g = gimple_build_call_internal (IFN_BITINTTOFLOAT, 2,
4335 148 : rhs1, build_int_cst (sitype, prec));
4336 148 : gimple_call_set_lhs (g, lhs);
4337 148 : if (!stmt_ends_bb_p (stmt))
4338 147 : gimple_call_set_nothrow (as_a <gcall *> (g), true);
4339 148 : gsi_replace (&m_gsi, g, true);
4340 : }
4341 327 : }
4342 :
4343 : /* Helper method for lower_addsub_overflow and lower_mul_overflow.
4344 : If check_zero is true, caller wants to check if all bits in [start, end)
4345 : are zero, otherwise if bits in [start, end) are either all zero or
4346 : all ones. L is the limb with index LIMB, START and END are measured
4347 : in bits. */
4348 :
4349 : tree
4350 6453 : bitint_large_huge::arith_overflow_extract_bits (unsigned int start,
4351 : unsigned int end, tree l,
4352 : unsigned int limb,
4353 : bool check_zero)
4354 : {
4355 6453 : unsigned startlimb = start / limb_prec;
4356 6453 : unsigned endlimb = (end - 1) / limb_prec;
4357 6453 : gimple *g;
4358 :
4359 6453 : if ((start % limb_prec) == 0 && (end % limb_prec) == 0)
4360 : return l;
4361 6184 : if (startlimb == endlimb && limb == startlimb)
4362 : {
4363 2039 : if (check_zero)
4364 : {
4365 1486 : wide_int w = wi::shifted_mask (start % limb_prec,
4366 1486 : end - start, false, limb_prec);
4367 2972 : g = gimple_build_assign (make_ssa_name (m_limb_type),
4368 : BIT_AND_EXPR, l,
4369 1486 : wide_int_to_tree (m_limb_type, w));
4370 1486 : insert_before (g);
4371 1486 : return gimple_assign_lhs (g);
4372 1486 : }
4373 553 : unsigned int shift = start % limb_prec;
4374 553 : if ((end % limb_prec) != 0)
4375 : {
4376 336 : unsigned int lshift = (-end) % limb_prec;
4377 336 : shift += lshift;
4378 336 : g = gimple_build_assign (make_ssa_name (m_limb_type),
4379 : LSHIFT_EXPR, l,
4380 : build_int_cst (unsigned_type_node,
4381 336 : lshift));
4382 336 : insert_before (g);
4383 336 : l = gimple_assign_lhs (g);
4384 : }
4385 553 : l = add_cast (signed_type_for (m_limb_type), l);
4386 553 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (l)),
4387 : RSHIFT_EXPR, l,
4388 553 : build_int_cst (unsigned_type_node, shift));
4389 553 : insert_before (g);
4390 553 : return add_cast (m_limb_type, gimple_assign_lhs (g));
4391 : }
4392 4145 : else if (limb == startlimb)
4393 : {
4394 1990 : if ((start % limb_prec) == 0)
4395 : return l;
4396 1876 : if (!check_zero)
4397 950 : l = add_cast (signed_type_for (m_limb_type), l);
4398 1876 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (l)),
4399 : RSHIFT_EXPR, l,
4400 : build_int_cst (unsigned_type_node,
4401 1876 : start % limb_prec));
4402 1876 : insert_before (g);
4403 1876 : l = gimple_assign_lhs (g);
4404 1876 : if (!check_zero)
4405 950 : l = add_cast (m_limb_type, l);
4406 : return l;
4407 : }
4408 2155 : else if (limb == endlimb)
4409 : {
4410 1704 : if ((end % limb_prec) == 0)
4411 : return l;
4412 1687 : if (check_zero)
4413 : {
4414 880 : wide_int w = wi::mask (end % limb_prec, false, limb_prec);
4415 1760 : g = gimple_build_assign (make_ssa_name (m_limb_type),
4416 : BIT_AND_EXPR, l,
4417 880 : wide_int_to_tree (m_limb_type, w));
4418 880 : insert_before (g);
4419 880 : return gimple_assign_lhs (g);
4420 880 : }
4421 807 : unsigned int shift = (-end) % limb_prec;
4422 807 : g = gimple_build_assign (make_ssa_name (m_limb_type),
4423 : LSHIFT_EXPR, l,
4424 807 : build_int_cst (unsigned_type_node, shift));
4425 807 : insert_before (g);
4426 807 : l = add_cast (signed_type_for (m_limb_type), gimple_assign_lhs (g));
4427 807 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (l)),
4428 : RSHIFT_EXPR, l,
4429 807 : build_int_cst (unsigned_type_node, shift));
4430 807 : insert_before (g);
4431 807 : return add_cast (m_limb_type, gimple_assign_lhs (g));
4432 : }
4433 : return l;
4434 : }
4435 :
4436 : /* Helper method for lower_addsub_overflow and lower_mul_overflow. Store
4437 : result including overflow flag into the right locations. */
4438 :
4439 : void
4440 4212 : bitint_large_huge::finish_arith_overflow (tree var, tree obj, tree type,
4441 : tree ovf, tree lhs, tree orig_obj,
4442 : gimple *stmt, unsigned nelts,
4443 : tree_code code)
4444 : {
4445 4212 : gimple *g;
4446 :
4447 4212 : if (obj == NULL_TREE
4448 4212 : && (!BITINT_TYPE_P (type)
4449 231 : || bitint_precision_kind (type) < bitint_prec_large))
4450 : {
4451 : /* Add support for 3 or more limbs filled in from normal integral
4452 : type if this assert fails. If no target chooses limb mode smaller
4453 : than half of largest supported normal integral type, this will not
4454 : be needed. */
4455 250 : gcc_assert (TYPE_PRECISION (type) <= 2 * limb_prec);
4456 250 : tree lhs_type = type;
4457 110 : if (BITINT_TYPE_P (type)
4458 250 : && bitint_precision_kind (type) == bitint_prec_middle)
4459 46 : lhs_type = build_nonstandard_integer_type (TYPE_PRECISION (type),
4460 46 : TYPE_UNSIGNED (type));
4461 250 : tree r1 = limb_access (NULL_TREE, var,
4462 : bitint_big_endian
4463 0 : ? size_int (nelts - 1) : size_zero_node, true);
4464 250 : g = gimple_build_assign (make_ssa_name (m_limb_type), r1);
4465 250 : insert_before (g);
4466 250 : r1 = gimple_assign_lhs (g);
4467 250 : if (!useless_type_conversion_p (lhs_type, TREE_TYPE (r1)))
4468 250 : r1 = add_cast (lhs_type, r1);
4469 250 : if (TYPE_PRECISION (lhs_type) > limb_prec)
4470 : {
4471 90 : tree r2 = limb_access (NULL_TREE, var,
4472 : bitint_big_endian
4473 0 : ? size_int (nelts - 2) : size_one_node, true);
4474 90 : g = gimple_build_assign (make_ssa_name (m_limb_type), r2);
4475 90 : insert_before (g);
4476 90 : r2 = gimple_assign_lhs (g);
4477 90 : r2 = add_cast (lhs_type, r2);
4478 90 : g = gimple_build_assign (make_ssa_name (lhs_type), LSHIFT_EXPR, r2,
4479 : build_int_cst (unsigned_type_node,
4480 90 : limb_prec));
4481 90 : insert_before (g);
4482 90 : g = gimple_build_assign (make_ssa_name (lhs_type), BIT_IOR_EXPR, r1,
4483 : gimple_assign_lhs (g));
4484 90 : insert_before (g);
4485 90 : r1 = gimple_assign_lhs (g);
4486 : }
4487 250 : if (lhs_type != type)
4488 46 : r1 = add_cast (type, r1);
4489 250 : ovf = add_cast (lhs_type, ovf);
4490 250 : if (lhs_type != type)
4491 46 : ovf = add_cast (type, ovf);
4492 250 : g = gimple_build_assign (lhs, COMPLEX_EXPR, r1, ovf);
4493 250 : m_gsi = gsi_for_stmt (stmt);
4494 250 : gsi_replace (&m_gsi, g, true);
4495 : }
4496 : else
4497 : {
4498 3962 : unsigned HOST_WIDE_INT obj_nelts = 0;
4499 3962 : tree atype = NULL_TREE;
4500 3962 : if (obj)
4501 : {
4502 3871 : obj_nelts = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (obj))) / limb_prec;
4503 3871 : if (orig_obj == NULL_TREE)
4504 2277 : obj_nelts >>= 1;
4505 3871 : atype = build_array_type_nelts (m_limb_type, obj_nelts);
4506 : }
4507 3962 : if (bitint_extended && (var || obj))
4508 : {
4509 0 : unsigned prec = TYPE_PRECISION (type);
4510 0 : unsigned prec_limbs = CEIL (prec, limb_prec);
4511 0 : bool ext_ms_limb
4512 : = (bitint_extended == bitint_ext_full
4513 0 : && abi_limb_prec > limb_prec
4514 0 : && (CEIL (prec, abi_limb_prec) * abi_limb_prec
4515 0 : > CEIL (prec, limb_prec) * limb_prec));
4516 : /* For .{ADD,SUB}_OVERFLOW the partial limb if any is
4517 : already extended in lower_addsub_overflow. */
4518 0 : if ((code == MULT_EXPR && (prec % limb_prec) != 0)
4519 0 : || (ext_ms_limb && !TYPE_UNSIGNED (type)))
4520 : {
4521 0 : tree plm1idx = size_int (bitint_big_endian
4522 : ? nelts - obj_nelts : prec_limbs - 1);
4523 0 : tree plm1type
4524 0 : = limb_access_type (type, bitint_big_endian
4525 : ? size_zero_node : plm1idx);
4526 0 : tree l = limb_access (bitint_big_endian ? NULL_TREE : type,
4527 : var ? var : obj, plm1idx, true);
4528 0 : tree rhs = make_ssa_name (TREE_TYPE (l));
4529 0 : g = gimple_build_assign (rhs, l);
4530 0 : insert_before (g);
4531 0 : if (code == MULT_EXPR && (prec % limb_prec) != 0)
4532 : {
4533 0 : if (!useless_type_conversion_p (plm1type, TREE_TYPE (rhs)))
4534 0 : rhs = add_cast (plm1type, rhs);
4535 0 : if (!useless_type_conversion_p (TREE_TYPE (l),
4536 0 : TREE_TYPE (rhs)))
4537 0 : rhs = add_cast (TREE_TYPE (l), rhs);
4538 0 : l = limb_access (bitint_big_endian ? NULL_TREE : type,
4539 : var ? var : obj, plm1idx, true);
4540 0 : g = gimple_build_assign (l, rhs);
4541 0 : insert_before (g);
4542 : }
4543 0 : if (ext_ms_limb && !TYPE_UNSIGNED (type))
4544 : {
4545 0 : rhs = add_cast (signed_type_for (m_limb_type), rhs);
4546 0 : tree lpm1 = build_int_cst (unsigned_type_node,
4547 0 : limb_prec - 1);
4548 0 : tree v = make_ssa_name (TREE_TYPE (rhs));
4549 0 : g = gimple_build_assign (v, RSHIFT_EXPR, rhs, lpm1);
4550 0 : insert_before (g);
4551 0 : unsigned int i
4552 0 : = CEIL (prec, abi_limb_prec) * abi_limb_prec / limb_prec;
4553 0 : v = add_cast (m_limb_type, v);
4554 0 : g = gimple_build_assign (limb_access (type, var ? var : obj,
4555 0 : size_int (i - 1),
4556 : true), v);
4557 0 : insert_before (g);
4558 0 : ext_ms_limb = false;
4559 : }
4560 : }
4561 0 : if (ext_ms_limb)
4562 : {
4563 0 : unsigned int i
4564 0 : = CEIL (prec, abi_limb_prec) * abi_limb_prec / limb_prec;
4565 0 : g = gimple_build_assign (limb_access (type, var ? var : obj,
4566 0 : size_int (i - 1), true),
4567 : build_zero_cst (m_limb_type));
4568 0 : insert_before (g);
4569 : }
4570 : }
4571 3962 : if (var && obj)
4572 : {
4573 519 : tree v1, v2;
4574 519 : tree off;
4575 519 : if (orig_obj == NULL_TREE)
4576 : {
4577 32 : off = build_zero_cst (build_pointer_type (TREE_TYPE (obj)));
4578 32 : v1 = build2 (MEM_REF, atype,
4579 : build_fold_addr_expr (unshare_expr (obj)), off);
4580 : }
4581 487 : else if (!useless_type_conversion_p (atype, TREE_TYPE (obj)))
4582 9 : v1 = build1 (VIEW_CONVERT_EXPR, atype, unshare_expr (obj));
4583 : else
4584 478 : v1 = unshare_expr (obj);
4585 519 : off = build_int_cst (build_pointer_type (TREE_TYPE (var)),
4586 : bitint_big_endian
4587 519 : ? (nelts - obj_nelts) * m_limb_size : 0);
4588 519 : v2 = build2 (MEM_REF, atype, build_fold_addr_expr (var), off);
4589 519 : g = gimple_build_assign (v1, v2);
4590 519 : insert_before (g);
4591 : }
4592 3443 : else if (obj && bitint_big_endian && nelts != obj_nelts)
4593 : {
4594 0 : gcc_assert (nelts > obj_nelts);
4595 0 : tree fn = builtin_decl_implicit (BUILT_IN_MEMMOVE);
4596 0 : tree off = build_int_cst (build_pointer_type (TREE_TYPE (obj)),
4597 0 : (nelts - obj_nelts) * m_limb_size);
4598 0 : tree src = build2 (MEM_REF, atype,
4599 : build_fold_addr_expr (unshare_expr (obj)), off);
4600 0 : g = gimple_build_call (fn, 3,
4601 : build_fold_addr_expr (unshare_expr (obj)),
4602 : build_fold_addr_expr (src),
4603 : build_int_cst (size_type_node,
4604 0 : obj_nelts * m_limb_size));
4605 0 : insert_before (g);
4606 : }
4607 3962 : if (orig_obj == NULL_TREE && obj)
4608 : {
4609 2277 : ovf = add_cast (m_limb_type, ovf);
4610 2277 : tree l = limb_access (NULL_TREE, obj,
4611 2277 : size_int (bitint_big_endian
4612 : ? obj_nelts * 2 - 1 : obj_nelts),
4613 : true);
4614 2277 : g = gimple_build_assign (l, ovf);
4615 2277 : insert_before (g);
4616 2277 : if (obj_nelts > 1)
4617 : {
4618 2277 : atype = build_array_type_nelts (m_limb_type, obj_nelts - 1);
4619 2277 : tree off = build_int_cst (build_pointer_type (TREE_TYPE (obj)),
4620 2277 : (obj_nelts + !bitint_big_endian)
4621 2277 : * m_limb_size);
4622 2277 : tree v1 = build2 (MEM_REF, atype,
4623 : build_fold_addr_expr (unshare_expr (obj)),
4624 : off);
4625 2277 : g = gimple_build_assign (v1, build_zero_cst (atype));
4626 2277 : insert_before (g);
4627 : }
4628 : }
4629 1685 : else if (TREE_CODE (TREE_TYPE (lhs)) == COMPLEX_TYPE)
4630 : {
4631 1640 : imm_use_iterator ui;
4632 1640 : use_operand_p use_p;
4633 1640 : FOR_EACH_IMM_USE_FAST (use_p, ui, lhs)
4634 : {
4635 1640 : g = USE_STMT (use_p);
4636 1640 : if (!is_gimple_assign (g)
4637 1640 : || gimple_assign_rhs_code (g) != IMAGPART_EXPR)
4638 0 : continue;
4639 1640 : tree lhs2 = gimple_assign_lhs (g);
4640 1640 : gimple *use_stmt;
4641 1640 : single_imm_use (lhs2, &use_p, &use_stmt);
4642 1640 : lhs2 = gimple_assign_lhs (use_stmt);
4643 1640 : gimple_stmt_iterator gsi = gsi_for_stmt (use_stmt);
4644 1640 : if (useless_type_conversion_p (TREE_TYPE (lhs2), TREE_TYPE (ovf)))
4645 1619 : g = gimple_build_assign (lhs2, ovf);
4646 : else
4647 21 : g = gimple_build_assign (lhs2, NOP_EXPR, ovf);
4648 1640 : gsi_replace (&gsi, g, true);
4649 1640 : if (gsi_stmt (m_gsi) == use_stmt)
4650 91 : m_gsi = gsi_for_stmt (g);
4651 1640 : break;
4652 1640 : }
4653 : }
4654 45 : else if (ovf != boolean_false_node)
4655 : {
4656 45 : g = gimple_build_cond (NE_EXPR, ovf, boolean_false_node,
4657 : NULL_TREE, NULL_TREE);
4658 45 : edge edge_true, edge_false;
4659 45 : if_then (g, profile_probability::very_unlikely (),
4660 : edge_true, edge_false);
4661 45 : tree zero = build_zero_cst (TREE_TYPE (lhs));
4662 45 : tree fn = ubsan_build_overflow_builtin (code, m_loc,
4663 45 : TREE_TYPE (lhs),
4664 : zero, zero, NULL);
4665 45 : force_gimple_operand_gsi (&m_gsi, fn, true, NULL_TREE,
4666 : true, GSI_SAME_STMT);
4667 45 : m_gsi = gsi_after_labels (edge_true->dest);
4668 : }
4669 : }
4670 4212 : if (var)
4671 : {
4672 781 : tree clobber = build_clobber (TREE_TYPE (var), CLOBBER_STORAGE_END);
4673 781 : g = gimple_build_assign (var, clobber);
4674 781 : gsi_insert_after (&m_gsi, g, GSI_SAME_STMT);
4675 : }
4676 4212 : }
4677 :
4678 : /* Helper function for lower_addsub_overflow and lower_mul_overflow.
4679 : Given precisions of result TYPE (PREC), argument 0 precision PREC0,
4680 : argument 1 precision PREC1 and minimum precision for the result
4681 : PREC2, compute *START, *END, *CHECK_ZERO and return OVF. */
4682 :
4683 : static tree
4684 4212 : arith_overflow (tree_code code, tree type, int prec, int prec0, int prec1,
4685 : int prec2, unsigned *start, unsigned *end, bool *check_zero)
4686 : {
4687 4212 : *start = 0;
4688 4212 : *end = 0;
4689 4212 : *check_zero = true;
4690 : /* Ignore this special rule for subtraction, even if both
4691 : prec0 >= 0 and prec1 >= 0, their subtraction can be negative
4692 : in infinite precision. */
4693 4212 : if (code != MINUS_EXPR && prec0 >= 0 && prec1 >= 0)
4694 : {
4695 : /* Result in [0, prec2) is unsigned, if prec > prec2,
4696 : all bits above it will be zero. */
4697 681 : if ((prec - !TYPE_UNSIGNED (type)) >= prec2)
4698 0 : return boolean_false_node;
4699 : else
4700 : {
4701 : /* ovf if any of bits in [start, end) is non-zero. */
4702 681 : *start = prec - !TYPE_UNSIGNED (type);
4703 681 : *end = prec2;
4704 : }
4705 : }
4706 3531 : else if (TYPE_UNSIGNED (type))
4707 : {
4708 : /* If result in [0, prec2) is signed and if prec > prec2,
4709 : all bits above it will be sign bit copies. */
4710 1950 : if (prec >= prec2)
4711 : {
4712 : /* ovf if bit prec - 1 is non-zero. */
4713 184 : *start = prec - 1;
4714 184 : *end = prec;
4715 : }
4716 : else
4717 : {
4718 : /* ovf if any of bits in [start, end) is non-zero. */
4719 1766 : *start = prec;
4720 1766 : *end = prec2;
4721 : }
4722 : }
4723 1581 : else if (prec >= prec2)
4724 2 : return boolean_false_node;
4725 : else
4726 : {
4727 : /* ovf if [start, end) bits aren't all zeros or all ones. */
4728 1579 : *start = prec - 1;
4729 1579 : *end = prec2;
4730 1579 : *check_zero = false;
4731 : }
4732 : return NULL_TREE;
4733 : }
4734 :
4735 : /* Lower a .{ADD,SUB}_OVERFLOW call with at least one large/huge _BitInt
4736 : argument or return type _Complex large/huge _BitInt. */
4737 :
4738 : void
4739 2733 : bitint_large_huge::lower_addsub_overflow (tree obj, gimple *stmt)
4740 : {
4741 2733 : tree arg0 = gimple_call_arg (stmt, 0);
4742 2733 : tree arg1 = gimple_call_arg (stmt, 1);
4743 2733 : tree lhs = gimple_call_lhs (stmt);
4744 2733 : gimple *g;
4745 :
4746 2733 : if (!lhs)
4747 : {
4748 0 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
4749 0 : gsi_remove (&gsi, true);
4750 0 : return;
4751 : }
4752 2733 : gimple *final_stmt = gsi_stmt (m_gsi);
4753 2733 : tree type = TREE_TYPE (lhs);
4754 2733 : if (TREE_CODE (type) == COMPLEX_TYPE)
4755 2703 : type = TREE_TYPE (type);
4756 2733 : int prec = TYPE_PRECISION (type);
4757 2733 : int prec0 = range_to_prec (arg0, stmt);
4758 2733 : int prec1 = range_to_prec (arg1, stmt);
4759 : /* If PREC0 >= 0 && PREC1 >= 0 and CODE is not MINUS_EXPR, PREC2 is
4760 : the minimum unsigned precision of any possible operation's
4761 : result, otherwise it is minimum signed precision.
4762 : Some examples:
4763 : If PREC0 or PREC1 is 8, it means that argument is [0, 0xff],
4764 : if PREC0 or PREC1 is 10, it means that argument is [0, 0x3ff],
4765 : if PREC0 or PREC1 is -8, it means that argument is [-0x80, 0x7f],
4766 : if PREC0 or PREC1 is -10, it means that argument is [-0x200, 0x1ff].
4767 : PREC0 CODE PREC1 RESULT PREC2 SIGNED vs. UNSIGNED
4768 : 8 + 8 [0, 0x1fe] 9 UNSIGNED
4769 : 8 + 10 [0, 0x4fe] 11 UNSIGNED
4770 : -8 + -8 [-0x100, 0xfe] 9 SIGNED
4771 : -8 + -10 [-0x280, 0x27e] 11 SIGNED
4772 : 8 + -8 [-0x80, 0x17e] 10 SIGNED
4773 : 8 + -10 [-0x200, 0x2fe] 11 SIGNED
4774 : 10 + -8 [-0x80, 0x47e] 12 SIGNED
4775 : 8 - 8 [-0xff, 0xff] 9 SIGNED
4776 : 8 - 10 [-0x3ff, 0xff] 11 SIGNED
4777 : 10 - 8 [-0xff, 0x3ff] 11 SIGNED
4778 : -8 - -8 [-0xff, 0xff] 9 SIGNED
4779 : -8 - -10 [-0x27f, 0x27f] 11 SIGNED
4780 : -10 - -8 [-0x27f, 0x27f] 11 SIGNED
4781 : 8 - -8 [-0x7f, 0x17f] 10 SIGNED
4782 : 8 - -10 [-0x1ff, 0x2ff] 11 SIGNED
4783 : 10 - -8 [-0x7f, 0x47f] 12 SIGNED
4784 : -8 - 8 [-0x17f, 0x7f] 10 SIGNED
4785 : -8 - 10 [-0x47f, 0x7f] 12 SIGNED
4786 : -10 - 8 [-0x2ff, 0x1ff] 11 SIGNED */
4787 2733 : int prec2 = MAX (prec0 < 0 ? -prec0 : prec0,
4788 : prec1 < 0 ? -prec1 : prec1);
4789 : /* If operands are either both signed or both unsigned,
4790 : we need just one additional bit. */
4791 3777 : prec2 = (((prec0 < 0) == (prec1 < 0)
4792 : /* If one operand is signed and one unsigned and
4793 : the signed one has larger precision, we need
4794 : just one extra bit, otherwise two. */
4795 707 : || (prec0 < 0 ? (prec2 == -prec0 && prec2 != prec1)
4796 337 : : (prec2 == -prec1 && prec2 != prec0)))
4797 2733 : ? prec2 + 1 : prec2 + 2);
4798 2733 : int prec3 = MAX (prec0 < 0 ? -prec0 : prec0,
4799 : prec1 < 0 ? -prec1 : prec1);
4800 2733 : prec3 = MAX (prec3, prec);
4801 2733 : tree var = NULL_TREE;
4802 2733 : tree orig_obj = obj;
4803 2733 : if (obj == NULL_TREE
4804 1753 : && BITINT_TYPE_P (type)
4805 1652 : && bitint_precision_kind (type) >= bitint_prec_large
4806 1536 : && m_names
4807 4245 : && bitmap_bit_p (m_names, SSA_NAME_VERSION (lhs)))
4808 : {
4809 1457 : int part = var_to_partition (m_map, lhs);
4810 1457 : gcc_assert (m_vars[part] != NULL_TREE);
4811 1457 : obj = m_vars[part];
4812 1457 : if (TREE_TYPE (lhs) == type)
4813 14 : orig_obj = obj;
4814 : }
4815 101 : if (!BITINT_TYPE_P (type)
4816 2733 : || bitint_precision_kind (type) < bitint_prec_large)
4817 : {
4818 217 : unsigned HOST_WIDE_INT nelts = CEIL (prec, limb_prec);
4819 217 : tree atype = build_array_type_nelts (m_limb_type, nelts);
4820 217 : var = create_tmp_var (atype);
4821 : }
4822 :
4823 2733 : enum tree_code code;
4824 2733 : switch (gimple_call_internal_fn (stmt))
4825 : {
4826 : case IFN_ADD_OVERFLOW:
4827 : case IFN_UBSAN_CHECK_ADD:
4828 : code = PLUS_EXPR;
4829 : break;
4830 1399 : case IFN_SUB_OVERFLOW:
4831 1399 : case IFN_UBSAN_CHECK_SUB:
4832 1399 : code = MINUS_EXPR;
4833 1399 : break;
4834 0 : default:
4835 0 : gcc_unreachable ();
4836 : }
4837 2733 : unsigned start, end;
4838 2733 : bool check_zero;
4839 2733 : tree ovf = arith_overflow (code, type, prec, prec0, prec1, prec2,
4840 : &start, &end, &check_zero);
4841 :
4842 2733 : unsigned startlimb, endlimb;
4843 2733 : if (ovf)
4844 : {
4845 : startlimb = ~0U;
4846 : endlimb = ~0U;
4847 : }
4848 : else
4849 : {
4850 2733 : startlimb = start / limb_prec;
4851 2733 : endlimb = (end - 1) / limb_prec;
4852 : }
4853 :
4854 2733 : int prec4 = ovf != NULL_TREE ? prec : prec3;
4855 2733 : bitint_prec_kind kind = bitint_precision_kind (prec4);
4856 2733 : unsigned cnt, rem = 0, fin = 0, nelts;
4857 2733 : tree idx = NULL_TREE, idx_first = NULL_TREE, idx_next = NULL_TREE;
4858 5466 : bool last_ovf = (ovf == NULL_TREE
4859 2733 : && CEIL (prec2, limb_prec) > CEIL (prec3, limb_prec));
4860 2733 : if (kind != bitint_prec_huge)
4861 1539 : nelts = cnt = CEIL (prec4, limb_prec) + last_ovf;
4862 : else
4863 : {
4864 1194 : rem = prec4 % (2 * limb_prec);
4865 1194 : fin = (prec4 - rem) / limb_prec;
4866 1194 : cnt = 2 + CEIL (rem, limb_prec) + last_ovf;
4867 1194 : nelts = fin + cnt - 2;
4868 1194 : idx = idx_first = create_loop (bitint_big_endian
4869 1194 : ? size_int (nelts - 1) : size_zero_node,
4870 : &idx_next);
4871 : }
4872 :
4873 2733 : if (kind == bitint_prec_huge)
4874 1194 : m_upwards_2limb = fin;
4875 2733 : m_upwards = true;
4876 :
4877 2733 : tree type0 = TREE_TYPE (arg0);
4878 2733 : tree type1 = TREE_TYPE (arg1);
4879 2733 : int prec5 = prec3;
4880 2733 : if (bitint_precision_kind (prec5) < bitint_prec_large)
4881 10 : prec5 = MAX (TYPE_PRECISION (type0), TYPE_PRECISION (type1));
4882 2733 : if (TYPE_PRECISION (type0) < prec5)
4883 : {
4884 146 : type0 = build_bitint_type (prec5, TYPE_UNSIGNED (type0));
4885 146 : if (TREE_CODE (arg0) == INTEGER_CST)
4886 27 : arg0 = fold_convert (type0, arg0);
4887 : }
4888 2733 : if (TYPE_PRECISION (type1) < prec5)
4889 : {
4890 156 : type1 = build_bitint_type (prec5, TYPE_UNSIGNED (type1));
4891 156 : if (TREE_CODE (arg1) == INTEGER_CST)
4892 76 : arg1 = fold_convert (type1, arg1);
4893 : }
4894 2733 : unsigned int data_cnt = 0;
4895 2733 : tree last_rhs1 = NULL_TREE, last_rhs2 = NULL_TREE;
4896 2733 : tree cmp = build_zero_cst (m_limb_type);
4897 2733 : unsigned prec_limbs = CEIL ((unsigned) prec, limb_prec);
4898 2733 : tree ovf_out = NULL_TREE, cmp_out = NULL_TREE;
4899 11904 : for (unsigned i = 0; i < cnt; i++)
4900 : {
4901 9171 : m_data_cnt = 0;
4902 9171 : tree rhs1, rhs2;
4903 9171 : if (kind != bitint_prec_huge)
4904 5303 : idx = size_int (bitint_big_endian ? nelts - 1 - i : i);
4905 3868 : else if (i >= 2)
4906 1480 : idx = size_int (bitint_big_endian ? nelts + 1 - fin - i : fin + i - 2);
4907 9171 : if (!last_ovf || i < cnt - 1)
4908 : {
4909 8225 : tree idx0 = idx, idx1 = idx;
4910 8225 : if (bitint_big_endian
4911 8225 : && CEIL ((unsigned) TYPE_PRECISION (type0), limb_prec) != nelts)
4912 : {
4913 0 : HOST_WIDE_INT diff
4914 0 : = ((HOST_WIDE_INT) CEIL (TYPE_PRECISION (type0), limb_prec)
4915 0 : - (HOST_WIDE_INT) nelts);
4916 0 : if (tree_fits_uhwi_p (idx))
4917 0 : idx0 = size_int (tree_to_uhwi (idx) + diff);
4918 : else
4919 : {
4920 0 : idx0 = make_ssa_name (sizetype);
4921 0 : g = gimple_build_assign (idx0, PLUS_EXPR, idx,
4922 0 : size_int (diff));
4923 0 : insert_before (g);
4924 : }
4925 : }
4926 8225 : if (type0 != TREE_TYPE (arg0))
4927 334 : rhs1 = handle_cast (type0, arg0, idx0);
4928 : else
4929 7891 : rhs1 = handle_operand (arg0, idx0);
4930 8225 : if (bitint_big_endian
4931 8225 : && CEIL ((unsigned) TYPE_PRECISION (type1), limb_prec) != nelts)
4932 : {
4933 0 : HOST_WIDE_INT diff
4934 0 : = ((HOST_WIDE_INT) CEIL (TYPE_PRECISION (type1), limb_prec)
4935 0 : - (HOST_WIDE_INT) nelts);
4936 0 : if (tree_fits_uhwi_p (idx))
4937 0 : idx1 = size_int (tree_to_uhwi (idx) + diff);
4938 : else
4939 : {
4940 0 : idx1 = make_ssa_name (sizetype);
4941 0 : g = gimple_build_assign (idx1, PLUS_EXPR, idx,
4942 0 : size_int (diff));
4943 0 : insert_before (g);
4944 : }
4945 : }
4946 8225 : if (type1 != TREE_TYPE (arg1))
4947 250 : rhs2 = handle_cast (type1, arg1, idx1);
4948 : else
4949 7975 : rhs2 = handle_operand (arg1, idx1);
4950 8225 : if (i == 0)
4951 2733 : data_cnt = m_data_cnt;
4952 8225 : if (!useless_type_conversion_p (m_limb_type, TREE_TYPE (rhs1)))
4953 1983 : rhs1 = add_cast (m_limb_type, rhs1);
4954 8225 : if (!useless_type_conversion_p (m_limb_type, TREE_TYPE (rhs2)))
4955 1983 : rhs2 = add_cast (m_limb_type, rhs2);
4956 : last_rhs1 = rhs1;
4957 : last_rhs2 = rhs2;
4958 : }
4959 : else
4960 : {
4961 946 : m_data_cnt = data_cnt;
4962 946 : if (TYPE_UNSIGNED (type0) || prec0 >= 0)
4963 421 : rhs1 = build_zero_cst (m_limb_type);
4964 : else
4965 : {
4966 525 : rhs1 = add_cast (signed_type_for (m_limb_type), last_rhs1);
4967 525 : if (TREE_CODE (rhs1) == INTEGER_CST)
4968 52 : rhs1 = build_int_cst (m_limb_type,
4969 74 : tree_int_cst_sgn (rhs1) < 0 ? -1 : 0);
4970 : else
4971 : {
4972 946 : tree lpm1 = build_int_cst (unsigned_type_node,
4973 473 : limb_prec - 1);
4974 473 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (rhs1)),
4975 : RSHIFT_EXPR, rhs1, lpm1);
4976 473 : insert_before (g);
4977 473 : rhs1 = add_cast (m_limb_type, gimple_assign_lhs (g));
4978 : }
4979 : }
4980 946 : if (TYPE_UNSIGNED (type1) || prec1 >= 0)
4981 543 : rhs2 = build_zero_cst (m_limb_type);
4982 : else
4983 : {
4984 403 : rhs2 = add_cast (signed_type_for (m_limb_type), last_rhs2);
4985 403 : if (TREE_CODE (rhs2) == INTEGER_CST)
4986 114 : rhs2 = build_int_cst (m_limb_type,
4987 153 : tree_int_cst_sgn (rhs2) < 0 ? -1 : 0);
4988 : else
4989 : {
4990 578 : tree lpm1 = build_int_cst (unsigned_type_node,
4991 289 : limb_prec - 1);
4992 289 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (rhs2)),
4993 : RSHIFT_EXPR, rhs2, lpm1);
4994 289 : insert_before (g);
4995 289 : rhs2 = add_cast (m_limb_type, gimple_assign_lhs (g));
4996 : }
4997 : }
4998 : }
4999 9171 : tree rhs = handle_plus_minus (code, rhs1, rhs2, idx);
5000 9171 : if (ovf != boolean_false_node)
5001 : {
5002 9171 : if (tree_fits_uhwi_p (idx))
5003 : {
5004 6783 : unsigned limb = tree_to_uhwi (idx);
5005 6783 : if (bitint_big_endian)
5006 0 : limb = nelts - 1 - limb;
5007 6783 : if (limb >= startlimb && limb <= endlimb)
5008 : {
5009 3370 : tree l = arith_overflow_extract_bits (start, end, rhs,
5010 : limb, check_zero);
5011 3370 : tree this_ovf = make_ssa_name (boolean_type_node);
5012 3370 : if (ovf == NULL_TREE && !check_zero)
5013 : {
5014 907 : cmp = l;
5015 907 : g = gimple_build_assign (make_ssa_name (m_limb_type),
5016 : PLUS_EXPR, l,
5017 : build_int_cst (m_limb_type, 1));
5018 907 : insert_before (g);
5019 907 : g = gimple_build_assign (this_ovf, GT_EXPR,
5020 : gimple_assign_lhs (g),
5021 : build_int_cst (m_limb_type, 1));
5022 : }
5023 : else
5024 2463 : g = gimple_build_assign (this_ovf, NE_EXPR, l, cmp);
5025 3370 : insert_before (g);
5026 3370 : if (ovf == NULL_TREE)
5027 : ovf = this_ovf;
5028 : else
5029 : {
5030 1064 : tree b = make_ssa_name (boolean_type_node);
5031 1064 : g = gimple_build_assign (b, BIT_IOR_EXPR, ovf, this_ovf);
5032 1064 : insert_before (g);
5033 1064 : ovf = b;
5034 : }
5035 : }
5036 : }
5037 2388 : else if (startlimb < fin)
5038 : {
5039 854 : if (m_first && startlimb + 2 < fin)
5040 : {
5041 324 : tree data_out;
5042 324 : ovf = prepare_data_in_out (boolean_false_node, idx, &data_out);
5043 324 : ovf_out = m_data.pop ();
5044 324 : m_data.pop ();
5045 324 : if (!check_zero)
5046 : {
5047 169 : cmp = prepare_data_in_out (cmp, idx, &data_out);
5048 169 : cmp_out = m_data.pop ();
5049 169 : m_data.pop ();
5050 : }
5051 : }
5052 854 : if (i != 0 || startlimb != fin - 1)
5053 : {
5054 839 : tree_code cmp_code;
5055 839 : bool single_comparison
5056 839 : = (startlimb + 2 >= fin || (startlimb & 1) != (i & 1));
5057 : if (!single_comparison)
5058 : cmp_code = GE_EXPR;
5059 515 : else if ((startlimb & 1) == (i & 1))
5060 : cmp_code = EQ_EXPR;
5061 : else
5062 412 : cmp_code = GT_EXPR;
5063 839 : if (bitint_big_endian)
5064 0 : g = gimple_build_cond (swap_tree_comparison (cmp_code),
5065 0 : idx, size_int (nelts - 1
5066 : - startlimb),
5067 : NULL_TREE, NULL_TREE);
5068 : else
5069 839 : g = gimple_build_cond (cmp_code, idx, size_int (startlimb),
5070 : NULL_TREE, NULL_TREE);
5071 839 : edge edge_true_true, edge_true_false, edge_false;
5072 839 : gimple *g2 = NULL;
5073 839 : if (!single_comparison)
5074 324 : g2 = gimple_build_cond (NE_EXPR, idx,
5075 324 : size_int (bitint_big_endian
5076 : ? nelts - 1 - startlimb
5077 : : startlimb),
5078 : NULL_TREE, NULL_TREE);
5079 839 : if_then_if_then_else (g, g2, profile_probability::likely (),
5080 : profile_probability::likely (),
5081 : edge_true_true, edge_true_false,
5082 : edge_false);
5083 839 : unsigned tidx = startlimb + (cmp_code == GT_EXPR);
5084 839 : tree l = arith_overflow_extract_bits (start, end, rhs, tidx,
5085 : check_zero);
5086 839 : tree this_ovf = make_ssa_name (boolean_type_node);
5087 839 : if (cmp_code != GT_EXPR && !check_zero)
5088 : {
5089 173 : g = gimple_build_assign (make_ssa_name (m_limb_type),
5090 : PLUS_EXPR, l,
5091 : build_int_cst (m_limb_type, 1));
5092 173 : insert_before (g);
5093 173 : g = gimple_build_assign (this_ovf, GT_EXPR,
5094 : gimple_assign_lhs (g),
5095 : build_int_cst (m_limb_type, 1));
5096 : }
5097 : else
5098 666 : g = gimple_build_assign (this_ovf, NE_EXPR, l, cmp);
5099 839 : insert_before (g);
5100 839 : if (cmp_code == GT_EXPR)
5101 : {
5102 412 : tree t = make_ssa_name (boolean_type_node);
5103 412 : g = gimple_build_assign (t, BIT_IOR_EXPR, ovf, this_ovf);
5104 412 : insert_before (g);
5105 412 : this_ovf = t;
5106 : }
5107 839 : tree this_ovf2 = NULL_TREE;
5108 839 : if (!single_comparison)
5109 : {
5110 324 : m_gsi = gsi_after_labels (edge_true_true->src);
5111 324 : tree t = make_ssa_name (boolean_type_node);
5112 324 : g = gimple_build_assign (t, NE_EXPR, rhs, cmp);
5113 324 : insert_before (g);
5114 324 : this_ovf2 = make_ssa_name (boolean_type_node);
5115 324 : g = gimple_build_assign (this_ovf2, BIT_IOR_EXPR,
5116 : ovf, t);
5117 324 : insert_before (g);
5118 : }
5119 839 : m_gsi = gsi_after_labels (edge_true_false->dest);
5120 839 : tree t;
5121 839 : if (i == 1 && ovf_out)
5122 : t = ovf_out;
5123 : else
5124 515 : t = make_ssa_name (boolean_type_node);
5125 839 : gphi *phi = create_phi_node (t, edge_true_false->dest);
5126 839 : add_phi_arg (phi, this_ovf, edge_true_false,
5127 : UNKNOWN_LOCATION);
5128 839 : add_phi_arg (phi, ovf ? ovf
5129 : : boolean_false_node, edge_false,
5130 : UNKNOWN_LOCATION);
5131 839 : if (edge_true_true)
5132 324 : add_phi_arg (phi, this_ovf2, edge_true_true,
5133 : UNKNOWN_LOCATION);
5134 839 : ovf = t;
5135 839 : if (!check_zero && cmp_code != GT_EXPR)
5136 : {
5137 173 : t = cmp_out ? cmp_out : make_ssa_name (m_limb_type);
5138 173 : phi = create_phi_node (t, edge_true_false->dest);
5139 173 : add_phi_arg (phi, l, edge_true_false, UNKNOWN_LOCATION);
5140 173 : add_phi_arg (phi, cmp, edge_false, UNKNOWN_LOCATION);
5141 173 : if (edge_true_true)
5142 169 : add_phi_arg (phi, cmp, edge_true_true,
5143 : UNKNOWN_LOCATION);
5144 : cmp = t;
5145 : }
5146 : }
5147 : }
5148 : }
5149 :
5150 9171 : if (var || obj)
5151 : {
5152 8943 : if (tree_fits_uhwi_p (idx)
5153 6673 : && (bitint_big_endian
5154 6673 : ? nelts - 1 - tree_to_uhwi (idx)
5155 6673 : : tree_to_uhwi (idx)) >= prec_limbs)
5156 : ;
5157 7555 : else if (!tree_fits_uhwi_p (idx)
5158 2270 : && (unsigned) prec < (fin - (i == 0)) * limb_prec)
5159 : {
5160 1458 : bool single_comparison
5161 729 : = (((unsigned) prec % limb_prec) == 0
5162 571 : || prec_limbs + 1 >= fin
5163 1231 : || (prec_limbs & 1) == (i & 1));
5164 729 : if (bitint_big_endian)
5165 0 : g = gimple_build_cond (GE_EXPR, idx,
5166 0 : size_int (nelts - prec_limbs),
5167 : NULL_TREE, NULL_TREE);
5168 : else
5169 729 : g = gimple_build_cond (LE_EXPR, idx, size_int (prec_limbs - 1),
5170 : NULL_TREE, NULL_TREE);
5171 729 : gimple *g2 = NULL;
5172 729 : if (!single_comparison)
5173 251 : g2 = gimple_build_cond (EQ_EXPR, idx,
5174 251 : size_int (bitint_big_endian
5175 : ? nelts - prec_limbs
5176 : : prec_limbs - 1),
5177 : NULL_TREE, NULL_TREE);
5178 729 : edge edge_true_true, edge_true_false, edge_false;
5179 729 : if_then_if_then_else (g, g2, profile_probability::likely (),
5180 : profile_probability::unlikely (),
5181 : edge_true_true, edge_true_false,
5182 : edge_false);
5183 729 : tree idxl = idx;
5184 729 : if (bitint_big_endian && prec_limbs != nelts)
5185 : {
5186 0 : HOST_WIDE_INT diff = ((HOST_WIDE_INT) prec_limbs
5187 0 : - (HOST_WIDE_INT) nelts);
5188 0 : if (tree_fits_uhwi_p (idx))
5189 0 : idxl = size_int (tree_to_uhwi (idx) + diff);
5190 : else
5191 : {
5192 0 : idxl = make_ssa_name (sizetype);
5193 0 : g = gimple_build_assign (idxl, PLUS_EXPR, idx,
5194 0 : size_int (diff));
5195 0 : insert_before (g);
5196 : }
5197 : }
5198 1082 : tree l = limb_access (type, var ? var : obj, idxl, true);
5199 729 : g = gimple_build_assign (l, rhs);
5200 729 : insert_before (g);
5201 729 : if (!single_comparison)
5202 : {
5203 251 : m_gsi = gsi_after_labels (edge_true_true->src);
5204 251 : tree plm1idx = size_int (bitint_big_endian
5205 : ? 0 : prec_limbs - 1);
5206 251 : tree plm1type = limb_access_type (type, plm1idx);
5207 251 : l = limb_access (type, var ? var : obj, plm1idx, true);
5208 251 : if (!useless_type_conversion_p (plm1type, TREE_TYPE (rhs)))
5209 251 : rhs = add_cast (plm1type, rhs);
5210 251 : if (!useless_type_conversion_p (TREE_TYPE (l),
5211 251 : TREE_TYPE (rhs)))
5212 251 : rhs = add_cast (TREE_TYPE (l), rhs);
5213 251 : g = gimple_build_assign (l, rhs);
5214 251 : insert_before (g);
5215 : }
5216 729 : m_gsi = gsi_after_labels (edge_true_false->dest);
5217 729 : }
5218 : else
5219 : {
5220 6826 : tree idxl = idx;
5221 6826 : if (bitint_big_endian && prec_limbs != nelts)
5222 : {
5223 0 : HOST_WIDE_INT diff = ((HOST_WIDE_INT) prec_limbs
5224 0 : - (HOST_WIDE_INT) nelts);
5225 0 : if (tree_fits_uhwi_p (idx))
5226 0 : idxl = size_int (tree_to_uhwi (idx) + diff);
5227 : else
5228 : {
5229 0 : idxl = make_ssa_name (sizetype);
5230 0 : g = gimple_build_assign (idxl, PLUS_EXPR, idx,
5231 0 : size_int (diff));
5232 0 : insert_before (g);
5233 : }
5234 : }
5235 13623 : tree l = limb_access (type, var ? var : obj, idxl, true);
5236 6826 : if (bitint_extended && tree_fits_uhwi_p (idxl))
5237 : {
5238 0 : tree atype = limb_access_type (type, idxl);
5239 0 : if (!useless_type_conversion_p (atype, TREE_TYPE (rhs)))
5240 0 : rhs = add_cast (atype, rhs);
5241 : }
5242 6826 : if (!useless_type_conversion_p (TREE_TYPE (l), TREE_TYPE (rhs)))
5243 0 : rhs = add_cast (TREE_TYPE (l), rhs);
5244 6826 : g = gimple_build_assign (l, rhs);
5245 6826 : insert_before (g);
5246 : }
5247 : }
5248 9171 : m_first = false;
5249 9171 : if (kind == bitint_prec_huge && i <= 1)
5250 : {
5251 2388 : if (i == 0)
5252 : {
5253 1194 : idx = make_ssa_name (sizetype);
5254 1194 : g = gimple_build_assign (idx, PLUS_EXPR, idx_first,
5255 : bitint_big_endian
5256 0 : ? size_int (-1) : size_one_node);
5257 1194 : insert_before (g);
5258 : }
5259 : else
5260 : {
5261 1194 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx_first,
5262 2388 : size_int (bitint_big_endian ? -2 : 2));
5263 1194 : insert_before (g);
5264 1194 : if (bitint_big_endian)
5265 0 : g = gimple_build_cond (NE_EXPR, idx_first,
5266 0 : size_int (nelts + 1 - fin),
5267 : NULL_TREE, NULL_TREE);
5268 : else
5269 1194 : g = gimple_build_cond (NE_EXPR, idx_next, size_int (fin),
5270 : NULL_TREE, NULL_TREE);
5271 1194 : insert_before (g);
5272 1194 : m_gsi = gsi_for_stmt (final_stmt);
5273 1194 : m_bb = NULL;
5274 : }
5275 : }
5276 : }
5277 :
5278 2733 : finish_arith_overflow (var, obj, type, ovf, lhs, orig_obj, stmt,
5279 : prec_limbs, code);
5280 : }
5281 :
5282 : /* Lower a .MUL_OVERFLOW call with at least one large/huge _BitInt
5283 : argument or return type _Complex large/huge _BitInt. */
5284 :
5285 : void
5286 1479 : bitint_large_huge::lower_mul_overflow (tree obj, gimple *stmt)
5287 : {
5288 1479 : tree arg0 = gimple_call_arg (stmt, 0);
5289 1479 : tree arg1 = gimple_call_arg (stmt, 1);
5290 1479 : tree lhs = gimple_call_lhs (stmt);
5291 1479 : if (!lhs)
5292 : {
5293 0 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
5294 0 : gsi_remove (&gsi, true);
5295 0 : return;
5296 : }
5297 1479 : gimple *final_stmt = gsi_stmt (m_gsi);
5298 1479 : tree type = TREE_TYPE (lhs);
5299 1479 : if (TREE_CODE (type) == COMPLEX_TYPE)
5300 1464 : type = TREE_TYPE (type);
5301 1479 : int prec = TYPE_PRECISION (type), prec0, prec1;
5302 1479 : arg0 = handle_operand_addr (arg0, stmt, NULL, &prec0);
5303 1479 : arg1 = handle_operand_addr (arg1, stmt, NULL, &prec1);
5304 1479 : int prec2 = ((prec0 < 0 ? -prec0 : prec0)
5305 1479 : + (prec1 < 0 ? -prec1 : prec1));
5306 1479 : if (prec0 == 1 || prec1 == 1)
5307 35 : --prec2;
5308 1479 : tree var = NULL_TREE;
5309 1479 : tree orig_obj = obj;
5310 1479 : bool force_var = false;
5311 1479 : if (obj == NULL_TREE
5312 886 : && BITINT_TYPE_P (type)
5313 877 : && bitint_precision_kind (type) >= bitint_prec_large
5314 853 : && m_names
5315 2320 : && bitmap_bit_p (m_names, SSA_NAME_VERSION (lhs)))
5316 : {
5317 841 : int part = var_to_partition (m_map, lhs);
5318 841 : gcc_assert (m_vars[part] != NULL_TREE);
5319 841 : obj = m_vars[part];
5320 841 : if (TREE_TYPE (lhs) == type)
5321 7 : orig_obj = obj;
5322 : }
5323 638 : else if (obj != NULL_TREE && DECL_P (obj))
5324 : {
5325 1755 : for (int i = 0; i < 2; ++i)
5326 : {
5327 1170 : tree arg = i ? arg1 : arg0;
5328 1170 : if (TREE_CODE (arg) == ADDR_EXPR)
5329 1170 : arg = TREE_OPERAND (arg, 0);
5330 1170 : if (get_base_address (arg) == obj)
5331 : {
5332 : force_var = true;
5333 : break;
5334 : }
5335 : }
5336 : }
5337 1479 : if (obj == NULL_TREE
5338 1479 : || force_var
5339 1434 : || !BITINT_TYPE_P (type)
5340 1434 : || bitint_precision_kind (type) < bitint_prec_large
5341 3747 : || prec2 > (CEIL (prec, limb_prec) * limb_prec * (orig_obj ? 1 : 2)))
5342 : {
5343 564 : unsigned HOST_WIDE_INT nelts = CEIL (MAX (prec, prec2), limb_prec);
5344 564 : tree atype = build_array_type_nelts (m_limb_type, nelts);
5345 564 : var = create_tmp_var (atype);
5346 : }
5347 1479 : tree addr = build_fold_addr_expr (var ? var : obj);
5348 1479 : addr = force_gimple_operand_gsi (&m_gsi, addr, true,
5349 : NULL_TREE, true, GSI_SAME_STMT);
5350 1479 : tree sitype = lang_hooks.types.type_for_mode (SImode, 0);
5351 1479 : gimple *g
5352 1479 : = gimple_build_call_internal (IFN_MULBITINT, 6,
5353 : addr, build_int_cst (sitype,
5354 1479 : MAX (prec2, prec)),
5355 1479 : arg0, build_int_cst (sitype, prec0),
5356 1479 : arg1, build_int_cst (sitype, prec1));
5357 1479 : insert_before (g);
5358 :
5359 1479 : unsigned start, end;
5360 1479 : bool check_zero;
5361 1479 : tree ovf = arith_overflow (MULT_EXPR, type, prec, prec0, prec1, prec2,
5362 : &start, &end, &check_zero);
5363 1479 : if (ovf == NULL_TREE)
5364 : {
5365 1477 : unsigned startlimb = start / limb_prec;
5366 1477 : unsigned endlimb = (end - 1) / limb_prec;
5367 1477 : unsigned nelts = CEIL (MAX (prec, prec2), limb_prec);
5368 1477 : unsigned cnt;
5369 1477 : bool use_loop = false;
5370 1477 : if (startlimb == endlimb)
5371 : cnt = 1;
5372 1190 : else if (startlimb + 1 == endlimb)
5373 : cnt = 2;
5374 1013 : else if ((end % limb_prec) == 0)
5375 : {
5376 : cnt = 2;
5377 : use_loop = true;
5378 : }
5379 : else
5380 : {
5381 767 : cnt = 3;
5382 767 : use_loop = startlimb + 2 < endlimb;
5383 : }
5384 767 : if (cnt == 1)
5385 : {
5386 507 : tree l = limb_access (NULL_TREE, var ? var : obj,
5387 287 : size_int (bitint_big_endian
5388 : ? nelts - 1 - startlimb
5389 : : startlimb), true);
5390 287 : g = gimple_build_assign (make_ssa_name (m_limb_type), l);
5391 287 : insert_before (g);
5392 287 : l = arith_overflow_extract_bits (start, end, gimple_assign_lhs (g),
5393 : startlimb, check_zero);
5394 287 : ovf = make_ssa_name (boolean_type_node);
5395 287 : if (check_zero)
5396 247 : g = gimple_build_assign (ovf, NE_EXPR, l,
5397 : build_zero_cst (m_limb_type));
5398 : else
5399 : {
5400 40 : g = gimple_build_assign (make_ssa_name (m_limb_type),
5401 : PLUS_EXPR, l,
5402 : build_int_cst (m_limb_type, 1));
5403 40 : insert_before (g);
5404 40 : g = gimple_build_assign (ovf, GT_EXPR, gimple_assign_lhs (g),
5405 : build_int_cst (m_limb_type, 1));
5406 : }
5407 287 : insert_before (g);
5408 : }
5409 : else
5410 : {
5411 1190 : basic_block edge_bb = NULL;
5412 1190 : gimple_stmt_iterator gsi = m_gsi;
5413 1190 : gsi_prev (&gsi);
5414 1190 : edge e = split_block (gsi_bb (gsi), gsi_stmt (gsi));
5415 1190 : edge_bb = e->src;
5416 1190 : m_gsi = gsi_end_bb (edge_bb);
5417 :
5418 1190 : tree cmp = build_zero_cst (m_limb_type);
5419 5527 : for (unsigned i = 0; i < cnt; i++)
5420 : {
5421 3147 : tree idx, idx_next = NULL_TREE;
5422 3147 : if (i == 0)
5423 1190 : idx = size_int (bitint_big_endian
5424 : ? nelts - 1 - startlimb : startlimb);
5425 1957 : else if (i == 2)
5426 767 : idx = size_int (bitint_big_endian
5427 : ? nelts - 1 - endlimb : endlimb);
5428 1190 : else if (use_loop)
5429 665 : idx = create_loop (size_int (bitint_big_endian
5430 : ? nelts - startlimb - 2
5431 : : startlimb + 1), &idx_next);
5432 : else
5433 525 : idx = size_int (bitint_big_endian
5434 : ? nelts - startlimb - 2 : startlimb + 1);
5435 4988 : tree l = limb_access (NULL_TREE, var ? var : obj, idx, true);
5436 3147 : g = gimple_build_assign (make_ssa_name (m_limb_type), l);
5437 3147 : insert_before (g);
5438 3147 : l = gimple_assign_lhs (g);
5439 3147 : if (i == 0 || i == 2)
5440 2724 : l = arith_overflow_extract_bits (start, end, l,
5441 : i == 0 ? startlimb : endlimb,
5442 : check_zero);
5443 1957 : if (i == 0 && !check_zero)
5444 : {
5445 459 : cmp = l;
5446 459 : g = gimple_build_assign (make_ssa_name (m_limb_type),
5447 : PLUS_EXPR, l,
5448 : build_int_cst (m_limb_type, 1));
5449 459 : insert_before (g);
5450 459 : g = gimple_build_cond (GT_EXPR, gimple_assign_lhs (g),
5451 : build_int_cst (m_limb_type, 1),
5452 : NULL_TREE, NULL_TREE);
5453 : }
5454 : else
5455 2688 : g = gimple_build_cond (NE_EXPR, l, cmp, NULL_TREE, NULL_TREE);
5456 3147 : insert_before (g);
5457 3147 : edge e1 = split_block (gsi_bb (m_gsi), g);
5458 3147 : e1->flags = EDGE_FALSE_VALUE;
5459 3147 : edge e2 = make_edge (e1->src, gimple_bb (final_stmt),
5460 : EDGE_TRUE_VALUE);
5461 3147 : e1->probability = profile_probability::likely ();
5462 3147 : e2->probability = e1->probability.invert ();
5463 3147 : if (i == 0)
5464 1190 : set_immediate_dominator (CDI_DOMINATORS, e2->dest, e2->src);
5465 3147 : m_gsi = gsi_after_labels (e1->dest);
5466 3147 : if (i == 1 && use_loop)
5467 : {
5468 665 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
5469 : bitint_big_endian
5470 0 : ? size_int (-1) : size_one_node);
5471 665 : insert_before (g);
5472 665 : if (bitint_big_endian)
5473 0 : g = gimple_build_cond (NE_EXPR, idx,
5474 0 : size_int (nelts - endlimb
5475 : - (cnt == 2)),
5476 : NULL_TREE, NULL_TREE);
5477 : else
5478 665 : g = gimple_build_cond (NE_EXPR, idx_next,
5479 665 : size_int (endlimb + (cnt == 2)),
5480 : NULL_TREE, NULL_TREE);
5481 665 : insert_before (g);
5482 665 : edge true_edge, false_edge;
5483 665 : extract_true_false_edges_from_block (gsi_bb (m_gsi),
5484 : &true_edge,
5485 : &false_edge);
5486 665 : m_gsi = gsi_after_labels (false_edge->dest);
5487 665 : m_bb = NULL;
5488 : }
5489 : }
5490 :
5491 1190 : ovf = make_ssa_name (boolean_type_node);
5492 1190 : basic_block bb = gimple_bb (final_stmt);
5493 1190 : gphi *phi = create_phi_node (ovf, bb);
5494 1190 : edge e1 = find_edge (gsi_bb (m_gsi), bb);
5495 1190 : edge_iterator ei;
5496 5527 : FOR_EACH_EDGE (e, ei, bb->preds)
5497 : {
5498 4337 : tree val = e == e1 ? boolean_false_node : boolean_true_node;
5499 4337 : add_phi_arg (phi, val, e, UNKNOWN_LOCATION);
5500 : }
5501 1190 : m_gsi = gsi_for_stmt (final_stmt);
5502 : }
5503 : }
5504 :
5505 1479 : finish_arith_overflow (var, obj, type, ovf, lhs, orig_obj, stmt,
5506 1479 : CEIL (MAX (prec, prec2), limb_prec), MULT_EXPR);
5507 : }
5508 :
5509 : /* Lower REALPART_EXPR or IMAGPART_EXPR stmt extracting part of result from
5510 : .{ADD,SUB,MUL}_OVERFLOW call. */
5511 :
5512 : void
5513 6022 : bitint_large_huge::lower_cplxpart_stmt (tree obj, gimple *stmt)
5514 : {
5515 6022 : tree rhs1 = gimple_assign_rhs1 (stmt);
5516 6022 : rhs1 = TREE_OPERAND (rhs1, 0);
5517 6022 : if (obj == NULL_TREE)
5518 : {
5519 5929 : int part = var_to_partition (m_map, gimple_assign_lhs (stmt));
5520 5929 : gcc_assert (m_vars[part] != NULL_TREE);
5521 : obj = m_vars[part];
5522 : }
5523 6022 : if (TREE_CODE (rhs1) == SSA_NAME
5524 6022 : && (m_names == NULL
5525 6021 : || !bitmap_bit_p (m_names, SSA_NAME_VERSION (rhs1))))
5526 : {
5527 1549 : lower_call (obj, SSA_NAME_DEF_STMT (rhs1));
5528 1549 : return;
5529 : }
5530 4473 : int part = var_to_partition (m_map, rhs1);
5531 4473 : gcc_assert (m_vars[part] != NULL_TREE);
5532 4473 : tree var = m_vars[part];
5533 4473 : unsigned HOST_WIDE_INT nelts
5534 4473 : = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (obj))) / limb_prec;
5535 4473 : tree atype = build_array_type_nelts (m_limb_type, nelts);
5536 4473 : if (!useless_type_conversion_p (atype, TREE_TYPE (obj)))
5537 122 : obj = build1 (VIEW_CONVERT_EXPR, atype, obj);
5538 4473 : tree off = build_int_cst (build_pointer_type (TREE_TYPE (var)),
5539 4473 : gimple_assign_rhs_code (stmt) == REALPART_EXPR
5540 4473 : ? 0 : nelts * m_limb_size);
5541 4473 : tree v2 = build2 (MEM_REF, atype, build_fold_addr_expr (var), off);
5542 4473 : gimple *g = gimple_build_assign (obj, v2);
5543 4473 : insert_before (g);
5544 : }
5545 :
5546 : /* Lower COMPLEX_EXPR stmt. */
5547 :
5548 : void
5549 28 : bitint_large_huge::lower_complexexpr_stmt (gimple *stmt)
5550 : {
5551 28 : tree lhs = gimple_assign_lhs (stmt);
5552 28 : tree rhs1 = gimple_assign_rhs1 (stmt);
5553 28 : tree rhs2 = gimple_assign_rhs2 (stmt);
5554 28 : int part = var_to_partition (m_map, lhs);
5555 28 : gcc_assert (m_vars[part] != NULL_TREE);
5556 28 : lhs = m_vars[part];
5557 28 : unsigned HOST_WIDE_INT nelts
5558 28 : = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (rhs1))) / limb_prec;
5559 28 : tree atype = build_array_type_nelts (m_limb_type, nelts);
5560 28 : tree zero = build_zero_cst (build_pointer_type (TREE_TYPE (lhs)));
5561 28 : tree v1 = build2 (MEM_REF, atype, build_fold_addr_expr (lhs), zero);
5562 28 : tree v2;
5563 28 : if (TREE_CODE (rhs1) == SSA_NAME)
5564 : {
5565 28 : part = var_to_partition (m_map, rhs1);
5566 28 : gcc_assert (m_vars[part] != NULL_TREE);
5567 : v2 = m_vars[part];
5568 : }
5569 0 : else if (integer_zerop (rhs1))
5570 0 : v2 = build_zero_cst (atype);
5571 : else
5572 0 : v2 = tree_output_constant_def (rhs1);
5573 28 : if (!useless_type_conversion_p (atype, TREE_TYPE (v2)))
5574 18 : v2 = build1 (VIEW_CONVERT_EXPR, atype, v2);
5575 28 : gimple *g = gimple_build_assign (v1, v2);
5576 28 : insert_before (g);
5577 28 : tree off = fold_convert (build_pointer_type (TREE_TYPE (lhs)),
5578 : TYPE_SIZE_UNIT (atype));
5579 28 : v1 = build2 (MEM_REF, atype, build_fold_addr_expr (lhs), off);
5580 28 : if (TREE_CODE (rhs2) == SSA_NAME)
5581 : {
5582 10 : part = var_to_partition (m_map, rhs2);
5583 10 : gcc_assert (m_vars[part] != NULL_TREE);
5584 : v2 = m_vars[part];
5585 : }
5586 18 : else if (integer_zerop (rhs2))
5587 18 : v2 = build_zero_cst (atype);
5588 : else
5589 0 : v2 = tree_output_constant_def (rhs2);
5590 28 : if (!useless_type_conversion_p (atype, TREE_TYPE (v2)))
5591 10 : v2 = build1 (VIEW_CONVERT_EXPR, atype, v2);
5592 28 : g = gimple_build_assign (v1, v2);
5593 28 : insert_before (g);
5594 28 : }
5595 :
5596 : /* Lower a .{CLZ,CTZ,CLRSB,FFS,PARITY,POPCOUNT} call with one large/huge _BitInt
5597 : argument. */
5598 :
5599 : void
5600 101 : bitint_large_huge::lower_bit_query (gimple *stmt)
5601 : {
5602 101 : tree arg0 = gimple_call_arg (stmt, 0);
5603 101 : tree arg1 = (gimple_call_num_args (stmt) == 2
5604 101 : ? gimple_call_arg (stmt, 1) : NULL_TREE);
5605 101 : tree lhs = gimple_call_lhs (stmt);
5606 101 : gimple *g;
5607 :
5608 101 : if (!lhs)
5609 : {
5610 0 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
5611 0 : gsi_remove (&gsi, true);
5612 0 : return;
5613 : }
5614 101 : tree type = TREE_TYPE (arg0);
5615 101 : gcc_assert (BITINT_TYPE_P (type));
5616 101 : bitint_prec_kind kind = bitint_precision_kind (type);
5617 101 : gcc_assert (kind >= bitint_prec_large);
5618 101 : enum internal_fn ifn = gimple_call_internal_fn (stmt);
5619 101 : enum built_in_function fcode = END_BUILTINS;
5620 101 : gcc_assert (TYPE_PRECISION (unsigned_type_node) == limb_prec
5621 : || TYPE_PRECISION (long_unsigned_type_node) == limb_prec
5622 : || TYPE_PRECISION (long_long_unsigned_type_node) == limb_prec);
5623 101 : switch (ifn)
5624 : {
5625 26 : case IFN_CLZ:
5626 26 : if (TYPE_PRECISION (unsigned_type_node) == limb_prec)
5627 : fcode = BUILT_IN_CLZ;
5628 26 : else if (TYPE_PRECISION (long_unsigned_type_node) == limb_prec)
5629 : fcode = BUILT_IN_CLZL;
5630 : else
5631 0 : fcode = BUILT_IN_CLZLL;
5632 : break;
5633 11 : case IFN_FFS:
5634 : /* .FFS (X) is .CTZ (X, -1) + 1, though under the hood
5635 : we don't add the addend at the end. */
5636 11 : arg1 = integer_zero_node;
5637 : /* FALLTHRU */
5638 39 : case IFN_CTZ:
5639 39 : if (TYPE_PRECISION (unsigned_type_node) == limb_prec)
5640 : fcode = BUILT_IN_CTZ;
5641 39 : else if (TYPE_PRECISION (long_unsigned_type_node) == limb_prec)
5642 : fcode = BUILT_IN_CTZL;
5643 : else
5644 0 : fcode = BUILT_IN_CTZLL;
5645 39 : m_upwards = true;
5646 39 : break;
5647 9 : case IFN_CLRSB:
5648 9 : if (TYPE_PRECISION (unsigned_type_node) == limb_prec)
5649 : fcode = BUILT_IN_CLRSB;
5650 9 : else if (TYPE_PRECISION (long_unsigned_type_node) == limb_prec)
5651 : fcode = BUILT_IN_CLRSBL;
5652 : else
5653 0 : fcode = BUILT_IN_CLRSBLL;
5654 : break;
5655 16 : case IFN_PARITY:
5656 16 : if (TYPE_PRECISION (unsigned_type_node) == limb_prec)
5657 : fcode = BUILT_IN_PARITY;
5658 16 : else if (TYPE_PRECISION (long_unsigned_type_node) == limb_prec)
5659 : fcode = BUILT_IN_PARITYL;
5660 : else
5661 0 : fcode = BUILT_IN_PARITYLL;
5662 16 : m_upwards = true;
5663 16 : break;
5664 11 : case IFN_POPCOUNT:
5665 11 : if (TYPE_PRECISION (unsigned_type_node) == limb_prec)
5666 : fcode = BUILT_IN_POPCOUNT;
5667 11 : else if (TYPE_PRECISION (long_unsigned_type_node) == limb_prec)
5668 : fcode = BUILT_IN_POPCOUNTL;
5669 : else
5670 0 : fcode = BUILT_IN_POPCOUNTLL;
5671 11 : m_upwards = true;
5672 11 : break;
5673 0 : default:
5674 0 : gcc_unreachable ();
5675 : }
5676 101 : tree fndecl = builtin_decl_explicit (fcode), res = NULL_TREE;
5677 101 : unsigned cnt = 0, rem = 0, end = 0, prec = TYPE_PRECISION (type);
5678 101 : unsigned nelts = CEIL (prec, limb_prec);
5679 101 : struct bq_details { edge e; tree val, addend; } *bqp = NULL;
5680 101 : basic_block edge_bb = NULL;
5681 101 : if (m_upwards)
5682 : {
5683 66 : tree idx = NULL_TREE, idx_first = NULL_TREE, idx_next = NULL_TREE;
5684 66 : if (kind == bitint_prec_large)
5685 : cnt = nelts;
5686 : else
5687 : {
5688 39 : rem = (prec % (2 * limb_prec));
5689 39 : end = (prec - rem) / limb_prec;
5690 39 : cnt = 2 + CEIL (rem, limb_prec);
5691 39 : idx = idx_first = create_loop (bitint_big_endian
5692 39 : ? size_int (nelts - 1)
5693 : : size_zero_node, &idx_next);
5694 : }
5695 :
5696 66 : if (ifn == IFN_CTZ || ifn == IFN_FFS)
5697 : {
5698 39 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
5699 39 : gsi_prev (&gsi);
5700 39 : edge e = split_block (gsi_bb (gsi), gsi_stmt (gsi));
5701 39 : edge_bb = e->src;
5702 39 : if (kind == bitint_prec_large)
5703 32 : m_gsi = gsi_end_bb (edge_bb);
5704 39 : bqp = XALLOCAVEC (struct bq_details, cnt);
5705 : }
5706 : else
5707 27 : m_after_stmt = stmt;
5708 66 : if (kind != bitint_prec_large)
5709 39 : m_upwards_2limb = end;
5710 :
5711 244 : for (unsigned i = 0; i < cnt; i++)
5712 : {
5713 178 : m_data_cnt = 0;
5714 178 : if (kind == bitint_prec_large)
5715 83 : idx = size_int (bitint_big_endian ? nelts - 1 - i : i);
5716 95 : else if (i >= 2)
5717 17 : idx = size_int (bitint_big_endian
5718 : ? nelts - 1 - end - (i > 2) : end + (i > 2));
5719 :
5720 178 : tree rhs1 = handle_operand (arg0, idx);
5721 178 : if (!useless_type_conversion_p (m_limb_type, TREE_TYPE (rhs1)))
5722 : {
5723 32 : if (!TYPE_UNSIGNED (TREE_TYPE (rhs1)))
5724 5 : rhs1 = add_cast (unsigned_type_for (TREE_TYPE (rhs1)), rhs1);
5725 32 : rhs1 = add_cast (m_limb_type, rhs1);
5726 : }
5727 :
5728 178 : tree in, out, tem;
5729 178 : if (ifn == IFN_PARITY)
5730 43 : in = prepare_data_in_out (build_zero_cst (m_limb_type), idx, &out);
5731 135 : else if (ifn == IFN_FFS)
5732 29 : in = prepare_data_in_out (integer_one_node, idx, &out);
5733 : else
5734 106 : in = prepare_data_in_out (integer_zero_node, idx, &out);
5735 :
5736 178 : switch (ifn)
5737 : {
5738 104 : case IFN_CTZ:
5739 104 : case IFN_FFS:
5740 104 : g = gimple_build_cond (NE_EXPR, rhs1,
5741 : build_zero_cst (m_limb_type),
5742 : NULL_TREE, NULL_TREE);
5743 104 : insert_before (g);
5744 104 : edge e1, e2;
5745 104 : e1 = split_block (gsi_bb (m_gsi), g);
5746 104 : e1->flags = EDGE_FALSE_VALUE;
5747 104 : e2 = make_edge (e1->src, gimple_bb (stmt), EDGE_TRUE_VALUE);
5748 104 : e1->probability = profile_probability::unlikely ();
5749 104 : e2->probability = e1->probability.invert ();
5750 104 : if (i == 0)
5751 39 : set_immediate_dominator (CDI_DOMINATORS, e2->dest, e2->src);
5752 104 : m_gsi = gsi_after_labels (e1->dest);
5753 104 : bqp[i].e = e2;
5754 104 : bqp[i].val = rhs1;
5755 104 : if (tree_fits_uhwi_p (idx))
5756 58 : bqp[i].addend
5757 58 : = build_int_cst (integer_type_node,
5758 : (bitint_big_endian
5759 58 : ? nelts - 1 - tree_to_uhwi (idx)
5760 58 : : tree_to_uhwi (idx)) * limb_prec
5761 58 : + (ifn == IFN_FFS));
5762 : else
5763 : {
5764 46 : bqp[i].addend = in;
5765 46 : if (i == 1)
5766 23 : res = out;
5767 : else
5768 23 : res = make_ssa_name (integer_type_node);
5769 46 : g = gimple_build_assign (res, PLUS_EXPR, in,
5770 : build_int_cst (integer_type_node,
5771 46 : limb_prec));
5772 46 : insert_before (g);
5773 46 : m_data[m_data_cnt] = res;
5774 : }
5775 : break;
5776 43 : case IFN_PARITY:
5777 43 : if (!integer_zerop (in))
5778 : {
5779 37 : if (kind == bitint_prec_huge && i == 1)
5780 10 : res = out;
5781 : else
5782 27 : res = make_ssa_name (m_limb_type);
5783 37 : g = gimple_build_assign (res, BIT_XOR_EXPR, in, rhs1);
5784 37 : insert_before (g);
5785 : }
5786 : else
5787 : res = rhs1;
5788 43 : m_data[m_data_cnt] = res;
5789 43 : break;
5790 31 : case IFN_POPCOUNT:
5791 31 : g = gimple_build_call (fndecl, 1, rhs1);
5792 31 : tem = make_ssa_name (integer_type_node);
5793 31 : gimple_call_set_lhs (g, tem);
5794 31 : insert_before (g);
5795 31 : if (!integer_zerop (in))
5796 : {
5797 26 : if (kind == bitint_prec_huge && i == 1)
5798 6 : res = out;
5799 : else
5800 20 : res = make_ssa_name (integer_type_node);
5801 26 : g = gimple_build_assign (res, PLUS_EXPR, in, tem);
5802 26 : insert_before (g);
5803 : }
5804 : else
5805 : res = tem;
5806 31 : m_data[m_data_cnt] = res;
5807 31 : break;
5808 0 : default:
5809 0 : gcc_unreachable ();
5810 : }
5811 :
5812 178 : m_first = false;
5813 178 : if (kind == bitint_prec_huge && i <= 1)
5814 : {
5815 78 : if (i == 0)
5816 : {
5817 39 : idx = make_ssa_name (sizetype);
5818 39 : g = gimple_build_assign (idx, PLUS_EXPR, idx_first,
5819 : bitint_big_endian
5820 0 : ? size_int (-1) : size_one_node);
5821 39 : insert_before (g);
5822 : }
5823 : else
5824 : {
5825 39 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx_first,
5826 78 : size_int (bitint_big_endian
5827 : ? -2 : 2));
5828 39 : insert_before (g);
5829 39 : if (bitint_big_endian)
5830 0 : g = gimple_build_cond (NE_EXPR, idx_first,
5831 0 : size_int (cnt - 1),
5832 : NULL_TREE, NULL_TREE);
5833 : else
5834 39 : g = gimple_build_cond (NE_EXPR, idx_next, size_int (end),
5835 : NULL_TREE, NULL_TREE);
5836 39 : insert_before (g);
5837 39 : if (ifn == IFN_CTZ || ifn == IFN_FFS)
5838 23 : m_gsi = gsi_after_labels (edge_bb);
5839 : else
5840 16 : m_gsi = gsi_for_stmt (stmt);
5841 39 : m_bb = NULL;
5842 : }
5843 : }
5844 : }
5845 : }
5846 : else
5847 : {
5848 35 : tree idx = NULL_TREE, idx_next = NULL_TREE, first = NULL_TREE;
5849 35 : int sub_one = 0;
5850 35 : if (kind == bitint_prec_large)
5851 : cnt = nelts;
5852 : else
5853 : {
5854 18 : rem = prec % limb_prec;
5855 18 : if (rem == 0 && (!TYPE_UNSIGNED (type) || ifn == IFN_CLRSB))
5856 : rem = limb_prec;
5857 18 : end = (prec - rem) / limb_prec;
5858 18 : cnt = 1 + (rem != 0);
5859 18 : if (ifn == IFN_CLRSB)
5860 5 : sub_one = 1;
5861 : }
5862 :
5863 35 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
5864 35 : gsi_prev (&gsi);
5865 35 : edge e = split_block (gsi_bb (gsi), gsi_stmt (gsi));
5866 35 : edge_bb = e->src;
5867 35 : m_gsi = gsi_end_bb (edge_bb);
5868 :
5869 35 : if (ifn == IFN_CLZ)
5870 26 : bqp = XALLOCAVEC (struct bq_details, cnt);
5871 : else
5872 : {
5873 9 : gsi = gsi_for_stmt (stmt);
5874 9 : gsi_prev (&gsi);
5875 9 : e = split_block (gsi_bb (gsi), gsi_stmt (gsi));
5876 9 : edge_bb = e->src;
5877 9 : bqp = XALLOCAVEC (struct bq_details, 2 * cnt);
5878 : }
5879 :
5880 116 : for (unsigned i = 0; i < cnt; i++)
5881 : {
5882 81 : m_data_cnt = 0;
5883 81 : if (kind == bitint_prec_large)
5884 51 : idx = size_int (bitint_big_endian ? i : cnt - i - 1);
5885 30 : else if (i == cnt - 1)
5886 18 : idx = create_loop (size_int (bitint_big_endian ? i : end - 1),
5887 : &idx_next);
5888 : else
5889 12 : idx = bitint_big_endian ? size_zero_node : size_int (end);
5890 :
5891 81 : tree rhs1 = handle_operand (arg0, idx);
5892 81 : if (!useless_type_conversion_p (m_limb_type, TREE_TYPE (rhs1)))
5893 : {
5894 19 : if (ifn == IFN_CLZ && !TYPE_UNSIGNED (TREE_TYPE (rhs1)))
5895 0 : rhs1 = add_cast (unsigned_type_for (TREE_TYPE (rhs1)), rhs1);
5896 19 : else if (ifn == IFN_CLRSB && TYPE_UNSIGNED (TREE_TYPE (rhs1)))
5897 0 : rhs1 = add_cast (signed_type_for (TREE_TYPE (rhs1)), rhs1);
5898 19 : rhs1 = add_cast (m_limb_type, rhs1);
5899 : }
5900 :
5901 81 : if (ifn == IFN_CLZ)
5902 : {
5903 59 : g = gimple_build_cond (NE_EXPR, rhs1,
5904 : build_zero_cst (m_limb_type),
5905 : NULL_TREE, NULL_TREE);
5906 59 : insert_before (g);
5907 59 : edge e1 = split_block (gsi_bb (m_gsi), g);
5908 59 : e1->flags = EDGE_FALSE_VALUE;
5909 59 : edge e2 = make_edge (e1->src, gimple_bb (stmt), EDGE_TRUE_VALUE);
5910 59 : e1->probability = profile_probability::unlikely ();
5911 59 : e2->probability = e1->probability.invert ();
5912 59 : if (i == 0)
5913 26 : set_immediate_dominator (CDI_DOMINATORS, e2->dest, e2->src);
5914 59 : m_gsi = gsi_after_labels (e1->dest);
5915 59 : bqp[i].e = e2;
5916 59 : bqp[i].val = rhs1;
5917 : }
5918 : else
5919 : {
5920 22 : if (i == 0)
5921 : {
5922 9 : first = rhs1;
5923 9 : g = gimple_build_assign (make_ssa_name (m_limb_type),
5924 : PLUS_EXPR, rhs1,
5925 : build_int_cst (m_limb_type, 1));
5926 9 : insert_before (g);
5927 9 : g = gimple_build_cond (GT_EXPR, gimple_assign_lhs (g),
5928 : build_int_cst (m_limb_type, 1),
5929 : NULL_TREE, NULL_TREE);
5930 9 : insert_before (g);
5931 : }
5932 : else
5933 : {
5934 13 : g = gimple_build_assign (make_ssa_name (m_limb_type),
5935 : BIT_XOR_EXPR, rhs1, first);
5936 13 : insert_before (g);
5937 13 : tree stype = signed_type_for (m_limb_type);
5938 13 : g = gimple_build_cond (LT_EXPR,
5939 : add_cast (stype,
5940 : gimple_assign_lhs (g)),
5941 : build_zero_cst (stype),
5942 : NULL_TREE, NULL_TREE);
5943 13 : insert_before (g);
5944 13 : edge e1 = split_block (gsi_bb (m_gsi), g);
5945 13 : e1->flags = EDGE_FALSE_VALUE;
5946 13 : edge e2 = make_edge (e1->src, gimple_bb (stmt),
5947 : EDGE_TRUE_VALUE);
5948 13 : e1->probability = profile_probability::unlikely ();
5949 13 : e2->probability = e1->probability.invert ();
5950 13 : if (i == 1)
5951 9 : set_immediate_dominator (CDI_DOMINATORS, e2->dest,
5952 : e2->src);
5953 13 : m_gsi = gsi_after_labels (e1->dest);
5954 13 : bqp[2 * i].e = e2;
5955 13 : g = gimple_build_cond (NE_EXPR, rhs1, first,
5956 : NULL_TREE, NULL_TREE);
5957 13 : insert_before (g);
5958 : }
5959 22 : edge e1 = split_block (gsi_bb (m_gsi), g);
5960 22 : e1->flags = EDGE_FALSE_VALUE;
5961 22 : edge e2 = make_edge (e1->src, edge_bb, EDGE_TRUE_VALUE);
5962 22 : e1->probability = profile_probability::unlikely ();
5963 22 : e2->probability = e1->probability.invert ();
5964 22 : if (i == 0)
5965 9 : set_immediate_dominator (CDI_DOMINATORS, e2->dest, e2->src);
5966 22 : m_gsi = gsi_after_labels (e1->dest);
5967 22 : bqp[2 * i + 1].e = e2;
5968 22 : bqp[i].val = rhs1;
5969 : }
5970 81 : if (tree_fits_uhwi_p (idx))
5971 126 : bqp[i].addend
5972 63 : = build_int_cst (integer_type_node,
5973 63 : (int) prec
5974 63 : - (((int) (bitint_big_endian
5975 0 : ? nelts - 1 - tree_to_uhwi (idx)
5976 63 : : tree_to_uhwi (idx)) + 1)
5977 63 : * limb_prec) - sub_one);
5978 : else
5979 : {
5980 18 : tree in, out;
5981 18 : in = build_int_cst (integer_type_node, rem - sub_one);
5982 18 : m_first = true;
5983 18 : in = prepare_data_in_out (in, idx, &out);
5984 18 : out = m_data[m_data_cnt + 1];
5985 18 : bqp[i].addend = in;
5986 18 : g = gimple_build_assign (out, PLUS_EXPR, in,
5987 : build_int_cst (integer_type_node,
5988 18 : limb_prec));
5989 18 : insert_before (g);
5990 18 : m_data[m_data_cnt] = out;
5991 : }
5992 :
5993 81 : m_first = false;
5994 81 : if (kind == bitint_prec_huge && i == cnt - 1)
5995 : {
5996 36 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
5997 : bitint_big_endian
5998 18 : ? size_one_node : size_int (-1));
5999 18 : insert_before (g);
6000 18 : g = gimple_build_cond (NE_EXPR, idx,
6001 : bitint_big_endian
6002 0 : ? size_int (nelts - 1) : size_zero_node,
6003 : NULL_TREE, NULL_TREE);
6004 18 : insert_before (g);
6005 18 : edge true_edge, false_edge;
6006 18 : extract_true_false_edges_from_block (gsi_bb (m_gsi),
6007 : &true_edge, &false_edge);
6008 18 : m_gsi = gsi_after_labels (false_edge->dest);
6009 18 : m_bb = NULL;
6010 : }
6011 : }
6012 : }
6013 101 : switch (ifn)
6014 : {
6015 65 : case IFN_CLZ:
6016 65 : case IFN_CTZ:
6017 65 : case IFN_FFS:
6018 65 : gphi *phi1, *phi2, *phi3;
6019 65 : basic_block bb;
6020 65 : bb = gsi_bb (m_gsi);
6021 65 : remove_edge (find_edge (bb, gimple_bb (stmt)));
6022 65 : phi1 = create_phi_node (make_ssa_name (m_limb_type),
6023 : gimple_bb (stmt));
6024 65 : phi2 = create_phi_node (make_ssa_name (integer_type_node),
6025 : gimple_bb (stmt));
6026 293 : for (unsigned i = 0; i < cnt; i++)
6027 : {
6028 163 : add_phi_arg (phi1, bqp[i].val, bqp[i].e, UNKNOWN_LOCATION);
6029 163 : add_phi_arg (phi2, bqp[i].addend, bqp[i].e, UNKNOWN_LOCATION);
6030 : }
6031 65 : if (arg1 == NULL_TREE)
6032 : {
6033 35 : g = gimple_build_builtin_unreachable (m_loc);
6034 35 : insert_before (g);
6035 : }
6036 65 : m_gsi = gsi_for_stmt (stmt);
6037 65 : g = gimple_build_call (fndecl, 1, gimple_phi_result (phi1));
6038 65 : gimple_call_set_lhs (g, make_ssa_name (integer_type_node));
6039 65 : insert_before (g);
6040 65 : if (arg1 == NULL_TREE)
6041 35 : g = gimple_build_assign (lhs, PLUS_EXPR,
6042 : gimple_phi_result (phi2),
6043 : gimple_call_lhs (g));
6044 : else
6045 : {
6046 30 : g = gimple_build_assign (make_ssa_name (integer_type_node),
6047 : PLUS_EXPR, gimple_phi_result (phi2),
6048 : gimple_call_lhs (g));
6049 30 : insert_before (g);
6050 30 : edge e1 = split_block (gimple_bb (stmt), g);
6051 30 : edge e2 = make_edge (bb, e1->dest, EDGE_FALLTHRU);
6052 30 : e2->probability = profile_probability::always ();
6053 30 : set_immediate_dominator (CDI_DOMINATORS, e1->dest,
6054 : get_immediate_dominator (CDI_DOMINATORS,
6055 : e1->src));
6056 30 : phi3 = create_phi_node (make_ssa_name (integer_type_node), e1->dest);
6057 30 : add_phi_arg (phi3, gimple_assign_lhs (g), e1, UNKNOWN_LOCATION);
6058 30 : add_phi_arg (phi3, arg1, e2, UNKNOWN_LOCATION);
6059 30 : m_gsi = gsi_for_stmt (stmt);
6060 30 : g = gimple_build_assign (lhs, gimple_phi_result (phi3));
6061 : }
6062 65 : gsi_replace (&m_gsi, g, true);
6063 65 : break;
6064 9 : case IFN_CLRSB:
6065 9 : bb = gsi_bb (m_gsi);
6066 9 : remove_edge (find_edge (bb, edge_bb));
6067 9 : edge e;
6068 9 : e = make_edge (bb, gimple_bb (stmt), EDGE_FALLTHRU);
6069 9 : e->probability = profile_probability::always ();
6070 9 : set_immediate_dominator (CDI_DOMINATORS, gimple_bb (stmt),
6071 : get_immediate_dominator (CDI_DOMINATORS,
6072 : edge_bb));
6073 9 : phi1 = create_phi_node (make_ssa_name (m_limb_type),
6074 : edge_bb);
6075 9 : phi2 = create_phi_node (make_ssa_name (integer_type_node),
6076 : edge_bb);
6077 9 : phi3 = create_phi_node (make_ssa_name (integer_type_node),
6078 : gimple_bb (stmt));
6079 40 : for (unsigned i = 0; i < cnt; i++)
6080 : {
6081 22 : add_phi_arg (phi1, bqp[i].val, bqp[2 * i + 1].e, UNKNOWN_LOCATION);
6082 22 : add_phi_arg (phi2, bqp[i].addend, bqp[2 * i + 1].e,
6083 : UNKNOWN_LOCATION);
6084 22 : tree a = bqp[i].addend;
6085 22 : if (i && kind == bitint_prec_large)
6086 8 : a = int_const_binop (PLUS_EXPR, a, integer_minus_one_node);
6087 22 : if (i)
6088 13 : add_phi_arg (phi3, a, bqp[2 * i].e, UNKNOWN_LOCATION);
6089 : }
6090 9 : add_phi_arg (phi3, build_int_cst (integer_type_node, prec - 1), e,
6091 : UNKNOWN_LOCATION);
6092 9 : m_gsi = gsi_after_labels (edge_bb);
6093 9 : g = gimple_build_call (fndecl, 1,
6094 : add_cast (signed_type_for (m_limb_type),
6095 : gimple_phi_result (phi1)));
6096 9 : gimple_call_set_lhs (g, make_ssa_name (integer_type_node));
6097 9 : insert_before (g);
6098 9 : g = gimple_build_assign (make_ssa_name (integer_type_node),
6099 : PLUS_EXPR, gimple_call_lhs (g),
6100 : gimple_phi_result (phi2));
6101 9 : insert_before (g);
6102 9 : if (kind != bitint_prec_large)
6103 : {
6104 5 : g = gimple_build_assign (make_ssa_name (integer_type_node),
6105 : PLUS_EXPR, gimple_assign_lhs (g),
6106 : integer_one_node);
6107 5 : insert_before (g);
6108 : }
6109 9 : add_phi_arg (phi3, gimple_assign_lhs (g),
6110 : find_edge (edge_bb, gimple_bb (stmt)), UNKNOWN_LOCATION);
6111 9 : m_gsi = gsi_for_stmt (stmt);
6112 9 : g = gimple_build_assign (lhs, gimple_phi_result (phi3));
6113 9 : gsi_replace (&m_gsi, g, true);
6114 9 : break;
6115 16 : case IFN_PARITY:
6116 16 : g = gimple_build_call (fndecl, 1, res);
6117 16 : gimple_call_set_lhs (g, lhs);
6118 16 : gsi_replace (&m_gsi, g, true);
6119 16 : break;
6120 11 : case IFN_POPCOUNT:
6121 11 : g = gimple_build_assign (lhs, res);
6122 11 : gsi_replace (&m_gsi, g, true);
6123 11 : break;
6124 0 : default:
6125 0 : gcc_unreachable ();
6126 : }
6127 : }
6128 :
6129 : /* Lower a .{BSWAP,BITREVERSE} call with one large/huge _BitInt argument. */
6130 :
6131 : void
6132 47 : bitint_large_huge::lower_bswap_bitreverse (tree obj, gimple *stmt)
6133 : {
6134 47 : tree arg = gimple_call_arg (stmt, 0);
6135 47 : tree lhs = gimple_call_lhs (stmt);
6136 47 : gimple *g;
6137 :
6138 47 : if (!lhs)
6139 : {
6140 0 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
6141 0 : gsi_remove (&gsi, true);
6142 0 : return;
6143 : }
6144 47 : tree type = TREE_TYPE (arg);
6145 47 : gcc_assert (BITINT_TYPE_P (type));
6146 47 : bitint_prec_kind kind = bitint_precision_kind (type);
6147 47 : gcc_assert (kind >= bitint_prec_large);
6148 47 : if (!obj)
6149 : {
6150 34 : int part = var_to_partition (m_map, lhs);
6151 34 : gcc_assert (m_vars[part] != NULL_TREE);
6152 : obj = m_vars[part];
6153 : }
6154 47 : enum internal_fn ifn = gimple_call_internal_fn (stmt);
6155 47 : enum built_in_function bcode = END_BUILTINS;
6156 47 : switch (limb_prec)
6157 : {
6158 0 : case 16:
6159 0 : bcode = ifn == IFN_BSWAP ? BUILT_IN_BSWAP16 : BUILT_IN_BITREVERSE16;
6160 : break;
6161 0 : case 32:
6162 0 : bcode = ifn == IFN_BSWAP ? BUILT_IN_BSWAP32 : BUILT_IN_BITREVERSE32;
6163 : break;
6164 47 : case 64:
6165 47 : bcode = ifn == IFN_BSWAP ? BUILT_IN_BSWAP64 : BUILT_IN_BITREVERSE64;
6166 : break;
6167 0 : case 128:
6168 0 : bcode = ifn == IFN_BSWAP ? BUILT_IN_BSWAP128 : BUILT_IN_BITREVERSE128;
6169 : break;
6170 0 : default:
6171 0 : gcc_unreachable ();
6172 : }
6173 47 : tree fndecl = builtin_decl_explicit (bcode);
6174 47 : unsigned prec = TYPE_PRECISION (type);
6175 94 : tree p = build_int_cst (sizetype,
6176 47 : prec / limb_prec - (prec % limb_prec == 0));
6177 : /* For .BSWAP/.BITREVERSE, we can use handle_operand with non-INTEGER_CST
6178 : idx even to access the most significant partial limb. */
6179 47 : m_var_msb = true;
6180 : /* For IFN .BSWAP or .BITREVERSE and
6181 : FN corresponding __builtin_bswapN or __builtin_bitreverseN where N
6182 : is limb_prec, lower
6183 : dst = IFN (src);
6184 : as
6185 : size_t p = prec / limb_prec - (prec % limb_prec == 0);
6186 : if constexpr ((prec % limb_prec) == 0)
6187 : {
6188 : for (idx = 0; idx <= p; ++idx)
6189 : dst[p - idx] = FN (src[idx]);
6190 : }
6191 : else
6192 : {
6193 : unsigned n1 = prec % limb_prec;
6194 : unsigned n2 = limb_prec - n1;
6195 : dst[p] = FN (src[0]) >> n2;
6196 : for (idx = p - 1; (ssize_t) idx >= 0; --idx)
6197 : dst[idx] = FN ((src[p - idx] << n2) | (src[p - idx - 1] >> n1));
6198 : } */
6199 47 : if (prec % limb_prec == 0)
6200 : {
6201 9 : tree idx_next;
6202 9 : tree idx = create_loop (size_zero_node, &idx_next);
6203 9 : tree pmidx = make_ssa_name (sizetype);
6204 9 : g = gimple_build_assign (pmidx, MINUS_EXPR, p, idx);
6205 9 : insert_before (g);
6206 9 : m_data_cnt = 0;
6207 9 : tree t = handle_operand (arg, idx);
6208 9 : m_first = false;
6209 9 : g = gimple_build_call (fndecl, 1, t);
6210 9 : t = make_ssa_name (m_limb_type);
6211 9 : gimple_call_set_lhs (g, t);
6212 9 : insert_before (g);
6213 9 : tree l = limb_access (TREE_TYPE (lhs), obj, pmidx, true);
6214 9 : g = gimple_build_assign (l, t);
6215 9 : insert_before (g);
6216 9 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx, size_one_node);
6217 9 : insert_before (g);
6218 9 : g = gimple_build_cond (LE_EXPR, idx_next, p, NULL_TREE, NULL_TREE);
6219 9 : insert_before (g);
6220 : }
6221 : else
6222 : {
6223 38 : tree n1 = build_int_cst (unsigned_type_node, prec % limb_prec);
6224 76 : tree n2 = build_int_cst (unsigned_type_node,
6225 38 : limb_prec - (prec % limb_prec));
6226 38 : m_data_cnt = 0;
6227 38 : tree t = handle_operand (arg, bitint_big_endian ? p : size_zero_node);
6228 38 : m_first = false;
6229 : /* Nothing is needed to bswap 8 bits or bitreverse 1 bit, so just
6230 : mask off higher bits in that case. */
6231 71 : if ((prec % limb_prec) != (ifn == IFN_BSWAP ? 8 : 1))
6232 : {
6233 26 : g = gimple_build_call (fndecl, 1, t);
6234 26 : t = make_ssa_name (m_limb_type);
6235 26 : gimple_call_set_lhs (g, t);
6236 26 : insert_before (g);
6237 26 : g = gimple_build_assign (make_ssa_name (m_limb_type), RSHIFT_EXPR,
6238 : t, n2);
6239 : }
6240 : else
6241 12 : g = gimple_build_assign (make_ssa_name (m_limb_type), BIT_AND_EXPR,
6242 : t, build_int_cst (m_limb_type,
6243 : ifn == IFN_BSWAP
6244 22 : ? 0xff : 1));
6245 38 : insert_before (g);
6246 38 : t = gimple_assign_lhs (g);
6247 38 : tree l = limb_access (TREE_TYPE (lhs), obj,
6248 : bitint_big_endian ? size_zero_node : p, true);
6249 38 : g = gimple_build_assign (l, t);
6250 38 : insert_before (g);
6251 38 : tree pm1 = build_int_cst (sizetype, prec / limb_prec - 1);
6252 38 : tree idx_next;
6253 38 : tree idx = create_loop (pm1, &idx_next);
6254 38 : tree pmidx = make_ssa_name (sizetype);
6255 38 : g = gimple_build_assign (pmidx, MINUS_EXPR, p, idx);
6256 38 : insert_before (g);
6257 38 : m_data_cnt = 0;
6258 76 : t = handle_operand (arg, bitint_big_endian ? idx : pmidx);
6259 38 : m_data_cnt = 0;
6260 38 : g = gimple_build_assign (make_ssa_name (m_limb_type), LSHIFT_EXPR,
6261 : t, n2);
6262 38 : insert_before (g);
6263 38 : t = gimple_assign_lhs (g);
6264 38 : tree t2 = make_ssa_name (sizetype);
6265 38 : if (bitint_big_endian)
6266 0 : g = gimple_build_assign (t2, PLUS_EXPR, idx, size_one_node);
6267 : else
6268 38 : g = gimple_build_assign (t2, PLUS_EXPR, pmidx, size_int (-1));
6269 38 : insert_before (g);
6270 38 : t2 = handle_operand (arg, t2);
6271 38 : g = gimple_build_assign (make_ssa_name (m_limb_type), RSHIFT_EXPR,
6272 : t2, n1);
6273 38 : insert_before (g);
6274 38 : t2 = gimple_assign_lhs (g);
6275 38 : g = gimple_build_assign (make_ssa_name (m_limb_type), BIT_IOR_EXPR,
6276 : t, t2);
6277 38 : insert_before (g);
6278 38 : t = gimple_assign_lhs (g);
6279 38 : g = gimple_build_call (fndecl, 1, t);
6280 38 : t = make_ssa_name (m_limb_type);
6281 38 : gimple_call_set_lhs (g, t);
6282 38 : insert_before (g);
6283 76 : l = limb_access (TREE_TYPE (lhs), obj,
6284 : bitint_big_endian ? pmidx : idx, true);
6285 38 : g = gimple_build_assign (l, t);
6286 38 : insert_before (g);
6287 38 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx, size_int (-1));
6288 38 : insert_before (g);
6289 38 : g = gimple_build_cond (NE_EXPR, idx, size_zero_node, NULL_TREE,
6290 : NULL_TREE);
6291 38 : insert_before (g);
6292 : }
6293 47 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
6294 47 : if (bitint_extended == bitint_ext_full
6295 0 : && abi_limb_prec > limb_prec
6296 0 : && (CEIL (prec, abi_limb_prec) * abi_limb_prec
6297 0 : > CEIL (prec, limb_prec) * limb_prec))
6298 : {
6299 0 : m_gsi = gsi;
6300 0 : tree p2 = build_int_cst (sizetype,
6301 : CEIL (prec, abi_limb_prec)
6302 0 : * abi_limb_prec / limb_prec - 1);
6303 0 : tree l = limb_access (TREE_TYPE (lhs), obj, p2, true);
6304 0 : g = gimple_build_assign (l, build_zero_cst (m_limb_type));
6305 0 : insert_before (g);
6306 : }
6307 : }
6308 :
6309 : /* Lower a .CLMUL call with two large/huge _BitInt arguments. */
6310 :
6311 : void
6312 30 : bitint_large_huge::lower_clmul (tree obj, gimple *stmt)
6313 : {
6314 30 : tree arg1 = gimple_call_arg (stmt, 0);
6315 30 : tree arg2 = gimple_call_arg (stmt, 1);
6316 30 : tree lhs = gimple_call_lhs (stmt);
6317 30 : gimple *g;
6318 :
6319 30 : if (!lhs)
6320 : {
6321 0 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
6322 0 : gsi_remove (&gsi, true);
6323 0 : return;
6324 : }
6325 60 : if (wi::popcount (get_nonzero_bits (arg1))
6326 48 : < wi::popcount (get_nonzero_bits (arg2)))
6327 0 : std::swap (arg1, arg2);
6328 30 : tree type = TREE_TYPE (arg1);
6329 30 : gcc_assert (BITINT_TYPE_P (type));
6330 30 : bitint_prec_kind kind = bitint_precision_kind (type);
6331 30 : gcc_assert (kind >= bitint_prec_large);
6332 30 : if (!obj)
6333 : {
6334 20 : int part = var_to_partition (m_map, lhs);
6335 20 : gcc_assert (m_vars[part] != NULL_TREE);
6336 : obj = m_vars[part];
6337 : }
6338 : /* For .CLMUL, we can use handle_operand with non-INTEGER_CST
6339 : idx even to access the most significant partial limb. */
6340 30 : m_var_msb = true;
6341 30 : unsigned prec = TYPE_PRECISION (type);
6342 : /* Lower
6343 : dst = .CLMUL (src1, src2);
6344 : as
6345 : size_t p = prec / limb_prec - (prec % limb_prec == 0);
6346 : dst = {};
6347 : for (idx = 0; idx <= p; ++idx)
6348 : {
6349 : limb_type t = src2[idx];
6350 : size_t i = 0, j = 0;
6351 : do
6352 : {
6353 : if (t & 1)
6354 : {
6355 : size_t idx3 = idx + j;
6356 : size_t k = -i & (limb_prec - 1);
6357 : r[idx] ^= (src1[0] << i);
6358 : for (idx2 = idx + 1; idx2 <= p; ++idx2)
6359 : r[idx2] ^= (src1[idx2 - idx] << i)
6360 : | (src1[idx2 - idx3] >> k);
6361 : }
6362 : t >>= 1;
6363 : ++i;
6364 : j = 1;
6365 : }
6366 : while (t);
6367 : } */
6368 60 : tree p = build_int_cst (sizetype,
6369 30 : prec / limb_prec - (prec % limb_prec == 0));
6370 30 : tree zero = build_zero_cst (TREE_TYPE (obj));
6371 30 : g = gimple_build_assign (obj, zero);
6372 30 : insert_before (g);
6373 30 : tree idx_next;
6374 30 : tree idx = create_loop (bitint_big_endian ? p : size_zero_node, &idx_next);
6375 30 : m_data_cnt = 0;
6376 30 : tree t_init = handle_operand (arg2, idx);
6377 30 : m_first = false;
6378 30 : tree i_next;
6379 30 : tree i = create_loop (size_zero_node, &i_next);
6380 30 : tree j = make_ssa_name (sizetype);
6381 30 : gphi *phi = create_phi_node (j, m_bb);
6382 30 : add_phi_arg (phi, size_zero_node, find_edge (m_preheader_bb, m_bb),
6383 : UNKNOWN_LOCATION);
6384 30 : add_phi_arg (phi, size_one_node, find_edge (m_bb, m_bb), UNKNOWN_LOCATION);
6385 30 : tree t = make_ssa_name (m_limb_type);
6386 30 : phi = create_phi_node (t, m_bb);
6387 30 : add_phi_arg (phi, t_init, find_edge (m_preheader_bb, m_bb),
6388 : UNKNOWN_LOCATION);
6389 30 : tree t_next = make_ssa_name (m_limb_type);
6390 30 : add_phi_arg (phi, t_next, find_edge (m_bb, m_bb), UNKNOWN_LOCATION);
6391 30 : tree t1 = make_ssa_name (m_limb_type);
6392 30 : g = gimple_build_assign (t1, BIT_AND_EXPR, t, build_one_cst (m_limb_type));
6393 30 : insert_before (g);
6394 30 : g = gimple_build_cond (NE_EXPR, t1, build_zero_cst (m_limb_type),
6395 : NULL_TREE, NULL_TREE);
6396 30 : edge edge_true, edge_false;
6397 30 : if_then (g, profile_probability::even (), edge_true, edge_false);
6398 30 : tree idx3 = make_ssa_name (sizetype), idx4 = idx;
6399 60 : g = gimple_build_assign (idx3, bitint_big_endian ? MINUS_EXPR : PLUS_EXPR,
6400 : idx, j);
6401 30 : insert_before (g);
6402 30 : if (bitint_big_endian)
6403 : {
6404 0 : g = gimple_build_assign (make_ssa_name (sizetype), MINUS_EXPR, p, idx3);
6405 0 : insert_before (g);
6406 0 : idx3 = gimple_assign_lhs (g);
6407 0 : idx4 = make_ssa_name (sizetype);
6408 0 : g = gimple_build_assign (idx4, MINUS_EXPR, p, idx);
6409 0 : insert_before (g);
6410 : }
6411 30 : g = gimple_build_assign (make_ssa_name (sizetype), NEGATE_EXPR, i);
6412 30 : insert_before (g);
6413 30 : tree k = make_ssa_name (sizetype);
6414 30 : g = gimple_build_assign (k, BIT_AND_EXPR, gimple_assign_lhs (g),
6415 30 : size_int (limb_prec - 1));
6416 30 : insert_before (g);
6417 30 : unsigned data_cnt = m_data_cnt;
6418 30 : m_first = true;
6419 30 : tree s1 = handle_operand (arg1, bitint_big_endian ? p : size_zero_node);
6420 30 : m_first = false;
6421 30 : m_data_cnt = data_cnt;
6422 30 : g = gimple_build_assign (make_ssa_name (m_limb_type), LSHIFT_EXPR, s1, i);
6423 30 : insert_before (g);
6424 30 : s1 = gimple_assign_lhs (g);
6425 30 : tree r = limb_access (TREE_TYPE (lhs), obj, idx, false);
6426 30 : g = gimple_build_assign (make_ssa_name (m_limb_type), r);
6427 30 : insert_before (g);
6428 30 : g = gimple_build_assign (make_ssa_name (m_limb_type), BIT_XOR_EXPR,
6429 : gimple_assign_lhs (g), s1);
6430 30 : insert_before (g);
6431 30 : r = limb_access (TREE_TYPE (lhs), obj, idx, true);
6432 30 : g = gimple_build_assign (r, gimple_assign_lhs (g));
6433 30 : insert_before (g);
6434 30 : tree idxp1 = make_ssa_name (sizetype);
6435 30 : g = gimple_build_assign (idxp1, PLUS_EXPR, idx,
6436 0 : bitint_big_endian ? size_int (-1) : size_one_node);
6437 30 : insert_before (g);
6438 30 : if (bitint_big_endian)
6439 0 : g = gimple_build_cond (NE_EXPR, idxp1, size_int (-1),
6440 : NULL_TREE, NULL_TREE);
6441 : else
6442 30 : g = gimple_build_cond (LE_EXPR, idxp1, p, NULL_TREE, NULL_TREE);
6443 30 : edge edge_true2, edge_false2;
6444 30 : if_then (g, profile_probability::likely (), edge_true2, edge_false2);
6445 30 : tree idx2_next;
6446 30 : tree idx2 = create_loop (idxp1, &idx2_next);
6447 30 : tree idx2midx = make_ssa_name (sizetype);
6448 30 : g = gimple_build_assign (idx2midx, MINUS_EXPR, idx2, idx4);
6449 30 : insert_before (g);
6450 30 : tree idx2midx3 = make_ssa_name (sizetype);
6451 30 : g = gimple_build_assign (idx2midx3, MINUS_EXPR, idx2, idx3);
6452 30 : insert_before (g);
6453 30 : m_data_cnt = data_cnt;
6454 30 : tree s2 = handle_operand (arg1, idx2midx);
6455 30 : g = gimple_build_assign (make_ssa_name (m_limb_type), LSHIFT_EXPR,
6456 : s2, i);
6457 30 : insert_before (g);
6458 30 : s2 = gimple_assign_lhs (g);
6459 30 : m_data_cnt = data_cnt;
6460 30 : tree s3 = handle_operand (arg1, idx2midx3);
6461 30 : g = gimple_build_assign (make_ssa_name (m_limb_type), RSHIFT_EXPR,
6462 : s3, k);
6463 30 : insert_before (g);
6464 30 : g = gimple_build_assign (make_ssa_name (m_limb_type), BIT_IOR_EXPR,
6465 : s2, gimple_assign_lhs (g));
6466 30 : insert_before (g);
6467 30 : s2 = gimple_assign_lhs (g);
6468 30 : r = limb_access (TREE_TYPE (lhs), obj, idx2, false);
6469 30 : g = gimple_build_assign (make_ssa_name (m_limb_type), r);
6470 30 : insert_before (g);
6471 30 : g = gimple_build_assign (make_ssa_name (m_limb_type), BIT_XOR_EXPR,
6472 : gimple_assign_lhs (g), s2);
6473 30 : insert_before (g);
6474 30 : r = limb_access (TREE_TYPE (lhs), obj, idx2, true);
6475 30 : g = gimple_build_assign (r, gimple_assign_lhs (g));
6476 30 : insert_before (g);
6477 30 : g = gimple_build_assign (idx2_next, PLUS_EXPR, idx2,
6478 0 : bitint_big_endian ? size_int (-1) : size_one_node);
6479 30 : insert_before (g);
6480 30 : if (bitint_big_endian)
6481 0 : g = gimple_build_cond (NE_EXPR, idx2_next, size_int (-1),
6482 : NULL_TREE, NULL_TREE);
6483 : else
6484 30 : g = gimple_build_cond (LE_EXPR, idx2_next, p, NULL_TREE, NULL_TREE);
6485 30 : insert_before (g);
6486 30 : m_gsi = gsi_after_labels (edge_false->dest);
6487 30 : g = gimple_build_assign (t_next, RSHIFT_EXPR, t, integer_one_node);
6488 30 : insert_before (g);
6489 30 : g = gimple_build_assign (i_next, PLUS_EXPR, i, size_one_node);
6490 30 : insert_before (g);
6491 30 : g = gimple_build_cond (NE_EXPR, t_next, build_zero_cst (m_limb_type),
6492 : NULL_TREE, NULL_TREE);
6493 30 : insert_before (g);
6494 30 : if ((EDGE_SUCC (edge_false->dest, 0)->flags & EDGE_FALSE_VALUE) != 0)
6495 30 : m_gsi = gsi_after_labels (EDGE_SUCC (edge_false->dest, 0)->dest);
6496 : else
6497 0 : m_gsi = gsi_after_labels (EDGE_SUCC (edge_false->dest, 1)->dest);
6498 30 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
6499 0 : bitint_big_endian ? size_int (-1) : size_one_node);
6500 30 : insert_before (g);
6501 30 : if (bitint_big_endian)
6502 0 : g = gimple_build_cond (NE_EXPR, idx_next, size_int (-1),
6503 : NULL_TREE, NULL_TREE);
6504 : else
6505 30 : g = gimple_build_cond (LE_EXPR, idx_next, p, NULL_TREE, NULL_TREE);
6506 30 : insert_before (g);
6507 30 : m_gsi = gsi_for_stmt (stmt);
6508 30 : if (bitint_extended && prec % limb_prec != 0)
6509 : {
6510 : /* The most significant limb has been updated either in the
6511 : loop or in the if after it. To simplify the code, just
6512 : read it back from memory and extend. */
6513 0 : idx = bitint_big_endian ? size_zero_node : p;
6514 0 : tree l = limb_access (TREE_TYPE (lhs), obj, idx, true);
6515 0 : tree type = limb_access_type (TREE_TYPE (lhs), idx);
6516 0 : tree v = make_ssa_name (m_limb_type);
6517 0 : g = gimple_build_assign (v, l);
6518 0 : insert_before (g);
6519 0 : v = add_cast (type, v);
6520 0 : l = limb_access (TREE_TYPE (lhs), obj, idx, true);
6521 0 : v = add_cast (m_limb_type, v);
6522 0 : g = gimple_build_assign (l, v);
6523 0 : insert_before (g);
6524 : }
6525 30 : if (bitint_extended == bitint_ext_full
6526 0 : && abi_limb_prec > limb_prec
6527 0 : && (CEIL (prec, abi_limb_prec) * abi_limb_prec
6528 0 : > CEIL (prec, limb_prec) * limb_prec))
6529 : {
6530 0 : tree p2 = build_int_cst (sizetype,
6531 : CEIL (prec, abi_limb_prec)
6532 0 : * abi_limb_prec / limb_prec - 1);
6533 0 : tree l = limb_access (TREE_TYPE (lhs), obj, p2, true);
6534 0 : g = gimple_build_assign (l, build_zero_cst (m_limb_type));
6535 0 : insert_before (g);
6536 : }
6537 : }
6538 :
6539 : /* Lower a call statement with one or more large/huge _BitInt
6540 : arguments or large/huge _BitInt return value. */
6541 :
6542 : void
6543 9012 : bitint_large_huge::lower_call (tree obj, gimple *stmt)
6544 : {
6545 9012 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
6546 9012 : unsigned int nargs = gimple_call_num_args (stmt);
6547 9012 : if (gimple_call_internal_p (stmt))
6548 4390 : switch (gimple_call_internal_fn (stmt))
6549 : {
6550 2733 : case IFN_ADD_OVERFLOW:
6551 2733 : case IFN_SUB_OVERFLOW:
6552 2733 : case IFN_UBSAN_CHECK_ADD:
6553 2733 : case IFN_UBSAN_CHECK_SUB:
6554 2733 : lower_addsub_overflow (obj, stmt);
6555 7123 : return;
6556 1479 : case IFN_MUL_OVERFLOW:
6557 1479 : case IFN_UBSAN_CHECK_MUL:
6558 1479 : lower_mul_overflow (obj, stmt);
6559 1479 : return;
6560 101 : case IFN_CLZ:
6561 101 : case IFN_CTZ:
6562 101 : case IFN_CLRSB:
6563 101 : case IFN_FFS:
6564 101 : case IFN_PARITY:
6565 101 : case IFN_POPCOUNT:
6566 101 : lower_bit_query (stmt);
6567 101 : return;
6568 47 : case IFN_BSWAP:
6569 47 : case IFN_BITREVERSE:
6570 47 : lower_bswap_bitreverse (obj, stmt);
6571 47 : return;
6572 30 : case IFN_CLMUL:
6573 30 : lower_clmul (obj, stmt);
6574 30 : return;
6575 : default:
6576 : break;
6577 : }
6578 4622 : bool returns_twice = (gimple_call_flags (stmt) & ECF_RETURNS_TWICE) != 0;
6579 10542 : for (unsigned int i = 0; i < nargs; ++i)
6580 : {
6581 5920 : tree arg = gimple_call_arg (stmt, i);
6582 9497 : if (TREE_CODE (arg) != SSA_NAME
6583 2440 : || !BITINT_TYPE_P (TREE_TYPE (arg))
6584 8298 : || bitint_precision_kind (TREE_TYPE (arg)) <= bitint_prec_middle)
6585 3577 : continue;
6586 2343 : if (SSA_NAME_IS_DEFAULT_DEF (arg)
6587 2343 : && (!SSA_NAME_VAR (arg) || VAR_P (SSA_NAME_VAR (arg))))
6588 : {
6589 1 : tree var = create_tmp_reg (TREE_TYPE (arg));
6590 1 : arg = get_or_create_ssa_default_def (cfun, var);
6591 : }
6592 : else
6593 : {
6594 2342 : int p = var_to_partition (m_map, arg);
6595 2342 : tree v = m_vars[p];
6596 2342 : gcc_assert (v != NULL_TREE);
6597 2342 : if (!types_compatible_p (TREE_TYPE (arg), TREE_TYPE (v)))
6598 2322 : v = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (arg), v);
6599 2342 : arg = make_ssa_name (TREE_TYPE (arg));
6600 2342 : gimple *g = gimple_build_assign (arg, v);
6601 2342 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
6602 2342 : if (returns_twice && bb_has_abnormal_pred (gimple_bb (stmt)))
6603 : {
6604 11 : m_returns_twice_calls.safe_push (stmt);
6605 11 : returns_twice = false;
6606 : }
6607 : }
6608 2343 : gimple_call_set_arg (stmt, i, arg);
6609 2343 : if (m_preserved == NULL)
6610 412 : m_preserved = BITMAP_ALLOC (NULL);
6611 2343 : bitmap_set_bit (m_preserved, SSA_NAME_VERSION (arg));
6612 : }
6613 4622 : tree lhs = gimple_call_lhs (stmt);
6614 4622 : if (lhs
6615 4491 : && TREE_CODE (lhs) == SSA_NAME
6616 4491 : && BITINT_TYPE_P (TREE_TYPE (lhs))
6617 9035 : && bitint_precision_kind (TREE_TYPE (lhs)) >= bitint_prec_large)
6618 : {
6619 4387 : int p = var_to_partition (m_map, lhs);
6620 4387 : tree v = m_vars[p];
6621 4387 : gcc_assert (v != NULL_TREE);
6622 4387 : if (!types_compatible_p (TREE_TYPE (lhs), TREE_TYPE (v)))
6623 4387 : v = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (lhs), v);
6624 4387 : gimple_call_set_lhs (stmt, v);
6625 4387 : SSA_NAME_DEF_STMT (lhs) = gimple_build_nop ();
6626 : }
6627 4622 : update_stmt (stmt);
6628 : }
6629 :
6630 : /* Lower __asm STMT which involves large/huge _BitInt values. */
6631 :
6632 : void
6633 3 : bitint_large_huge::lower_asm (gimple *stmt)
6634 : {
6635 3 : gasm *g = as_a <gasm *> (stmt);
6636 3 : unsigned noutputs = gimple_asm_noutputs (g);
6637 3 : unsigned ninputs = gimple_asm_ninputs (g);
6638 :
6639 5 : for (unsigned i = 0; i < noutputs; ++i)
6640 : {
6641 2 : tree t = gimple_asm_output_op (g, i);
6642 2 : tree s = TREE_VALUE (t);
6643 2 : if (TREE_CODE (s) == SSA_NAME
6644 1 : && BITINT_TYPE_P (TREE_TYPE (s))
6645 3 : && bitint_precision_kind (TREE_TYPE (s)) >= bitint_prec_large)
6646 : {
6647 1 : int part = var_to_partition (m_map, s);
6648 1 : gcc_assert (m_vars[part] != NULL_TREE);
6649 1 : TREE_VALUE (t) = m_vars[part];
6650 : }
6651 : }
6652 8 : for (unsigned i = 0; i < ninputs; ++i)
6653 : {
6654 5 : tree t = gimple_asm_input_op (g, i);
6655 5 : tree s = TREE_VALUE (t);
6656 5 : if (TREE_CODE (s) == SSA_NAME
6657 4 : && BITINT_TYPE_P (TREE_TYPE (s))
6658 9 : && bitint_precision_kind (TREE_TYPE (s)) >= bitint_prec_large)
6659 : {
6660 4 : if (SSA_NAME_IS_DEFAULT_DEF (s)
6661 4 : && (!SSA_NAME_VAR (s) || VAR_P (SSA_NAME_VAR (s))))
6662 : {
6663 1 : TREE_VALUE (t) = create_tmp_var (TREE_TYPE (s), "bitint");
6664 1 : mark_addressable (TREE_VALUE (t));
6665 : }
6666 : else
6667 : {
6668 3 : int part = var_to_partition (m_map, s);
6669 3 : gcc_assert (m_vars[part] != NULL_TREE);
6670 3 : TREE_VALUE (t) = m_vars[part];
6671 : }
6672 : }
6673 : }
6674 3 : update_stmt (stmt);
6675 3 : }
6676 :
6677 : /* Lower statement STMT which involves large/huge _BitInt values
6678 : into code accessing individual limbs. */
6679 :
6680 : void
6681 43332 : bitint_large_huge::lower_stmt (gimple *stmt)
6682 : {
6683 43332 : m_first = true;
6684 43332 : m_lhs = NULL_TREE;
6685 43332 : m_data.truncate (0);
6686 43332 : m_data_cnt = 0;
6687 43332 : m_gsi = gsi_for_stmt (stmt);
6688 43332 : m_after_stmt = NULL;
6689 43332 : m_bb = NULL;
6690 43332 : m_init_gsi = m_gsi;
6691 43332 : gsi_prev (&m_init_gsi);
6692 43332 : m_preheader_bb = NULL;
6693 43332 : m_upwards_2limb = 0;
6694 43332 : m_upwards = false;
6695 43332 : m_var_msb = false;
6696 43332 : m_cast_conditional = false;
6697 43332 : m_bitfld_load = 0;
6698 43332 : m_loc = gimple_location (stmt);
6699 43332 : if (is_gimple_call (stmt))
6700 : {
6701 7325 : lower_call (NULL_TREE, stmt);
6702 7325 : return;
6703 : }
6704 36007 : if (gimple_code (stmt) == GIMPLE_ASM)
6705 : {
6706 3 : lower_asm (stmt);
6707 3 : return;
6708 : }
6709 36004 : tree lhs = NULL_TREE, cmp_op1 = NULL_TREE, cmp_op2 = NULL_TREE;
6710 36004 : tree_code cmp_code = comparison_op (stmt, &cmp_op1, &cmp_op2);
6711 36004 : bool eq_p = (cmp_code == EQ_EXPR || cmp_code == NE_EXPR);
6712 36004 : bool mergeable_cast_p = false;
6713 36004 : bool final_cast_p = false;
6714 36004 : if (gimple_assign_cast_p (stmt))
6715 : {
6716 5459 : lhs = gimple_assign_lhs (stmt);
6717 5459 : tree rhs1 = gimple_assign_rhs1 (stmt);
6718 5459 : if (TREE_CODE (rhs1) == VIEW_CONVERT_EXPR)
6719 46 : rhs1 = TREE_OPERAND (rhs1, 0);
6720 9680 : if (BITINT_TYPE_P (TREE_TYPE (lhs))
6721 1238 : && bitint_precision_kind (TREE_TYPE (lhs)) >= bitint_prec_large
6722 6661 : && INTEGRAL_TYPE_P (TREE_TYPE (rhs1)))
6723 : mergeable_cast_p = true;
6724 4547 : else if (BITINT_TYPE_P (TREE_TYPE (rhs1))
6725 4257 : && bitint_precision_kind (TREE_TYPE (rhs1)) >= bitint_prec_large
6726 8659 : && (INTEGRAL_TYPE_P (TREE_TYPE (lhs))
6727 36 : || POINTER_TYPE_P (TREE_TYPE (lhs))
6728 35 : || gimple_assign_rhs_code (stmt) == VIEW_CONVERT_EXPR))
6729 : {
6730 4257 : final_cast_p = true;
6731 4257 : if (((TREE_CODE (TREE_TYPE (lhs)) == INTEGER_TYPE
6732 572 : && TYPE_PRECISION (TREE_TYPE (lhs)) > MAX_FIXED_MODE_SIZE)
6733 4257 : || (!INTEGRAL_TYPE_P (TREE_TYPE (lhs))
6734 36 : && !POINTER_TYPE_P (TREE_TYPE (lhs))))
6735 4292 : && gimple_assign_rhs_code (stmt) == VIEW_CONVERT_EXPR)
6736 : {
6737 : /* Handle VIEW_CONVERT_EXPRs to not generally supported
6738 : huge INTEGER_TYPEs like uint256_t or uint512_t. These
6739 : are usually emitted from memcpy folding and backends
6740 : support moves with them but that is usually it.
6741 : Similarly handle VCEs to vector/complex types etc. */
6742 35 : gcc_assert (TREE_CODE (rhs1) == SSA_NAME);
6743 35 : if (SSA_NAME_IS_DEFAULT_DEF (rhs1)
6744 35 : && (!SSA_NAME_VAR (rhs1) || VAR_P (SSA_NAME_VAR (rhs1))))
6745 : {
6746 0 : tree var = create_tmp_reg (TREE_TYPE (lhs));
6747 0 : rhs1 = get_or_create_ssa_default_def (cfun, var);
6748 0 : gimple_assign_set_rhs1 (stmt, rhs1);
6749 0 : gimple_assign_set_rhs_code (stmt, SSA_NAME);
6750 : }
6751 35 : else if (m_names == NULL
6752 35 : || !bitmap_bit_p (m_names, SSA_NAME_VERSION (rhs1)))
6753 : {
6754 0 : gimple *g = SSA_NAME_DEF_STMT (rhs1);
6755 0 : gcc_assert (gimple_assign_load_p (g));
6756 0 : tree mem = gimple_assign_rhs1 (g);
6757 0 : tree ltype = TREE_TYPE (lhs);
6758 0 : addr_space_t as = TYPE_ADDR_SPACE (TREE_TYPE (mem));
6759 0 : if (as != TYPE_ADDR_SPACE (ltype))
6760 0 : ltype
6761 0 : = build_qualified_type (ltype,
6762 0 : TYPE_QUALS (ltype).with_as (as));
6763 0 : rhs1 = build1 (VIEW_CONVERT_EXPR, ltype, unshare_expr (mem));
6764 0 : gimple_assign_set_rhs1 (stmt, rhs1);
6765 : }
6766 : else
6767 : {
6768 35 : int part = var_to_partition (m_map, rhs1);
6769 35 : gcc_assert (m_vars[part] != NULL_TREE);
6770 35 : rhs1 = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (lhs),
6771 : m_vars[part]);
6772 35 : gimple_assign_set_rhs1 (stmt, rhs1);
6773 : }
6774 35 : update_stmt (stmt);
6775 35 : return;
6776 : }
6777 4222 : if (TREE_CODE (rhs1) == SSA_NAME
6778 4222 : && (m_names == NULL
6779 4185 : || !bitmap_bit_p (m_names, SSA_NAME_VERSION (rhs1))))
6780 : {
6781 1723 : gimple *g = SSA_NAME_DEF_STMT (rhs1);
6782 1723 : if (is_gimple_assign (g)
6783 1723 : && gimple_assign_rhs_code (g) == IMAGPART_EXPR)
6784 : {
6785 1640 : tree rhs2 = TREE_OPERAND (gimple_assign_rhs1 (g), 0);
6786 1640 : if (TREE_CODE (rhs2) == SSA_NAME
6787 1640 : && (m_names == NULL
6788 1603 : || !bitmap_bit_p (m_names, SSA_NAME_VERSION (rhs2))))
6789 : {
6790 1640 : g = SSA_NAME_DEF_STMT (rhs2);
6791 1640 : int ovf = optimizable_arith_overflow (g);
6792 1640 : if (ovf == 2)
6793 : /* If .{ADD,SUB,MUL}_OVERFLOW has both REALPART_EXPR
6794 : and IMAGPART_EXPR uses, where the latter is cast to
6795 : non-_BitInt, it will be optimized when handling
6796 : the REALPART_EXPR. */
6797 : return;
6798 91 : if (ovf == 1)
6799 : {
6800 91 : lower_call (NULL_TREE, g);
6801 91 : return;
6802 : }
6803 : }
6804 : }
6805 : }
6806 : }
6807 145 : else if (BITINT_TYPE_P (TREE_TYPE (lhs))
6808 145 : && bitint_precision_kind (TREE_TYPE (lhs)) >= bitint_prec_large
6809 145 : && !INTEGRAL_TYPE_P (TREE_TYPE (rhs1))
6810 145 : && !POINTER_TYPE_P (TREE_TYPE (rhs1))
6811 290 : && gimple_assign_rhs_code (stmt) == VIEW_CONVERT_EXPR)
6812 : {
6813 10 : int part = var_to_partition (m_map, lhs);
6814 10 : gcc_assert (m_vars[part] != NULL_TREE);
6815 10 : lhs = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (rhs1), m_vars[part]);
6816 10 : insert_before (gimple_build_assign (lhs, rhs1));
6817 10 : return;
6818 : }
6819 : }
6820 34319 : if (gimple_store_p (stmt))
6821 : {
6822 9086 : tree rhs1 = gimple_assign_rhs1 (stmt);
6823 9086 : if (TREE_CODE (rhs1) == SSA_NAME
6824 9086 : && (m_names == NULL
6825 8016 : || !bitmap_bit_p (m_names, SSA_NAME_VERSION (rhs1))))
6826 : {
6827 1681 : gimple *g = SSA_NAME_DEF_STMT (rhs1);
6828 1681 : m_loc = gimple_location (g);
6829 1681 : lhs = gimple_assign_lhs (stmt);
6830 1681 : if (is_gimple_assign (g) && !mergeable_op (g))
6831 667 : switch (gimple_assign_rhs_code (g))
6832 : {
6833 119 : case LSHIFT_EXPR:
6834 119 : case RSHIFT_EXPR:
6835 119 : lower_shift_stmt (lhs, g);
6836 510 : handled:
6837 510 : m_gsi = gsi_for_stmt (stmt);
6838 510 : unlink_stmt_vdef (stmt);
6839 1020 : release_ssa_name (gimple_vdef (stmt));
6840 510 : gsi_remove (&m_gsi, true);
6841 510 : return;
6842 207 : case MULT_EXPR:
6843 207 : case TRUNC_DIV_EXPR:
6844 207 : case EXACT_DIV_EXPR:
6845 207 : case TRUNC_MOD_EXPR:
6846 207 : lower_muldiv_stmt (lhs, g);
6847 207 : goto handled;
6848 44 : case FIX_TRUNC_EXPR:
6849 44 : lower_float_conv_stmt (lhs, g);
6850 44 : goto handled;
6851 93 : case REALPART_EXPR:
6852 93 : case IMAGPART_EXPR:
6853 93 : lower_cplxpart_stmt (lhs, g);
6854 93 : goto handled;
6855 8 : case VIEW_CONVERT_EXPR:
6856 8 : {
6857 8 : tree rhs1 = gimple_assign_rhs1 (g);
6858 8 : rhs1 = TREE_OPERAND (rhs1, 0);
6859 8 : if (!INTEGRAL_TYPE_P (TREE_TYPE (rhs1))
6860 7 : && !POINTER_TYPE_P (TREE_TYPE (rhs1)))
6861 : {
6862 7 : tree ltype = TREE_TYPE (rhs1);
6863 7 : addr_space_t as = TYPE_ADDR_SPACE (TREE_TYPE (lhs));
6864 7 : ltype
6865 7 : = build_qualified_type (ltype,
6866 7 : TYPE_QUALS (TREE_TYPE (lhs))
6867 : .with_as (as));
6868 7 : lhs = build1 (VIEW_CONVERT_EXPR, ltype, lhs);
6869 7 : gimple_assign_set_lhs (stmt, lhs);
6870 7 : gimple_assign_set_rhs1 (stmt, rhs1);
6871 7 : gimple_assign_set_rhs_code (stmt, TREE_CODE (rhs1));
6872 7 : update_stmt (stmt);
6873 7 : return;
6874 : }
6875 : }
6876 : break;
6877 : default:
6878 : break;
6879 : }
6880 1014 : else if (optimizable_arith_overflow (g) == 3)
6881 : {
6882 24 : lower_call (lhs, g);
6883 24 : goto handled;
6884 : }
6885 990 : else if (is_gimple_call (g) && gimple_call_internal_p (g))
6886 23 : switch (gimple_call_internal_fn (g))
6887 : {
6888 23 : case IFN_BSWAP:
6889 23 : case IFN_BITREVERSE:
6890 23 : case IFN_CLMUL:
6891 23 : lower_call (lhs, g);
6892 23 : goto handled;
6893 : default:
6894 : break;
6895 : }
6896 :
6897 1164 : m_loc = gimple_location (stmt);
6898 : }
6899 : }
6900 33802 : if (mergeable_op (stmt)
6901 22277 : || gimple_store_p (stmt)
6902 22277 : || gimple_assign_load_p (stmt)
6903 : || eq_p
6904 17451 : || mergeable_cast_p
6905 43997 : || (is_gimple_assign (stmt)
6906 9807 : && gimple_assign_rhs_code (stmt) == PAREN_EXPR))
6907 : {
6908 23609 : lhs = lower_mergeable_stmt (stmt, cmp_code, cmp_op1, cmp_op2);
6909 23609 : if (!eq_p)
6910 : return;
6911 : }
6912 10193 : else if (cmp_code != ERROR_MARK)
6913 722 : lhs = lower_comparison_stmt (stmt, cmp_code, cmp_op1, cmp_op2);
6914 16858 : if (cmp_code != ERROR_MARK)
6915 : {
6916 7387 : if (gimple_code (stmt) == GIMPLE_COND)
6917 : {
6918 6613 : gcond *cstmt = as_a <gcond *> (stmt);
6919 6613 : gimple_cond_set_lhs (cstmt, lhs);
6920 6613 : gimple_cond_set_rhs (cstmt, boolean_false_node);
6921 6613 : gimple_cond_set_code (cstmt, cmp_code);
6922 6613 : update_stmt (stmt);
6923 6613 : return;
6924 : }
6925 774 : if (gimple_assign_rhs_code (stmt) == COND_EXPR)
6926 : {
6927 0 : tree cond = build2 (cmp_code, boolean_type_node, lhs,
6928 : boolean_false_node);
6929 0 : gimple_assign_set_rhs1 (stmt, cond);
6930 0 : lhs = gimple_assign_lhs (stmt);
6931 0 : gcc_assert (!BITINT_TYPE_P (TREE_TYPE (lhs))
6932 : || (bitint_precision_kind (TREE_TYPE (lhs))
6933 : <= bitint_prec_middle));
6934 0 : update_stmt (stmt);
6935 0 : return;
6936 : }
6937 774 : gimple_assign_set_rhs1 (stmt, lhs);
6938 774 : gimple_assign_set_rhs2 (stmt, boolean_false_node);
6939 774 : gimple_assign_set_rhs_code (stmt, cmp_code);
6940 774 : update_stmt (stmt);
6941 774 : return;
6942 : }
6943 9471 : if (final_cast_p)
6944 : {
6945 2582 : tree lhs_type = TREE_TYPE (lhs);
6946 : /* Add support for 3 or more limbs filled in from normal integral
6947 : type if this assert fails. If no target chooses limb mode smaller
6948 : than half of largest supported normal integral type, this will not
6949 : be needed. */
6950 2582 : gcc_assert (TYPE_PRECISION (lhs_type) <= 2 * limb_prec);
6951 2582 : gimple *g;
6952 2546 : if ((BITINT_TYPE_P (lhs_type)
6953 36 : && bitint_precision_kind (lhs_type) == bitint_prec_middle)
6954 5149 : || POINTER_TYPE_P (lhs_type))
6955 16 : lhs_type = build_nonstandard_integer_type (TYPE_PRECISION (lhs_type),
6956 16 : TYPE_UNSIGNED (lhs_type));
6957 2582 : m_data_cnt = 0;
6958 2582 : tree rhs1 = gimple_assign_rhs1 (stmt);
6959 2582 : unsigned int prec = TYPE_PRECISION (TREE_TYPE (rhs1));
6960 2582 : unsigned int cnt = CEIL (prec, limb_prec);
6961 2582 : tree r1 = handle_operand (rhs1, size_int (bitint_big_endian
6962 : ? cnt - 1 : 0));
6963 2582 : if (!useless_type_conversion_p (lhs_type, TREE_TYPE (r1)))
6964 2473 : r1 = add_cast (lhs_type, r1);
6965 2582 : if (TYPE_PRECISION (lhs_type) > limb_prec)
6966 : {
6967 70 : m_data_cnt = 0;
6968 70 : m_first = false;
6969 70 : tree r2 = handle_operand (rhs1, size_int (bitint_big_endian
6970 : ? cnt - 2 : 1));
6971 70 : r2 = add_cast (lhs_type, r2);
6972 70 : g = gimple_build_assign (make_ssa_name (lhs_type), LSHIFT_EXPR, r2,
6973 : build_int_cst (unsigned_type_node,
6974 70 : limb_prec));
6975 70 : insert_before (g);
6976 70 : g = gimple_build_assign (make_ssa_name (lhs_type), BIT_IOR_EXPR, r1,
6977 : gimple_assign_lhs (g));
6978 70 : insert_before (g);
6979 70 : r1 = gimple_assign_lhs (g);
6980 : }
6981 2582 : if (lhs_type != TREE_TYPE (lhs))
6982 16 : g = gimple_build_assign (lhs, NOP_EXPR, r1);
6983 : else
6984 2566 : g = gimple_build_assign (lhs, r1);
6985 2582 : gsi_replace (&m_gsi, g, true);
6986 2582 : return;
6987 : }
6988 6889 : if (is_gimple_assign (stmt))
6989 6889 : switch (gimple_assign_rhs_code (stmt))
6990 : {
6991 483 : case LSHIFT_EXPR:
6992 483 : case RSHIFT_EXPR:
6993 483 : lower_shift_stmt (NULL_TREE, stmt);
6994 483 : return;
6995 166 : case MULT_EXPR:
6996 166 : case TRUNC_DIV_EXPR:
6997 166 : case EXACT_DIV_EXPR:
6998 166 : case TRUNC_MOD_EXPR:
6999 166 : lower_muldiv_stmt (NULL_TREE, stmt);
7000 166 : return;
7001 283 : case FIX_TRUNC_EXPR:
7002 283 : case FLOAT_EXPR:
7003 283 : lower_float_conv_stmt (NULL_TREE, stmt);
7004 283 : return;
7005 5929 : case REALPART_EXPR:
7006 5929 : case IMAGPART_EXPR:
7007 5929 : lower_cplxpart_stmt (NULL_TREE, stmt);
7008 5929 : return;
7009 28 : case COMPLEX_EXPR:
7010 28 : lower_complexexpr_stmt (stmt);
7011 28 : return;
7012 : default:
7013 : break;
7014 : }
7015 0 : gcc_unreachable ();
7016 : }
7017 :
7018 : /* Helper for walk_non_aliased_vuses. Determine if we arrived at
7019 : the desired memory state. */
7020 :
7021 : void *
7022 2190 : vuse_eq (ao_ref *, tree vuse1, void *data)
7023 : {
7024 2190 : tree vuse2 = (tree) data;
7025 2190 : if (vuse1 == vuse2)
7026 820 : return data;
7027 :
7028 : return NULL;
7029 : }
7030 :
7031 : /* Return true if STMT uses a library function and needs to take
7032 : address of its inputs. We need to avoid bit-fields in those
7033 : cases. Similarly, we need to avoid overlap between destination
7034 : and source limb arrays. */
7035 :
7036 : bool
7037 15561 : stmt_needs_operand_addr (gimple *stmt)
7038 : {
7039 15561 : if (is_gimple_assign (stmt))
7040 10374 : switch (gimple_assign_rhs_code (stmt))
7041 : {
7042 585 : case MULT_EXPR:
7043 585 : case TRUNC_DIV_EXPR:
7044 585 : case EXACT_DIV_EXPR:
7045 585 : case TRUNC_MOD_EXPR:
7046 585 : case FLOAT_EXPR:
7047 585 : return true;
7048 : default:
7049 : break;
7050 : }
7051 5187 : else if (is_gimple_call (stmt) && gimple_call_internal_p (stmt))
7052 46 : switch (gimple_call_internal_fn (stmt))
7053 : {
7054 17 : case IFN_MUL_OVERFLOW:
7055 17 : case IFN_UBSAN_CHECK_MUL:
7056 17 : return true;
7057 : /* These three actually don't take address, but reshuffle
7058 : all bytes or bits, so need similar treatment. */
7059 4 : case IFN_BSWAP:
7060 4 : case IFN_BITREVERSE:
7061 4 : case IFN_CLMUL:
7062 4 : return true;
7063 : default:
7064 : break;
7065 : }
7066 : return false;
7067 : }
7068 :
7069 : /* Dominator walker used to discover which large/huge _BitInt
7070 : loads could be sunk into all their uses. */
7071 :
7072 634 : class bitint_dom_walker : public dom_walker
7073 : {
7074 : public:
7075 317 : bitint_dom_walker (bitmap names, bitmap loads)
7076 634 : : dom_walker (CDI_DOMINATORS), m_names (names), m_loads (loads) {}
7077 :
7078 : edge before_dom_children (basic_block) final override;
7079 :
7080 : private:
7081 : bitmap m_names, m_loads;
7082 : };
7083 :
7084 : edge
7085 4784 : bitint_dom_walker::before_dom_children (basic_block bb)
7086 : {
7087 4784 : gphi *phi = get_virtual_phi (bb);
7088 4784 : tree vop;
7089 4784 : if (phi)
7090 800 : vop = gimple_phi_result (phi);
7091 3984 : else if (bb == ENTRY_BLOCK_PTR_FOR_FN (cfun))
7092 : vop = NULL_TREE;
7093 : else
7094 3667 : vop = (tree) get_immediate_dominator (CDI_DOMINATORS, bb)->aux;
7095 :
7096 4784 : auto_vec<tree, 16> worklist;
7097 9568 : for (gimple_stmt_iterator gsi = gsi_start_bb (bb);
7098 20734 : !gsi_end_p (gsi); gsi_next (&gsi))
7099 : {
7100 15950 : gimple *stmt = gsi_stmt (gsi);
7101 15950 : if (is_gimple_debug (stmt))
7102 2769 : continue;
7103 :
7104 16000 : if (!vop && gimple_vuse (stmt))
7105 : vop = gimple_vuse (stmt);
7106 :
7107 15561 : tree cvop = vop;
7108 29098 : if (gimple_vdef (stmt))
7109 : vop = gimple_vdef (stmt);
7110 :
7111 15561 : tree lhs = gimple_get_lhs (stmt);
7112 17941 : if (lhs
7113 11163 : && TREE_CODE (lhs) == SSA_NAME
7114 8912 : && BITINT_TYPE_P (TREE_TYPE (lhs))
7115 5937 : && bitint_precision_kind (TREE_TYPE (lhs)) >= bitint_prec_large
7116 21308 : && !bitmap_bit_p (m_names, SSA_NAME_VERSION (lhs)))
7117 : /* If lhs of stmt is large/huge _BitInt SSA_NAME not in m_names,
7118 : it means it will be handled in a loop or straight line code
7119 : at the location of its (ultimate) immediate use, so for
7120 : vop checking purposes check these only at the ultimate
7121 : immediate use. */
7122 2380 : continue;
7123 :
7124 13181 : ssa_op_iter oi;
7125 13181 : use_operand_p use_p;
7126 22193 : FOR_EACH_SSA_USE_OPERAND (use_p, stmt, oi, SSA_OP_USE)
7127 : {
7128 9012 : tree s = USE_FROM_PTR (use_p);
7129 14534 : if (BITINT_TYPE_P (TREE_TYPE (s))
7130 9012 : && bitint_precision_kind (TREE_TYPE (s)) >= bitint_prec_large)
7131 3293 : worklist.safe_push (s);
7132 : }
7133 :
7134 13181 : bool needs_operand_addr = stmt_needs_operand_addr (stmt);
7135 32495 : while (worklist.length () > 0)
7136 : {
7137 6133 : tree s = worklist.pop ();
7138 :
7139 6133 : if (!bitmap_bit_p (m_names, SSA_NAME_VERSION (s)))
7140 : {
7141 2380 : gimple *g = SSA_NAME_DEF_STMT (s);
7142 2380 : needs_operand_addr |= stmt_needs_operand_addr (g);
7143 5459 : FOR_EACH_SSA_USE_OPERAND (use_p, g, oi, SSA_OP_USE)
7144 : {
7145 3079 : tree s2 = USE_FROM_PTR (use_p);
7146 3269 : if (BITINT_TYPE_P (TREE_TYPE (s2))
7147 3079 : && (bitint_precision_kind (TREE_TYPE (s2))
7148 : >= bitint_prec_large))
7149 2840 : worklist.safe_push (s2);
7150 : }
7151 3116 : continue;
7152 2380 : }
7153 3753 : if (!SSA_NAME_OCCURS_IN_ABNORMAL_PHI (s)
7154 3753 : && gimple_assign_cast_p (SSA_NAME_DEF_STMT (s)))
7155 : {
7156 226 : tree rhs = gimple_assign_rhs1 (SSA_NAME_DEF_STMT (s));
7157 396 : if (TREE_CODE (rhs) == SSA_NAME
7158 226 : && bitmap_bit_p (m_loads, SSA_NAME_VERSION (rhs)))
7159 : s = rhs;
7160 : else
7161 170 : continue;
7162 : }
7163 3527 : else if (!bitmap_bit_p (m_loads, SSA_NAME_VERSION (s)))
7164 564 : continue;
7165 :
7166 3019 : gimple *g = SSA_NAME_DEF_STMT (s);
7167 3019 : tree rhs1 = gimple_assign_rhs1 (g);
7168 3019 : if (needs_operand_addr
7169 218 : && TREE_CODE (rhs1) == COMPONENT_REF
7170 3036 : && DECL_BIT_FIELD_TYPE (TREE_OPERAND (rhs1, 1)))
7171 : {
7172 4 : tree fld = TREE_OPERAND (rhs1, 1);
7173 : /* For little-endian, we can allow as inputs bit-fields
7174 : which start at a limb boundary. */
7175 6 : if (!bitint_big_endian
7176 4 : && DECL_OFFSET_ALIGN (fld) >= TYPE_ALIGN (TREE_TYPE (rhs1))
7177 4 : && tree_fits_uhwi_p (DECL_FIELD_BIT_OFFSET (fld))
7178 4 : && (tree_to_uhwi (DECL_FIELD_BIT_OFFSET (fld))
7179 4 : % limb_prec) == 0)
7180 : ;
7181 : else
7182 : {
7183 2 : bitmap_clear_bit (m_loads, SSA_NAME_VERSION (s));
7184 2 : continue;
7185 : }
7186 : }
7187 :
7188 3017 : ao_ref ref;
7189 3017 : ao_ref_init (&ref, rhs1);
7190 3017 : tree lvop = gimple_vuse (g);
7191 3017 : unsigned limit = 64;
7192 3017 : tree vuse = cvop;
7193 3017 : if (vop != cvop
7194 1338 : && is_gimple_assign (stmt)
7195 1336 : && gimple_store_p (stmt)
7196 4353 : && (needs_operand_addr
7197 1143 : || !operand_equal_p (lhs, gimple_assign_rhs1 (g), 0)))
7198 : vuse = vop;
7199 3017 : if (vuse != lvop
7200 3017 : && walk_non_aliased_vuses (&ref, vuse, false, vuse_eq,
7201 : NULL, NULL, NULL, limit, lvop) == NULL)
7202 532 : bitmap_clear_bit (m_loads, SSA_NAME_VERSION (s));
7203 : }
7204 : }
7205 :
7206 4784 : bb->aux = (void *) vop;
7207 4784 : return NULL;
7208 4784 : }
7209 :
7210 : }
7211 :
7212 : /* Replacement for normal processing of STMT in tree-ssa-coalesce.cc
7213 : build_ssa_conflict_graph.
7214 : The differences are:
7215 : 1) don't process assignments with large/huge _BitInt lhs not in NAMES
7216 : 2) for large/huge _BitInt multiplication/division/modulo process def
7217 : only after processing uses rather than before to make uses conflict
7218 : with the definition
7219 : 3) for large/huge _BitInt uses not in NAMES mark the uses of their
7220 : SSA_NAME_DEF_STMT (recursively), because those uses will be sunk into
7221 : the final statement. */
7222 :
7223 : void
7224 86418 : build_bitint_stmt_ssa_conflicts (gimple *stmt, live_track *live,
7225 : ssa_conflicts *graph, bitmap names,
7226 : void (*def) (live_track *, tree,
7227 : ssa_conflicts *),
7228 : void (*use) (live_track *, tree),
7229 : void (*clear) (live_track *, tree))
7230 : {
7231 86418 : bool muldiv_p = false;
7232 86418 : tree lhs = NULL_TREE;
7233 86418 : if (is_gimple_assign (stmt))
7234 : {
7235 47347 : lhs = gimple_assign_lhs (stmt);
7236 47347 : if (TREE_CODE (lhs) == SSA_NAME)
7237 : {
7238 33823 : tree type = TREE_TYPE (lhs);
7239 33823 : if (TREE_CODE (type) == COMPLEX_TYPE)
7240 73 : type = TREE_TYPE (type);
7241 13031 : if (BITINT_TYPE_P (type)
7242 33828 : && bitint_precision_kind (type) >= bitint_prec_large)
7243 : {
7244 20114 : if (!bitmap_bit_p (names, SSA_NAME_VERSION (lhs)))
7245 4955 : return;
7246 :
7247 : /* A copy between 2 partitions does not introduce an interference
7248 : by itself. If they did, you would never be able to coalesce
7249 : two things which are copied. If the two variables really do
7250 : conflict, they will conflict elsewhere in the program.
7251 :
7252 : This is handled by simply removing the SRC of the copy from
7253 : the live list, and processing the stmt normally.
7254 :
7255 : Don't do this if lhs is not in names though, in such cases
7256 : it is actually used at some point later in the basic
7257 : block. */
7258 15159 : if (gimple_assign_copy_p (stmt))
7259 : {
7260 1697 : tree rhs1 = gimple_assign_rhs1 (stmt);
7261 1697 : if (TREE_CODE (rhs1) == SSA_NAME)
7262 52 : clear (live, rhs1);
7263 : }
7264 :
7265 15159 : switch (gimple_assign_rhs_code (stmt))
7266 : {
7267 166 : case MULT_EXPR:
7268 166 : case TRUNC_DIV_EXPR:
7269 166 : case EXACT_DIV_EXPR:
7270 166 : case TRUNC_MOD_EXPR:
7271 166 : muldiv_p = true;
7272 : default:
7273 : break;
7274 : }
7275 : }
7276 : }
7277 : }
7278 39071 : else if (is_gimple_call (stmt) && gimple_call_internal_p (stmt))
7279 4363 : switch (gimple_call_internal_fn (stmt))
7280 : {
7281 2699 : case IFN_ADD_OVERFLOW:
7282 2699 : case IFN_SUB_OVERFLOW:
7283 2699 : case IFN_UBSAN_CHECK_ADD:
7284 2699 : case IFN_UBSAN_CHECK_SUB:
7285 2699 : if (bitint_big_endian)
7286 : {
7287 0 : lhs = gimple_call_lhs (stmt);
7288 0 : if (lhs)
7289 81463 : muldiv_p = true;
7290 : }
7291 : break;
7292 1540 : case IFN_MUL_OVERFLOW:
7293 1540 : case IFN_UBSAN_CHECK_MUL:
7294 1540 : case IFN_BSWAP:
7295 1540 : case IFN_BITREVERSE:
7296 1540 : case IFN_CLMUL:
7297 1540 : lhs = gimple_call_lhs (stmt);
7298 1540 : if (lhs)
7299 81463 : muldiv_p = true;
7300 : break;
7301 : default:
7302 : break;
7303 : }
7304 :
7305 162926 : auto_vec<tree, 16> worklist;
7306 81463 : ssa_op_iter iter;
7307 81463 : tree var;
7308 : /* On little-endian, mergeable ops process limbs from 0 up so except
7309 : for multiplication/division/modulo there is no risk in using the
7310 : same underlying variable for lhs and some operand, even when casts
7311 : are involved, the lhs limb is stored only after processing the source
7312 : limbs with the same index.
7313 : For multiplication/division/modulo, the libgcc library function requires
7314 : no aliasing between result and sources.
7315 : On big-endian, even mergeable ops limb processing can be problematic
7316 : though, because it can apply various index corrections e.g. when there
7317 : is a cast from operand with different number of limbs. So, make the
7318 : lhs conflict with all the operands which are (for now virtually) used on
7319 : the current stmt if there is any mismatch in the number of limbs between
7320 : operands and the lhs. */
7321 81463 : if (bitint_big_endian && lhs && !muldiv_p)
7322 : {
7323 0 : tree ltype = TREE_TYPE (lhs);
7324 0 : if (TREE_CODE (ltype) == COMPLEX_TYPE)
7325 : muldiv_p = true;
7326 0 : else if (TREE_CODE (lhs) == SSA_NAME
7327 0 : && BITINT_TYPE_P (ltype)
7328 0 : && bitint_precision_kind (ltype) >= bitint_prec_large)
7329 : {
7330 0 : unsigned lnelts = CEIL (TYPE_PRECISION (ltype), limb_prec);
7331 0 : FOR_EACH_SSA_TREE_OPERAND (var, stmt, iter, SSA_OP_USE)
7332 : {
7333 0 : tree type = TREE_TYPE (var);
7334 0 : if (TREE_CODE (type) == COMPLEX_TYPE)
7335 0 : type = TREE_TYPE (type);
7336 0 : if (BITINT_TYPE_P (type)
7337 0 : && bitint_precision_kind (type) >= bitint_prec_large)
7338 : {
7339 0 : if (bitmap_bit_p (names, SSA_NAME_VERSION (var)))
7340 : {
7341 0 : unsigned nelts = CEIL (TYPE_PRECISION (type), limb_prec);
7342 0 : if (TREE_CODE (TREE_TYPE (var)) == COMPLEX_TYPE
7343 0 : || lnelts != nelts)
7344 : {
7345 0 : muldiv_p = true;
7346 : break;
7347 : }
7348 : }
7349 : else
7350 0 : worklist.safe_push (var);
7351 : }
7352 : }
7353 :
7354 0 : while (!muldiv_p && worklist.length () > 0)
7355 : {
7356 0 : tree s = worklist.pop ();
7357 0 : FOR_EACH_SSA_TREE_OPERAND (var, SSA_NAME_DEF_STMT (s), iter,
7358 : SSA_OP_USE)
7359 : {
7360 0 : tree type = TREE_TYPE (var);
7361 0 : if (TREE_CODE (type) == COMPLEX_TYPE)
7362 0 : type = TREE_TYPE (type);
7363 0 : if (BITINT_TYPE_P (type)
7364 0 : && bitint_precision_kind (type) >= bitint_prec_large)
7365 : {
7366 0 : if (bitmap_bit_p (names, SSA_NAME_VERSION (var)))
7367 : {
7368 0 : unsigned nelts = CEIL (TYPE_PRECISION (type),
7369 : limb_prec);
7370 0 : if (TREE_CODE (TREE_TYPE (var)) == COMPLEX_TYPE
7371 0 : || lnelts != nelts)
7372 : {
7373 : muldiv_p = true;
7374 : break;
7375 : }
7376 : }
7377 : else
7378 0 : worklist.safe_push (var);
7379 : }
7380 : }
7381 : }
7382 0 : worklist.truncate (0);
7383 : }
7384 : }
7385 :
7386 81463 : if (!muldiv_p)
7387 : {
7388 : /* For stmts with more than one SSA_NAME definition pretend all the
7389 : SSA_NAME outputs but the first one are live at this point, so
7390 : that conflicts are added in between all those even when they are
7391 : actually not really live after the asm, because expansion might
7392 : copy those into pseudos after the asm and if multiple outputs
7393 : share the same partition, it might overwrite those that should
7394 : be live. E.g.
7395 : asm volatile (".." : "=r" (a) : "=r" (b) : "0" (a), "1" (a));
7396 : return a;
7397 : See PR70593. */
7398 79757 : bool first = true;
7399 116303 : FOR_EACH_SSA_TREE_OPERAND (var, stmt, iter, SSA_OP_DEF)
7400 36546 : if (first)
7401 : first = false;
7402 : else
7403 0 : use (live, var);
7404 :
7405 116303 : FOR_EACH_SSA_TREE_OPERAND (var, stmt, iter, SSA_OP_DEF)
7406 36546 : def (live, var, graph);
7407 : }
7408 :
7409 138309 : FOR_EACH_SSA_TREE_OPERAND (var, stmt, iter, SSA_OP_USE)
7410 : {
7411 56846 : tree type = TREE_TYPE (var);
7412 56846 : if (TREE_CODE (type) == COMPLEX_TYPE)
7413 6313 : type = TREE_TYPE (type);
7414 19311 : if (BITINT_TYPE_P (type)
7415 56863 : && bitint_precision_kind (type) >= bitint_prec_large)
7416 : {
7417 36446 : if (bitmap_bit_p (names, SSA_NAME_VERSION (var)))
7418 31290 : use (live, var);
7419 : else
7420 5156 : worklist.safe_push (var);
7421 : }
7422 : }
7423 :
7424 89629 : while (worklist.length () > 0)
7425 : {
7426 8166 : tree s = worklist.pop ();
7427 17092 : FOR_EACH_SSA_TREE_OPERAND (var, SSA_NAME_DEF_STMT (s), iter, SSA_OP_USE)
7428 : {
7429 8926 : tree type = TREE_TYPE (var);
7430 8926 : if (TREE_CODE (type) == COMPLEX_TYPE)
7431 1695 : type = TREE_TYPE (type);
7432 544 : if (BITINT_TYPE_P (type)
7433 8931 : && bitint_precision_kind (type) >= bitint_prec_large)
7434 : {
7435 8250 : if (bitmap_bit_p (names, SSA_NAME_VERSION (var)))
7436 5240 : use (live, var);
7437 : else
7438 3010 : worklist.safe_push (var);
7439 : }
7440 : }
7441 : }
7442 :
7443 81463 : if (muldiv_p)
7444 1706 : def (live, lhs, graph);
7445 : }
7446 :
7447 : /* If STMT is .{ADD,SUB,MUL}_OVERFLOW with INTEGER_CST arguments,
7448 : return the largest bitint_prec_kind of them, otherwise return
7449 : bitint_prec_small. */
7450 :
7451 : static bitint_prec_kind
7452 210531 : arith_overflow_arg_kind (gimple *stmt)
7453 : {
7454 210531 : bitint_prec_kind ret = bitint_prec_small;
7455 210531 : if (is_gimple_call (stmt) && gimple_call_internal_p (stmt))
7456 89168 : switch (gimple_call_internal_fn (stmt))
7457 : {
7458 : case IFN_ADD_OVERFLOW:
7459 : case IFN_SUB_OVERFLOW:
7460 : case IFN_MUL_OVERFLOW:
7461 225999 : for (int i = 0; i < 2; ++i)
7462 : {
7463 150666 : tree a = gimple_call_arg (stmt, i);
7464 150666 : if (TREE_CODE (a) == INTEGER_CST
7465 150666 : && BITINT_TYPE_P (TREE_TYPE (a)))
7466 : {
7467 5951 : bitint_prec_kind kind = bitint_precision_kind (TREE_TYPE (a));
7468 150666 : ret = MAX (ret, kind);
7469 : }
7470 : }
7471 : break;
7472 : default:
7473 : break;
7474 : }
7475 210531 : return ret;
7476 : }
7477 :
7478 : /* Entry point for _BitInt(N) operation lowering during optimization. */
7479 :
7480 : static unsigned int
7481 1518056 : gimple_lower_bitint (void)
7482 : {
7483 1518056 : small_max_prec = mid_min_prec = large_min_prec = huge_min_prec = 0;
7484 1518056 : limb_prec = abi_limb_prec = 0;
7485 1518056 : bitint_big_endian = false;
7486 1518056 : bool loop_init_done = false;
7487 :
7488 1518056 : unsigned int i;
7489 65169656 : for (i = 0; i < num_ssa_names; ++i)
7490 : {
7491 63659086 : tree s = ssa_name (i);
7492 63659086 : if (s == NULL)
7493 13592634 : continue;
7494 50066452 : tree type = TREE_TYPE (s);
7495 50066452 : if (TREE_CODE (type) == COMPLEX_TYPE)
7496 : {
7497 199821 : if (arith_overflow_arg_kind (SSA_NAME_DEF_STMT (s))
7498 : != bitint_prec_small)
7499 : break;
7500 199701 : type = TREE_TYPE (type);
7501 : }
7502 50048364 : if (BITINT_TYPE_P (type)
7503 50066339 : && bitint_precision_kind (type) != bitint_prec_small)
7504 : break;
7505 : /* We need to also rewrite stores of large/huge _BitInt INTEGER_CSTs
7506 : into memory. Such functions could have no large/huge SSA_NAMEs. */
7507 50059036 : if (SSA_NAME_IS_VIRTUAL_OPERAND (s))
7508 : {
7509 21945073 : gimple *g = SSA_NAME_DEF_STMT (s);
7510 21945073 : if (is_gimple_assign (g) && gimple_store_p (g))
7511 : {
7512 11163373 : tree t = gimple_assign_rhs1 (g);
7513 22323949 : if (BITINT_TYPE_P (TREE_TYPE (t))
7514 11163373 : && (bitint_precision_kind (TREE_TYPE (t))
7515 : >= bitint_prec_large))
7516 : break;
7517 : }
7518 : }
7519 : /* Similarly, e.g. with -frounding-math casts from _BitInt INTEGER_CSTs
7520 : to floating point types need to be rewritten. */
7521 28113963 : else if (SCALAR_FLOAT_TYPE_P (type))
7522 : {
7523 2337326 : gimple *g = SSA_NAME_DEF_STMT (s);
7524 2337326 : if (is_gimple_assign (g) && gimple_assign_rhs_code (g) == FLOAT_EXPR)
7525 : {
7526 129583 : tree t = gimple_assign_rhs1 (g);
7527 129583 : if (TREE_CODE (t) == INTEGER_CST
7528 110 : && BITINT_TYPE_P (TREE_TYPE (t))
7529 129584 : && (bitint_precision_kind (TREE_TYPE (t))
7530 : != bitint_prec_small))
7531 : break;
7532 : }
7533 : }
7534 : }
7535 3036112 : if (i == num_ssa_names)
7536 : return 0;
7537 :
7538 7486 : basic_block bb;
7539 7486 : auto_vec<gimple *, 4> switch_statements;
7540 47328 : FOR_EACH_BB_FN (bb, cfun)
7541 : {
7542 118728 : if (gswitch *swtch = safe_dyn_cast <gswitch *> (*gsi_last_bb (bb)))
7543 : {
7544 32 : tree idx = gimple_switch_index (swtch);
7545 40 : if (!BITINT_TYPE_P (TREE_TYPE (idx))
7546 32 : || bitint_precision_kind (TREE_TYPE (idx)) < bitint_prec_large)
7547 21 : continue;
7548 :
7549 11 : if (optimize)
7550 6 : group_case_labels_stmt (swtch);
7551 11 : if (gimple_switch_num_labels (swtch) == 1)
7552 : {
7553 0 : single_succ_edge (bb)->flags |= EDGE_FALLTHRU;
7554 0 : gimple_stmt_iterator gsi = gsi_for_stmt (swtch);
7555 0 : gsi_remove (&gsi, true);
7556 : }
7557 : else
7558 11 : switch_statements.safe_push (swtch);
7559 : }
7560 : }
7561 :
7562 7486 : if (!switch_statements.is_empty ())
7563 : {
7564 11 : bool expanded = false;
7565 11 : gimple *stmt;
7566 11 : unsigned int j;
7567 11 : i = 0;
7568 22 : FOR_EACH_VEC_ELT (switch_statements, j, stmt)
7569 : {
7570 11 : gswitch *swtch = as_a<gswitch *> (stmt);
7571 11 : tree_switch_conversion::switch_decision_tree dt (swtch);
7572 11 : expanded |= dt.analyze_switch_statement ();
7573 11 : }
7574 :
7575 11 : if (expanded)
7576 : {
7577 11 : free_dominance_info (CDI_DOMINATORS);
7578 11 : free_dominance_info (CDI_POST_DOMINATORS);
7579 11 : mark_virtual_operands_for_renaming (cfun);
7580 11 : cleanup_tree_cfg (TODO_update_ssa);
7581 : }
7582 : }
7583 :
7584 7486 : if (flag_sanitize & (SANITIZE_ADDRESS | SANITIZE_HWADDRESS))
7585 : {
7586 14 : hash_map<tree, tree> shadow_vars_mapping;
7587 14 : bool need_commit_edge_insert = false;
7588 14 : bool any_asan_poison = false;
7589 129 : for (unsigned j = 0; j < num_ssa_names; ++j)
7590 : {
7591 115 : tree s = ssa_name (j);
7592 115 : if (s == NULL)
7593 31 : continue;
7594 84 : tree type = TREE_TYPE (s);
7595 37 : if (BITINT_TYPE_P (type)
7596 47 : && gimple_call_internal_p (SSA_NAME_DEF_STMT (s),
7597 : IFN_ASAN_POISON)
7598 112 : && bitint_precision_kind (type) >= bitint_prec_large)
7599 : {
7600 28 : any_asan_poison = true;
7601 28 : gimple_stmt_iterator gsi = gsi_for_stmt (SSA_NAME_DEF_STMT (s));
7602 28 : asan_expand_poison_ifn (&gsi, &need_commit_edge_insert,
7603 : shadow_vars_mapping);
7604 : }
7605 : }
7606 14 : if (any_asan_poison)
7607 : {
7608 14 : if (need_commit_edge_insert)
7609 0 : gsi_commit_edge_inserts ();
7610 14 : i = 0;
7611 14 : free_dominance_info (CDI_DOMINATORS);
7612 14 : free_dominance_info (CDI_POST_DOMINATORS);
7613 14 : mark_virtual_operands_for_renaming (cfun);
7614 14 : cleanup_tree_cfg (TODO_update_ssa);
7615 : }
7616 14 : }
7617 :
7618 7486 : struct bitint_large_huge large_huge;
7619 7486 : bool has_large_huge_parm_result = false;
7620 7486 : bool has_large_huge = false;
7621 7486 : unsigned int ret = 0, first_large_huge = ~0U;
7622 7486 : bool edge_insertions = false;
7623 129462 : for (; i < num_ssa_names; ++i)
7624 : {
7625 121976 : tree s = ssa_name (i);
7626 121976 : if (s == NULL)
7627 3140 : continue;
7628 118836 : tree type = TREE_TYPE (s);
7629 118836 : if (TREE_CODE (type) == COMPLEX_TYPE)
7630 : {
7631 5375 : if (arith_overflow_arg_kind (SSA_NAME_DEF_STMT (s))
7632 : >= bitint_prec_large)
7633 1958 : has_large_huge = true;
7634 5375 : type = TREE_TYPE (type);
7635 : }
7636 72015 : if (BITINT_TYPE_P (type)
7637 118862 : && bitint_precision_kind (type) >= bitint_prec_large)
7638 : {
7639 36648 : if (first_large_huge == ~0U)
7640 5911 : first_large_huge = i;
7641 36648 : gimple *stmt = SSA_NAME_DEF_STMT (s), *g;
7642 36648 : gimple_stmt_iterator gsi;
7643 36648 : tree_code rhs_code;
7644 : /* Unoptimize certain constructs to simpler alternatives to
7645 : avoid having to lower all of them. */
7646 36648 : if (is_gimple_assign (stmt) && gimple_bb (stmt))
7647 22419 : switch (rhs_code = gimple_assign_rhs_code (stmt))
7648 : {
7649 : default:
7650 : break;
7651 373 : case MULT_EXPR:
7652 373 : case TRUNC_DIV_EXPR:
7653 373 : case EXACT_DIV_EXPR:
7654 373 : case TRUNC_MOD_EXPR:
7655 373 : if (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (s))
7656 : {
7657 2 : location_t loc = gimple_location (stmt);
7658 2 : gsi = gsi_for_stmt (stmt);
7659 2 : tree rhs1 = gimple_assign_rhs1 (stmt);
7660 2 : tree rhs2 = gimple_assign_rhs2 (stmt);
7661 : /* For multiplication and division with (ab)
7662 : lhs and one or both operands force the operands
7663 : into new SSA_NAMEs to avoid coalescing failures. */
7664 2 : if (TREE_CODE (rhs1) == SSA_NAME
7665 2 : && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (rhs1))
7666 : {
7667 2 : first_large_huge = 0;
7668 2 : tree t = make_ssa_name (TREE_TYPE (rhs1));
7669 2 : g = gimple_build_assign (t, SSA_NAME, rhs1);
7670 2 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7671 2 : gimple_set_location (g, loc);
7672 2 : gimple_assign_set_rhs1 (stmt, t);
7673 2 : if (rhs1 == rhs2)
7674 : {
7675 0 : gimple_assign_set_rhs2 (stmt, t);
7676 0 : rhs2 = t;
7677 : }
7678 2 : update_stmt (stmt);
7679 : }
7680 2 : if (TREE_CODE (rhs2) == SSA_NAME
7681 2 : && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (rhs2))
7682 : {
7683 0 : first_large_huge = 0;
7684 0 : tree t = make_ssa_name (TREE_TYPE (rhs2));
7685 0 : g = gimple_build_assign (t, SSA_NAME, rhs2);
7686 0 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7687 0 : gimple_set_location (g, loc);
7688 0 : gimple_assign_set_rhs2 (stmt, t);
7689 0 : update_stmt (stmt);
7690 : }
7691 : }
7692 : break;
7693 3 : case LROTATE_EXPR:
7694 3 : case RROTATE_EXPR:
7695 3 : {
7696 3 : first_large_huge = 0;
7697 3 : location_t loc = gimple_location (stmt);
7698 3 : gsi = gsi_for_stmt (stmt);
7699 3 : tree rhs1 = gimple_assign_rhs1 (stmt);
7700 3 : tree type = TREE_TYPE (rhs1);
7701 3 : tree n = gimple_assign_rhs2 (stmt), m;
7702 3 : tree p = build_int_cst (TREE_TYPE (n),
7703 3 : TYPE_PRECISION (type));
7704 3 : if (TREE_CODE (n) == INTEGER_CST)
7705 : {
7706 0 : if (integer_zerop (n))
7707 : m = n;
7708 : else
7709 0 : m = fold_build2 (MINUS_EXPR, TREE_TYPE (n), p, n);
7710 : }
7711 : else
7712 : {
7713 3 : tree tem = make_ssa_name (TREE_TYPE (n));
7714 3 : g = gimple_build_assign (tem, MINUS_EXPR, p, n);
7715 3 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7716 3 : gimple_set_location (g, loc);
7717 3 : m = make_ssa_name (TREE_TYPE (n));
7718 3 : g = gimple_build_assign (m, TRUNC_MOD_EXPR, tem, p);
7719 3 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7720 3 : gimple_set_location (g, loc);
7721 : }
7722 3 : if (!TYPE_UNSIGNED (type))
7723 : {
7724 0 : tree utype = build_bitint_type (TYPE_PRECISION (type),
7725 : 1);
7726 0 : if (TREE_CODE (rhs1) == INTEGER_CST)
7727 0 : rhs1 = fold_convert (utype, rhs1);
7728 : else
7729 : {
7730 0 : tree t = make_ssa_name (utype);
7731 0 : g = gimple_build_assign (t, NOP_EXPR, rhs1);
7732 0 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7733 0 : gimple_set_location (g, loc);
7734 0 : rhs1 = t;
7735 : }
7736 : }
7737 4 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (rhs1)),
7738 : rhs_code == LROTATE_EXPR
7739 : ? LSHIFT_EXPR : RSHIFT_EXPR,
7740 : rhs1, n);
7741 3 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7742 3 : gimple_set_location (g, loc);
7743 3 : tree op1 = gimple_assign_lhs (g);
7744 4 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (rhs1)),
7745 : rhs_code == LROTATE_EXPR
7746 : ? RSHIFT_EXPR : LSHIFT_EXPR,
7747 : rhs1, m);
7748 3 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7749 3 : gimple_set_location (g, loc);
7750 3 : tree op2 = gimple_assign_lhs (g);
7751 3 : tree lhs = gimple_assign_lhs (stmt);
7752 3 : if (!TYPE_UNSIGNED (type))
7753 : {
7754 0 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (op1)),
7755 : BIT_IOR_EXPR, op1, op2);
7756 0 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7757 0 : gimple_set_location (g, loc);
7758 0 : g = gimple_build_assign (lhs, NOP_EXPR,
7759 : gimple_assign_lhs (g));
7760 : }
7761 : else
7762 3 : g = gimple_build_assign (lhs, BIT_IOR_EXPR, op1, op2);
7763 3 : gsi_replace (&gsi, g, true);
7764 3 : gimple_set_location (g, loc);
7765 : }
7766 3 : break;
7767 17 : case ABS_EXPR:
7768 17 : case ABSU_EXPR:
7769 17 : case MIN_EXPR:
7770 17 : case MAX_EXPR:
7771 17 : case COND_EXPR:
7772 17 : first_large_huge = 0;
7773 17 : gsi = gsi_for_stmt (stmt);
7774 17 : tree lhs = gimple_assign_lhs (stmt);
7775 17 : tree rhs1 = gimple_assign_rhs1 (stmt), rhs2 = NULL_TREE;
7776 17 : location_t loc = gimple_location (stmt);
7777 17 : if (rhs_code == ABS_EXPR)
7778 4 : g = gimple_build_cond (LT_EXPR, rhs1,
7779 4 : build_zero_cst (TREE_TYPE (rhs1)),
7780 : NULL_TREE, NULL_TREE);
7781 13 : else if (rhs_code == ABSU_EXPR)
7782 : {
7783 4 : rhs2 = make_ssa_name (TREE_TYPE (lhs));
7784 4 : g = gimple_build_assign (rhs2, NOP_EXPR, rhs1);
7785 4 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7786 4 : gimple_set_location (g, loc);
7787 4 : g = gimple_build_cond (LT_EXPR, rhs1,
7788 4 : build_zero_cst (TREE_TYPE (rhs1)),
7789 : NULL_TREE, NULL_TREE);
7790 4 : rhs1 = rhs2;
7791 : }
7792 9 : else if (rhs_code == MIN_EXPR || rhs_code == MAX_EXPR)
7793 : {
7794 9 : rhs2 = gimple_assign_rhs2 (stmt);
7795 9 : if (TREE_CODE (rhs1) == INTEGER_CST)
7796 0 : std::swap (rhs1, rhs2);
7797 9 : g = gimple_build_cond (LT_EXPR, rhs1, rhs2,
7798 : NULL_TREE, NULL_TREE);
7799 9 : if (rhs_code == MAX_EXPR)
7800 5 : std::swap (rhs1, rhs2);
7801 : }
7802 : else
7803 : {
7804 0 : g = gimple_build_cond (NE_EXPR, rhs1,
7805 0 : build_zero_cst (TREE_TYPE (rhs1)),
7806 : NULL_TREE, NULL_TREE);
7807 0 : rhs1 = gimple_assign_rhs2 (stmt);
7808 0 : rhs2 = gimple_assign_rhs3 (stmt);
7809 : }
7810 17 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7811 17 : gimple_set_location (g, loc);
7812 17 : edge e1 = split_block (gsi_bb (gsi), g);
7813 17 : edge e2 = split_block (e1->dest, (gimple *) NULL);
7814 17 : edge e3 = make_edge (e1->src, e2->dest, EDGE_FALSE_VALUE);
7815 17 : e3->probability = profile_probability::even ();
7816 17 : e1->flags = EDGE_TRUE_VALUE;
7817 17 : e1->probability = e3->probability.invert ();
7818 17 : if (dom_info_available_p (CDI_DOMINATORS))
7819 9 : set_immediate_dominator (CDI_DOMINATORS, e2->dest, e1->src);
7820 17 : if (rhs_code == ABS_EXPR || rhs_code == ABSU_EXPR)
7821 : {
7822 8 : gsi = gsi_after_labels (e1->dest);
7823 8 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (rhs1)),
7824 : NEGATE_EXPR, rhs1);
7825 8 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7826 8 : gimple_set_location (g, loc);
7827 8 : rhs2 = gimple_assign_lhs (g);
7828 8 : std::swap (rhs1, rhs2);
7829 : }
7830 17 : gsi = gsi_for_stmt (stmt);
7831 17 : gsi_remove (&gsi, true);
7832 17 : gphi *phi = create_phi_node (lhs, e2->dest);
7833 17 : add_phi_arg (phi, rhs1, e2, UNKNOWN_LOCATION);
7834 17 : add_phi_arg (phi, rhs2, e3, UNKNOWN_LOCATION);
7835 17 : break;
7836 : }
7837 14229 : else if (gimple_call_internal_p (stmt, IFN_CLMUL_WIDEN))
7838 : {
7839 : /* For now, lower this to two IFN_CLMUL and three
7840 : IFN_BITREVERSE calls. Could be implemented in
7841 : lower_clmul too, just needs special care on padding
7842 : bits and split the shifting into up to two disjunct
7843 : results, or treat result as 2*prec array and adjust
7844 : afterwards. */
7845 10 : first_large_huge = 0;
7846 10 : location_t loc = gimple_location (stmt);
7847 10 : tree arg0 = gimple_call_arg (stmt, 0);
7848 10 : tree arg1 = gimple_call_arg (stmt, 1);
7849 10 : tree type = TREE_TYPE (arg0);
7850 10 : gsi = gsi_for_stmt (stmt);
7851 10 : gimple *g = gimple_build_call_internal (IFN_CLMUL, 2,
7852 : arg0, arg1);
7853 10 : tree real = make_ssa_name (type);
7854 10 : gimple_call_set_lhs (g, real);
7855 10 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7856 10 : gimple_set_location (g, loc);
7857 10 : g = gimple_build_call_internal (IFN_BITREVERSE, 1, arg0);
7858 10 : tree rarg0 = make_ssa_name (type);
7859 10 : gimple_call_set_lhs (g, rarg0);
7860 10 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7861 10 : gimple_set_location (g, loc);
7862 10 : g = gimple_build_call_internal (IFN_BITREVERSE, 1, arg1);
7863 10 : tree rarg1 = make_ssa_name (type);
7864 10 : gimple_call_set_lhs (g, rarg1);
7865 10 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7866 10 : gimple_set_location (g, loc);
7867 10 : g = gimple_build_call_internal (IFN_CLMUL, 2, rarg0, rarg1);
7868 10 : tree rimag = make_ssa_name (type);
7869 10 : gimple_call_set_lhs (g, rimag);
7870 10 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7871 10 : gimple_set_location (g, loc);
7872 10 : g = gimple_build_call_internal (IFN_BITREVERSE, 1, rimag);
7873 10 : tree imag = make_ssa_name (type);
7874 10 : gimple_call_set_lhs (g, imag);
7875 10 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7876 10 : gimple_set_location (g, loc);
7877 10 : g = gimple_build_assign (make_ssa_name (type), RSHIFT_EXPR, imag,
7878 : build_one_cst (unsigned_type_node));
7879 10 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7880 10 : gimple_set_location (g, loc);
7881 10 : g = gimple_build_assign (gimple_call_lhs (stmt), COMPLEX_EXPR,
7882 : real, gimple_assign_lhs (g));
7883 10 : gimple_set_location (g, loc);
7884 10 : gsi_replace (&gsi, g, true);
7885 : }
7886 14219 : else if (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (s)
7887 76 : && is_gimple_call (stmt)
7888 14239 : && gimple_call_internal_p (stmt))
7889 20 : switch (gimple_call_internal_fn (stmt))
7890 : {
7891 12 : case IFN_UBSAN_CHECK_ADD:
7892 12 : case IFN_UBSAN_CHECK_SUB:
7893 12 : if (!bitint_big_endian)
7894 : break;
7895 : /* FALLTHRU */
7896 : case IFN_UBSAN_CHECK_MUL:
7897 : case IFN_BSWAP:
7898 : case IFN_BITREVERSE:
7899 : case IFN_CLMUL:
7900 22 : for (unsigned i = 0; i < gimple_call_num_args (stmt); ++i)
7901 : {
7902 14 : location_t loc = gimple_location (stmt);
7903 14 : gsi = gsi_for_stmt (stmt);
7904 : /* Similar case to multiplication/division with (ab)
7905 : above for internal functions which set muldiv_p. */
7906 14 : tree arg = gimple_call_arg (stmt, i);
7907 14 : if (TREE_CODE (arg) == SSA_NAME
7908 14 : && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (arg))
7909 : {
7910 8 : first_large_huge = 0;
7911 8 : tree t = make_ssa_name (TREE_TYPE (arg));
7912 8 : g = gimple_build_assign (t, SSA_NAME, arg);
7913 8 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7914 8 : gimple_set_location (g, loc);
7915 8 : gimple_call_set_arg (stmt, i, t);
7916 8 : if (i == 0
7917 8 : && gimple_call_num_args (stmt) >= 2
7918 14 : && gimple_call_arg (stmt, 1) == arg)
7919 0 : gimple_call_set_arg (stmt, 1, t);
7920 8 : update_stmt (stmt);
7921 : }
7922 : }
7923 : break;
7924 : default:
7925 : break;
7926 : }
7927 36903 : else if (gphi *phi = dyn_cast <gphi *> (stmt))
7928 : {
7929 : /* Remove PHIs with no uses, coalescing apparently
7930 : relies on this. */
7931 255 : if (has_zero_uses (s))
7932 : {
7933 8 : remove_gimple_phi_args (phi);
7934 8 : gsi = gsi_for_stmt (stmt);
7935 8 : remove_phi_node (&gsi, true);
7936 : }
7937 : }
7938 : }
7939 : /* We need to also rewrite stores of large/huge _BitInt INTEGER_CSTs
7940 : into memory. Such functions could have no large/huge SSA_NAMEs. */
7941 82188 : else if (SSA_NAME_IS_VIRTUAL_OPERAND (s))
7942 : {
7943 52623 : gimple *g = SSA_NAME_DEF_STMT (s);
7944 52623 : if (is_gimple_assign (g) && gimple_store_p (g))
7945 : {
7946 17176 : tree t = gimple_assign_rhs1 (g);
7947 19503 : if (BITINT_TYPE_P (TREE_TYPE (t))
7948 17176 : && (bitint_precision_kind (TREE_TYPE (t))
7949 : >= bitint_prec_large))
7950 : has_large_huge = true;
7951 : }
7952 : }
7953 : /* Similarly, e.g. with -frounding-math casts from _BitInt INTEGER_CSTs
7954 : to floating point types need to be rewritten. */
7955 29565 : else if (SCALAR_FLOAT_TYPE_P (type))
7956 : {
7957 742 : gimple *g = SSA_NAME_DEF_STMT (s);
7958 742 : if (is_gimple_assign (g) && gimple_assign_rhs_code (g) == FLOAT_EXPR)
7959 : {
7960 187 : tree t = gimple_assign_rhs1 (g);
7961 187 : if (TREE_CODE (t) == INTEGER_CST
7962 1 : && BITINT_TYPE_P (TREE_TYPE (t))
7963 188 : && (bitint_precision_kind (TREE_TYPE (t))
7964 : >= bitint_prec_large))
7965 : has_large_huge = true;
7966 : }
7967 : }
7968 : }
7969 105674 : for (i = first_large_huge; i < num_ssa_names; ++i)
7970 : {
7971 98188 : tree s = ssa_name (i);
7972 98188 : if (s == NULL)
7973 2808 : continue;
7974 95380 : tree type = TREE_TYPE (s);
7975 95380 : if (TREE_CODE (type) == COMPLEX_TYPE)
7976 4134 : type = TREE_TYPE (type);
7977 58064 : if (BITINT_TYPE_P (type)
7978 95404 : && bitint_precision_kind (type) >= bitint_prec_large)
7979 : {
7980 36640 : use_operand_p use_p;
7981 36640 : gimple *use_stmt;
7982 36640 : has_large_huge = true;
7983 38304 : if (optimize
7984 54098 : && optimizable_arith_overflow (SSA_NAME_DEF_STMT (s)))
7985 6635 : continue;
7986 : /* Ignore large/huge _BitInt SSA_NAMEs which have single use in
7987 : the same bb and could be handled in the same loop with the
7988 : immediate use. */
7989 34976 : if (optimize
7990 15794 : && !SSA_NAME_OCCURS_IN_ABNORMAL_PHI (s)
7991 15727 : && single_imm_use (s, &use_p, &use_stmt)
7992 50036 : && gimple_bb (SSA_NAME_DEF_STMT (s)) == gimple_bb (use_stmt))
7993 : {
7994 10587 : if (mergeable_op (SSA_NAME_DEF_STMT (s)))
7995 : {
7996 2238 : if (mergeable_op (use_stmt))
7997 1927 : continue;
7998 311 : tree_code cmp_code = comparison_op (use_stmt, NULL, NULL);
7999 311 : if (cmp_code == EQ_EXPR || cmp_code == NE_EXPR)
8000 26 : continue;
8001 285 : if (gimple_assign_cast_p (use_stmt))
8002 : {
8003 130 : tree lhs = gimple_assign_lhs (use_stmt);
8004 260 : if (INTEGRAL_TYPE_P (TREE_TYPE (lhs))
8005 : /* Don't merge with VIEW_CONVERT_EXPRs to
8006 : huge INTEGER_TYPEs used sometimes in memcpy
8007 : expansion. */
8008 248 : && (TREE_CODE (TREE_TYPE (lhs)) != INTEGER_TYPE
8009 6 : || (TYPE_PRECISION (TREE_TYPE (lhs))
8010 12 : <= MAX_FIXED_MODE_SIZE)))
8011 118 : continue;
8012 : }
8013 155 : else if (gimple_store_p (use_stmt)
8014 0 : && is_gimple_assign (use_stmt)
8015 0 : && !gimple_has_volatile_ops (use_stmt)
8016 155 : && !stmt_ends_bb_p (use_stmt))
8017 0 : continue;
8018 : }
8019 8516 : if (gimple_assign_cast_p (SSA_NAME_DEF_STMT (s)))
8020 : {
8021 874 : tree rhs1 = gimple_assign_rhs1 (SSA_NAME_DEF_STMT (s));
8022 874 : if (TREE_CODE (rhs1) == VIEW_CONVERT_EXPR)
8023 : {
8024 20 : rhs1 = TREE_OPERAND (rhs1, 0);
8025 20 : if (!INTEGRAL_TYPE_P (TREE_TYPE (rhs1))
8026 16 : && !POINTER_TYPE_P (TREE_TYPE (rhs1))
8027 16 : && gimple_store_p (use_stmt))
8028 7 : continue;
8029 : }
8030 1734 : if (INTEGRAL_TYPE_P (TREE_TYPE (rhs1))
8031 776 : && ((is_gimple_assign (use_stmt)
8032 729 : && (gimple_assign_rhs_code (use_stmt)
8033 : != COMPLEX_EXPR))
8034 47 : || gimple_code (use_stmt) == GIMPLE_COND)
8035 766 : && (!gimple_store_p (use_stmt)
8036 132 : || (is_gimple_assign (use_stmt)
8037 132 : && !gimple_has_volatile_ops (use_stmt)
8038 132 : && !stmt_ends_bb_p (use_stmt)))
8039 1633 : && (TREE_CODE (rhs1) != SSA_NAME
8040 766 : || !SSA_NAME_OCCURS_IN_ABNORMAL_PHI (rhs1)))
8041 : {
8042 766 : if (is_gimple_assign (use_stmt))
8043 729 : switch (gimple_assign_rhs_code (use_stmt))
8044 : {
8045 54 : case TRUNC_DIV_EXPR:
8046 54 : case EXACT_DIV_EXPR:
8047 54 : case TRUNC_MOD_EXPR:
8048 54 : case FLOAT_EXPR:
8049 : /* For division, modulo and casts to floating
8050 : point, avoid representing unsigned operands
8051 : using negative prec if they were sign-extended
8052 : from narrower precision. */
8053 54 : if (TYPE_UNSIGNED (TREE_TYPE (s))
8054 28 : && !TYPE_UNSIGNED (TREE_TYPE (rhs1))
8055 63 : && (TYPE_PRECISION (TREE_TYPE (s))
8056 9 : > TYPE_PRECISION (TREE_TYPE (rhs1))))
8057 8 : goto force_name;
8058 : /* FALLTHRU */
8059 115 : case MULT_EXPR:
8060 130 : if (!BITINT_TYPE_P (TREE_TYPE (rhs1))
8061 115 : || (bitint_precision_kind (TREE_TYPE (rhs1))
8062 : < bitint_prec_large))
8063 50 : continue;
8064 : /* Uses which use handle_operand_addr can't
8065 : deal with nested casts. */
8066 65 : if (TREE_CODE (rhs1) == SSA_NAME
8067 65 : && gimple_assign_cast_p
8068 65 : (SSA_NAME_DEF_STMT (rhs1))
8069 43 : && has_single_use (rhs1)
8070 108 : && (gimple_bb (SSA_NAME_DEF_STMT (rhs1))
8071 43 : == gimple_bb (SSA_NAME_DEF_STMT (s))))
8072 43 : goto force_name;
8073 : break;
8074 0 : case VIEW_CONVERT_EXPR:
8075 0 : {
8076 0 : tree lhs = gimple_assign_lhs (use_stmt);
8077 : /* Don't merge with VIEW_CONVERT_EXPRs to
8078 : non-integral types. */
8079 0 : if (!INTEGRAL_TYPE_P (TREE_TYPE (lhs)))
8080 0 : goto force_name;
8081 : /* Don't merge with VIEW_CONVERT_EXPRs to
8082 : huge INTEGER_TYPEs used sometimes in memcpy
8083 : expansion. */
8084 0 : if (TREE_CODE (TREE_TYPE (lhs)) == INTEGER_TYPE
8085 0 : && (TYPE_PRECISION (TREE_TYPE (lhs))
8086 0 : > MAX_FIXED_MODE_SIZE))
8087 0 : goto force_name;
8088 : }
8089 : break;
8090 : default:
8091 : break;
8092 : }
8093 850 : if (!BITINT_TYPE_P (TREE_TYPE (rhs1))
8094 665 : || (bitint_precision_kind (TREE_TYPE (rhs1))
8095 : < bitint_prec_large))
8096 247 : continue;
8097 418 : if ((TYPE_PRECISION (TREE_TYPE (rhs1))
8098 418 : >= TYPE_PRECISION (TREE_TYPE (s)))
8099 418 : && mergeable_op (use_stmt))
8100 60 : continue;
8101 : /* Prevent merging a widening non-mergeable cast
8102 : on result of some narrower mergeable op
8103 : together with later mergeable operations. E.g.
8104 : result of _BitInt(223) addition shouldn't be
8105 : sign-extended to _BitInt(513) and have another
8106 : _BitInt(513) added to it, as handle_plus_minus
8107 : with its PHI node handling inside of handle_cast
8108 : will not work correctly. An exception is if
8109 : use_stmt is a store, this is handled directly
8110 : in lower_mergeable_stmt. */
8111 709 : if (TREE_CODE (rhs1) != SSA_NAME
8112 358 : || !has_single_use (rhs1)
8113 272 : || (gimple_bb (SSA_NAME_DEF_STMT (rhs1))
8114 272 : != gimple_bb (SSA_NAME_DEF_STMT (s)))
8115 216 : || !mergeable_op (SSA_NAME_DEF_STMT (rhs1))
8116 445 : || gimple_store_p (use_stmt))
8117 351 : continue;
8118 7 : if ((TYPE_PRECISION (TREE_TYPE (rhs1))
8119 7 : < TYPE_PRECISION (TREE_TYPE (s)))
8120 9 : && gimple_assign_cast_p (SSA_NAME_DEF_STMT (rhs1)))
8121 : {
8122 : /* Another exception is if the widening cast is
8123 : from mergeable same precision cast from something
8124 : not mergeable. */
8125 0 : tree rhs2
8126 0 : = gimple_assign_rhs1 (SSA_NAME_DEF_STMT (rhs1));
8127 0 : if (BITINT_TYPE_P (TREE_TYPE (rhs2))
8128 0 : && (TYPE_PRECISION (TREE_TYPE (rhs1))
8129 0 : == TYPE_PRECISION (TREE_TYPE (rhs2))))
8130 : {
8131 0 : if (TREE_CODE (rhs2) != SSA_NAME
8132 0 : || !has_single_use (rhs2)
8133 0 : || (gimple_bb (SSA_NAME_DEF_STMT (rhs2))
8134 0 : != gimple_bb (SSA_NAME_DEF_STMT (s)))
8135 0 : || !mergeable_op (SSA_NAME_DEF_STMT (rhs2)))
8136 0 : continue;
8137 : }
8138 : }
8139 : }
8140 : }
8141 7750 : if (is_gimple_assign (SSA_NAME_DEF_STMT (s)))
8142 5464 : switch (gimple_assign_rhs_code (SSA_NAME_DEF_STMT (s)))
8143 : {
8144 1814 : case REALPART_EXPR:
8145 1814 : case IMAGPART_EXPR:
8146 1814 : {
8147 1814 : gimple *ds = SSA_NAME_DEF_STMT (s);
8148 1814 : tree rhs1 = gimple_assign_rhs1 (ds);
8149 1814 : rhs1 = TREE_OPERAND (rhs1, 0);
8150 1814 : if (TREE_CODE (rhs1) == SSA_NAME)
8151 : {
8152 1814 : gimple *g = SSA_NAME_DEF_STMT (rhs1);
8153 1814 : if (optimizable_arith_overflow (g))
8154 : {
8155 1650 : if (gimple_assign_rhs_code (ds) == IMAGPART_EXPR)
8156 1640 : continue;
8157 10 : if (gimple_store_p (use_stmt))
8158 : {
8159 : /* Punt if the cast use of IMAGPART_EXPR stmt
8160 : appears before the store use_stmt, because
8161 : optimizable arith overflow can't be
8162 : lowered at the store location in that case.
8163 : See PR121828. */
8164 10 : gimple_stmt_iterator gsi
8165 10 : = gsi_for_stmt (use_stmt);
8166 10 : unsigned int cnt = 0;
8167 12 : do
8168 : {
8169 12 : gsi_prev_nondebug (&gsi);
8170 12 : if (gsi_end_p (gsi))
8171 : break;
8172 12 : gimple *g2 = gsi_stmt (gsi);
8173 12 : if (g2 == ds)
8174 : break;
8175 3 : if (++cnt == 64)
8176 : break;
8177 3 : if (!gimple_assign_cast_p (g2))
8178 2 : continue;
8179 1 : tree rhs2 = gimple_assign_rhs1 (g2);
8180 1 : if (TREE_CODE (rhs2) != SSA_NAME)
8181 0 : continue;
8182 1 : gimple *g3 = SSA_NAME_DEF_STMT (rhs2);
8183 1 : if (!is_gimple_assign (g3))
8184 0 : continue;
8185 1 : if (gimple_assign_rhs_code (g3)
8186 : != IMAGPART_EXPR)
8187 0 : continue;
8188 1 : rhs2 = gimple_assign_rhs1 (g3);
8189 1 : rhs2 = TREE_OPERAND (rhs2, 0);
8190 1 : if (rhs2 != rhs1)
8191 0 : continue;
8192 : cnt = 64;
8193 : break;
8194 : }
8195 : while (1);
8196 10 : if (cnt == 64)
8197 : break;
8198 : }
8199 : }
8200 : }
8201 : }
8202 : /* FALLTHRU */
8203 876 : case LSHIFT_EXPR:
8204 876 : case RSHIFT_EXPR:
8205 876 : case MULT_EXPR:
8206 876 : case TRUNC_DIV_EXPR:
8207 876 : case EXACT_DIV_EXPR:
8208 876 : case TRUNC_MOD_EXPR:
8209 876 : case FIX_TRUNC_EXPR:
8210 876 : if (gimple_store_p (use_stmt)
8211 466 : && is_gimple_assign (use_stmt)
8212 466 : && !gimple_has_volatile_ops (use_stmt)
8213 1342 : && !stmt_ends_bb_p (use_stmt))
8214 : {
8215 466 : tree lhs = gimple_assign_lhs (use_stmt);
8216 : /* As multiply/division passes address of the lhs
8217 : to library function and that assumes it can extend
8218 : it to whole number of limbs, avoid merging those
8219 : with bit-field stores. Don't allow it for
8220 : shifts etc. either, so that the bit-field store
8221 : handling doesn't have to be done everywhere. */
8222 466 : if (TREE_CODE (lhs) == COMPONENT_REF
8223 466 : && DECL_BIT_FIELD_TYPE (TREE_OPERAND (lhs, 1)))
8224 : break;
8225 463 : continue;
8226 463 : }
8227 : break;
8228 : default:
8229 : break;
8230 : }
8231 2286 : else if (is_gimple_call (SSA_NAME_DEF_STMT (s))
8232 2286 : && gimple_call_internal_p (SSA_NAME_DEF_STMT (s)))
8233 75 : switch (gimple_call_internal_fn (SSA_NAME_DEF_STMT (s)))
8234 : {
8235 73 : case IFN_BSWAP:
8236 73 : case IFN_BITREVERSE:
8237 73 : case IFN_CLMUL:
8238 73 : if (gimple_store_p (use_stmt)
8239 23 : && is_gimple_assign (use_stmt)
8240 23 : && !gimple_has_volatile_ops (use_stmt)
8241 96 : && !stmt_ends_bb_p (use_stmt))
8242 : {
8243 23 : tree lhs = gimple_assign_lhs (use_stmt);
8244 23 : if (TREE_CODE (lhs) == COMPONENT_REF
8245 23 : && DECL_BIT_FIELD_TYPE (TREE_OPERAND (lhs, 1)))
8246 : break;
8247 23 : continue;
8248 23 : }
8249 : break;
8250 : default:
8251 : break;
8252 : }
8253 : }
8254 :
8255 : /* Also ignore uninitialized uses. */
8256 30013 : if (SSA_NAME_IS_DEFAULT_DEF (s)
8257 30013 : && (!SSA_NAME_VAR (s) || VAR_P (SSA_NAME_VAR (s))))
8258 59 : continue;
8259 :
8260 30005 : force_name:
8261 30005 : if (!large_huge.m_names)
8262 5788 : large_huge.m_names = BITMAP_ALLOC (NULL);
8263 30005 : bitmap_set_bit (large_huge.m_names, SSA_NAME_VERSION (s));
8264 30005 : if (has_single_use (s))
8265 : {
8266 : tree s2 = s, s3 = s;
8267 : /* The coalescing hook special cases SSA_NAME copies.
8268 : Make sure not to mark in m_single_use_names single
8269 : use SSA_NAMEs copied from non-single use SSA_NAMEs. */
8270 26243 : while (gimple_assign_copy_p (SSA_NAME_DEF_STMT (s2)))
8271 : {
8272 948 : s2 = gimple_assign_rhs1 (SSA_NAME_DEF_STMT (s2));
8273 948 : if (TREE_CODE (s2) != SSA_NAME)
8274 : break;
8275 48 : if (!has_single_use (s2))
8276 : {
8277 : s2 = NULL_TREE;
8278 : break;
8279 : }
8280 : s3 = s2;
8281 : }
8282 26229 : if (s2)
8283 : {
8284 : /* Only set m_single_use_names for SSA_NAMEs where
8285 : the definition and use are within the same loop,
8286 : otherwise if the use could be executed more times
8287 : than the def stmt, we can't emit clobbers.
8288 : Or if irreducible loops are involved, only handle
8289 : same bb definitions. */
8290 26195 : gimple *def = SSA_NAME_DEF_STMT (s3), *use;
8291 26195 : use_operand_p use_p;
8292 26195 : single_imm_use (s, &use_p, &use);
8293 26195 : basic_block def_bb, use_bb;
8294 26195 : if (SSA_NAME_IS_DEFAULT_DEF (s3))
8295 : def_bb = NULL;
8296 : else
8297 20960 : def_bb = gimple_bb (def);
8298 26195 : use_bb = gimple_bb (use);
8299 26195 : if (def_bb != use_bb)
8300 : {
8301 7069 : if (!loop_init_done)
8302 : {
8303 4218 : unsigned flags
8304 : = (AVOID_CFG_MODIFICATIONS
8305 : | LOOPS_HAVE_MARKED_IRREDUCIBLE_REGIONS);
8306 4218 : loop_optimizer_init (flags);
8307 4218 : loop_init_done = true;
8308 : }
8309 7069 : if ((use_bb->flags & BB_IRREDUCIBLE_LOOP) == 0)
8310 : {
8311 7021 : class loop *def_loop
8312 7021 : = def_bb ? def_bb->loop_father
8313 7021 : : get_loop (cfun, 0);
8314 7021 : class loop *use_loop = use_bb->loop_father;
8315 7021 : if (def_loop == use_loop)
8316 26195 : def_bb = use_bb;
8317 : }
8318 : }
8319 26195 : if (def_bb == use_bb)
8320 : {
8321 26078 : if (!large_huge.m_single_use_names)
8322 5651 : large_huge.m_single_use_names = BITMAP_ALLOC (NULL);
8323 26078 : bitmap_set_bit (large_huge.m_single_use_names,
8324 26078 : SSA_NAME_VERSION (s));
8325 : }
8326 : }
8327 : }
8328 30005 : if (SSA_NAME_VAR (s)
8329 7585 : && ((TREE_CODE (SSA_NAME_VAR (s)) == PARM_DECL
8330 5515 : && SSA_NAME_IS_DEFAULT_DEF (s))
8331 2107 : || TREE_CODE (SSA_NAME_VAR (s)) == RESULT_DECL))
8332 : has_large_huge_parm_result = true;
8333 30005 : if (optimize
8334 10823 : && !SSA_NAME_OCCURS_IN_ABNORMAL_PHI (s)
8335 10756 : && gimple_assign_load_p (SSA_NAME_DEF_STMT (s))
8336 6090 : && !gimple_has_volatile_ops (SSA_NAME_DEF_STMT (s))
8337 33001 : && !stmt_ends_bb_p (SSA_NAME_DEF_STMT (s)))
8338 : {
8339 2996 : use_operand_p use_p;
8340 2996 : imm_use_iterator iter;
8341 2996 : bool optimizable_load = true;
8342 6057 : FOR_EACH_IMM_USE_FAST (use_p, iter, s)
8343 : {
8344 3159 : gimple *use_stmt = USE_STMT (use_p);
8345 3159 : if (is_gimple_debug (use_stmt))
8346 0 : continue;
8347 3159 : if (is_gimple_call (use_stmt)
8348 3159 : && gimple_call_internal_p (use_stmt))
8349 66 : switch (gimple_call_internal_fn (use_stmt))
8350 : {
8351 10 : case IFN_CLZ:
8352 10 : case IFN_CTZ:
8353 10 : case IFN_CLRSB:
8354 10 : case IFN_FFS:
8355 10 : case IFN_PARITY:
8356 10 : case IFN_POPCOUNT:
8357 10 : case IFN_BSWAP:
8358 10 : case IFN_BITREVERSE:
8359 10 : case IFN_CLMUL:
8360 10 : continue;
8361 : default:
8362 : break;
8363 : }
8364 3149 : if (gimple_code (use_stmt) == GIMPLE_PHI
8365 : || is_gimple_call (use_stmt)
8366 : || gimple_code (use_stmt) == GIMPLE_ASM
8367 : || (is_gimple_assign (use_stmt)
8368 1907 : && (gimple_assign_rhs_code (use_stmt)
8369 : == COMPLEX_EXPR)))
8370 : {
8371 : optimizable_load = false;
8372 : break;
8373 : }
8374 2996 : }
8375 :
8376 2996 : ssa_op_iter oi;
8377 4045 : FOR_EACH_SSA_USE_OPERAND (use_p, SSA_NAME_DEF_STMT (s),
8378 : oi, SSA_OP_USE)
8379 : {
8380 1049 : tree s2 = USE_FROM_PTR (use_p);
8381 1049 : if (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (s2))
8382 : {
8383 : optimizable_load = false;
8384 : break;
8385 : }
8386 : }
8387 :
8388 2996 : if (optimizable_load && !stmt_ends_bb_p (SSA_NAME_DEF_STMT (s)))
8389 : {
8390 2898 : if (!large_huge.m_loads)
8391 317 : large_huge.m_loads = BITMAP_ALLOC (NULL);
8392 2898 : bitmap_set_bit (large_huge.m_loads, SSA_NAME_VERSION (s));
8393 : }
8394 : }
8395 : }
8396 : /* We need to also rewrite stores of large/huge _BitInt INTEGER_CSTs
8397 : into memory. Such functions could have no large/huge SSA_NAMEs. */
8398 58740 : else if (SSA_NAME_IS_VIRTUAL_OPERAND (s))
8399 : {
8400 41687 : gimple *g = SSA_NAME_DEF_STMT (s);
8401 41687 : if (is_gimple_assign (g) && gimple_store_p (g))
8402 : {
8403 14443 : tree t = gimple_assign_rhs1 (g);
8404 16394 : if (BITINT_TYPE_P (TREE_TYPE (t))
8405 14443 : && bitint_precision_kind (TREE_TYPE (t)) >= bitint_prec_large)
8406 : has_large_huge = true;
8407 : }
8408 : }
8409 : }
8410 :
8411 7486 : if (large_huge.m_names || has_large_huge)
8412 : {
8413 5988 : ret = TODO_update_ssa_only_virtuals | TODO_cleanup_cfg;
8414 5988 : calculate_dominance_info (CDI_DOMINATORS);
8415 5988 : if (optimize)
8416 3192 : enable_ranger (cfun);
8417 5988 : if (large_huge.m_loads)
8418 : {
8419 317 : basic_block entry = ENTRY_BLOCK_PTR_FOR_FN (cfun);
8420 317 : entry->aux = NULL;
8421 634 : bitint_dom_walker (large_huge.m_names,
8422 317 : large_huge.m_loads).walk (entry);
8423 317 : bitmap_and_compl_into (large_huge.m_names, large_huge.m_loads);
8424 317 : clear_aux_for_blocks ();
8425 317 : BITMAP_FREE (large_huge.m_loads);
8426 : }
8427 5988 : large_huge.m_limb_type = build_nonstandard_integer_type (limb_prec, 1);
8428 5988 : large_huge.m_limb_size
8429 5988 : = tree_to_uhwi (TYPE_SIZE_UNIT (large_huge.m_limb_type));
8430 : }
8431 7486 : if (large_huge.m_names)
8432 : {
8433 5788 : large_huge.m_map
8434 11576 : = init_var_map (num_ssa_names, NULL, large_huge.m_names);
8435 5788 : coalesce_ssa_name (large_huge.m_map);
8436 5788 : partition_view_normal (large_huge.m_map);
8437 5788 : if (dump_file && (dump_flags & TDF_DETAILS))
8438 : {
8439 0 : fprintf (dump_file, "After Coalescing:\n");
8440 0 : dump_var_map (dump_file, large_huge.m_map);
8441 : }
8442 5788 : large_huge.m_vars
8443 5788 : = XCNEWVEC (tree, num_var_partitions (large_huge.m_map));
8444 5788 : bitmap_iterator bi;
8445 5788 : if (has_large_huge_parm_result)
8446 20084 : EXECUTE_IF_SET_IN_BITMAP (large_huge.m_names, 0, i, bi)
8447 : {
8448 15815 : tree s = ssa_name (i);
8449 15815 : if (SSA_NAME_VAR (s)
8450 6436 : && ((TREE_CODE (SSA_NAME_VAR (s)) == PARM_DECL
8451 5515 : && SSA_NAME_IS_DEFAULT_DEF (s))
8452 958 : || TREE_CODE (SSA_NAME_VAR (s)) == RESULT_DECL))
8453 : {
8454 5478 : int p = var_to_partition (large_huge.m_map, s);
8455 5478 : if (large_huge.m_vars[p] == NULL_TREE)
8456 : {
8457 5478 : large_huge.m_vars[p] = SSA_NAME_VAR (s);
8458 5478 : mark_addressable (SSA_NAME_VAR (s));
8459 : }
8460 : }
8461 : }
8462 5788 : tree atype = NULL_TREE;
8463 5788 : if (dump_file && (dump_flags & TDF_DETAILS))
8464 0 : fprintf (dump_file, "Mapping SSA_NAMEs to decls:\n");
8465 33429 : EXECUTE_IF_SET_IN_BITMAP (large_huge.m_names, 0, i, bi)
8466 : {
8467 27641 : tree s = ssa_name (i);
8468 27641 : int p = var_to_partition (large_huge.m_map, s);
8469 27641 : if (large_huge.m_vars[p] == NULL_TREE)
8470 : {
8471 19068 : if (atype == NULL_TREE
8472 33265 : || !tree_int_cst_equal (TYPE_SIZE (atype),
8473 14197 : TYPE_SIZE (TREE_TYPE (s))))
8474 : {
8475 7862 : unsigned HOST_WIDE_INT nelts
8476 7862 : = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (s))) / limb_prec;
8477 7862 : atype = build_array_type_nelts (large_huge.m_limb_type,
8478 7862 : nelts);
8479 : }
8480 19068 : large_huge.m_vars[p] = create_tmp_var (atype, "bitint");
8481 19068 : mark_addressable (large_huge.m_vars[p]);
8482 : }
8483 27641 : if (dump_file && (dump_flags & TDF_DETAILS))
8484 : {
8485 0 : print_generic_expr (dump_file, s, TDF_SLIM);
8486 0 : fprintf (dump_file, " -> ");
8487 0 : print_generic_expr (dump_file, large_huge.m_vars[p], TDF_SLIM);
8488 0 : fprintf (dump_file, "\n");
8489 : }
8490 : }
8491 : }
8492 :
8493 47615 : FOR_EACH_BB_REVERSE_FN (bb, cfun)
8494 : {
8495 40129 : gimple_stmt_iterator prev;
8496 202661 : for (gimple_stmt_iterator gsi = gsi_last_bb (bb); !gsi_end_p (gsi);
8497 122403 : gsi = prev)
8498 : {
8499 122403 : prev = gsi;
8500 122403 : gsi_prev (&prev);
8501 122403 : ssa_op_iter iter;
8502 122403 : gimple *stmt = gsi_stmt (gsi);
8503 122403 : if (is_gimple_debug (stmt))
8504 79071 : continue;
8505 117242 : bitint_prec_kind kind = bitint_prec_small;
8506 117242 : tree t;
8507 352448 : FOR_EACH_SSA_TREE_OPERAND (t, stmt, iter, SSA_OP_ALL_OPERANDS)
8508 235206 : if (BITINT_TYPE_P (TREE_TYPE (t)))
8509 : {
8510 80397 : bitint_prec_kind this_kind
8511 80397 : = bitint_precision_kind (TREE_TYPE (t));
8512 235425 : kind = MAX (kind, this_kind);
8513 : }
8514 117242 : if (is_gimple_assign (stmt) && gimple_store_p (stmt))
8515 : {
8516 17198 : t = gimple_assign_rhs1 (stmt);
8517 17198 : if (BITINT_TYPE_P (TREE_TYPE (t)))
8518 : {
8519 14853 : bitint_prec_kind this_kind
8520 14853 : = bitint_precision_kind (TREE_TYPE (t));
8521 14853 : kind = MAX (kind, this_kind);
8522 : }
8523 : }
8524 117242 : if (is_gimple_assign (stmt)
8525 117242 : && gimple_assign_rhs_code (stmt) == FLOAT_EXPR)
8526 : {
8527 189 : t = gimple_assign_rhs1 (stmt);
8528 218 : if (BITINT_TYPE_P (TREE_TYPE (t))
8529 191 : && TREE_CODE (t) == INTEGER_CST)
8530 : {
8531 1 : bitint_prec_kind this_kind
8532 1 : = bitint_precision_kind (TREE_TYPE (t));
8533 1 : kind = MAX (kind, this_kind);
8534 : }
8535 : }
8536 117242 : if (is_gimple_call (stmt))
8537 : {
8538 26413 : t = gimple_call_lhs (stmt);
8539 26413 : if (t && TREE_CODE (TREE_TYPE (t)) == COMPLEX_TYPE)
8540 : {
8541 5335 : bitint_prec_kind this_kind = arith_overflow_arg_kind (stmt);
8542 5335 : kind = MAX (kind, this_kind);
8543 5335 : if (BITINT_TYPE_P (TREE_TYPE (TREE_TYPE (t))))
8544 : {
8545 5210 : this_kind
8546 5210 : = bitint_precision_kind (TREE_TYPE (TREE_TYPE (t)));
8547 5210 : kind = MAX (kind, this_kind);
8548 : }
8549 : }
8550 : }
8551 116920 : if (kind == bitint_prec_small)
8552 47029 : continue;
8553 70213 : switch (gimple_code (stmt))
8554 : {
8555 11310 : case GIMPLE_CALL:
8556 : /* For now. We'll need to handle some internal functions and
8557 : perhaps some builtins. */
8558 11310 : if (kind == bitint_prec_middle)
8559 2298 : continue;
8560 : break;
8561 4 : case GIMPLE_ASM:
8562 4 : if (kind == bitint_prec_middle)
8563 1 : continue;
8564 : break;
8565 1129 : case GIMPLE_RETURN:
8566 1129 : continue;
8567 49013 : case GIMPLE_ASSIGN:
8568 49013 : if (gimple_clobber_p (stmt))
8569 3521 : continue;
8570 45492 : if (kind >= bitint_prec_large)
8571 : break;
8572 8848 : if (gimple_assign_single_p (stmt))
8573 : /* No need to lower copies, loads or stores. */
8574 5826 : continue;
8575 3022 : if (gimple_assign_cast_p (stmt))
8576 : {
8577 2443 : tree lhs = gimple_assign_lhs (stmt);
8578 2443 : tree rhs = gimple_assign_rhs1 (stmt);
8579 4886 : if (INTEGRAL_TYPE_P (TREE_TYPE (lhs))
8580 2443 : && INTEGRAL_TYPE_P (TREE_TYPE (rhs))
8581 4880 : && (TYPE_PRECISION (TREE_TYPE (lhs))
8582 2437 : == TYPE_PRECISION (TREE_TYPE (rhs))))
8583 : /* No need to lower casts to same precision. */
8584 35 : continue;
8585 : }
8586 : break;
8587 : default:
8588 : break;
8589 1129 : }
8590 :
8591 11744 : if (kind == bitint_prec_middle)
8592 : {
8593 5131 : tree type = NULL_TREE;
8594 : /* Middle _BitInt(N) is rewritten to casts to INTEGER_TYPEs
8595 : with the same precision and back. */
8596 5131 : unsigned int nops = gimple_num_ops (stmt);
8597 17227 : for (unsigned int i = is_gimple_assign (stmt) ? 1 : 0;
8598 17227 : i < nops; ++i)
8599 12096 : if (tree op = gimple_op (stmt, i))
8600 : {
8601 7834 : tree nop = maybe_cast_middle_bitint (&gsi, op, type);
8602 7834 : if (nop != op)
8603 6861 : gimple_set_op (stmt, i, nop);
8604 973 : else if (COMPARISON_CLASS_P (op))
8605 : {
8606 0 : TREE_OPERAND (op, 0)
8607 0 : = maybe_cast_middle_bitint (&gsi,
8608 0 : TREE_OPERAND (op, 0),
8609 : type);
8610 0 : TREE_OPERAND (op, 1)
8611 0 : = maybe_cast_middle_bitint (&gsi,
8612 0 : TREE_OPERAND (op, 1),
8613 : type);
8614 : }
8615 973 : else if (TREE_CODE (op) == CASE_LABEL_EXPR)
8616 : {
8617 100 : CASE_LOW (op)
8618 100 : = maybe_cast_middle_bitint (&gsi, CASE_LOW (op),
8619 : type);
8620 200 : CASE_HIGH (op)
8621 200 : = maybe_cast_middle_bitint (&gsi, CASE_HIGH (op),
8622 : type);
8623 : }
8624 : }
8625 5131 : if (tree lhs = gimple_get_lhs (stmt))
8626 4559 : if (BITINT_TYPE_P (TREE_TYPE (lhs))
8627 2987 : && (bitint_precision_kind (TREE_TYPE (lhs))
8628 : == bitint_prec_middle))
8629 : {
8630 1403 : int prec = TYPE_PRECISION (TREE_TYPE (lhs));
8631 1403 : int uns = TYPE_UNSIGNED (TREE_TYPE (lhs));
8632 1403 : type = build_nonstandard_integer_type (prec, uns);
8633 1403 : tree lhs2 = make_ssa_name (type);
8634 1403 : gimple_set_lhs (stmt, lhs2);
8635 1403 : gimple *g = gimple_build_assign (lhs, NOP_EXPR, lhs2);
8636 1403 : if (stmt_ends_bb_p (stmt))
8637 : {
8638 4 : edge e = find_fallthru_edge (gsi_bb (gsi)->succs);
8639 4 : gsi_insert_on_edge (e, g);
8640 4 : edge_insertions = true;
8641 : }
8642 : else
8643 1399 : gsi_insert_after (&gsi, g, GSI_SAME_STMT);
8644 : }
8645 5131 : update_stmt (stmt);
8646 5131 : continue;
8647 5131 : }
8648 :
8649 52272 : if (tree lhs = gimple_get_lhs (stmt))
8650 45525 : if (TREE_CODE (lhs) == SSA_NAME)
8651 : {
8652 36439 : tree type = TREE_TYPE (lhs);
8653 36439 : if (TREE_CODE (type) == COMPLEX_TYPE)
8654 4207 : type = TREE_TYPE (type);
8655 5444 : if (BITINT_TYPE_P (type)
8656 31007 : && bitint_precision_kind (type) >= bitint_prec_large
8657 67244 : && (large_huge.m_names == NULL
8658 30600 : || !bitmap_bit_p (large_huge.m_names,
8659 30600 : SSA_NAME_VERSION (lhs))))
8660 8940 : continue;
8661 : }
8662 :
8663 43332 : large_huge.lower_stmt (stmt);
8664 : }
8665 :
8666 40129 : tree atype = NULL_TREE;
8667 48943 : for (gphi_iterator gsi = gsi_start_phis (bb); !gsi_end_p (gsi);
8668 8814 : gsi_next (&gsi))
8669 : {
8670 8814 : gphi *phi = gsi.phi ();
8671 8814 : tree lhs = gimple_phi_result (phi);
8672 17316 : if (!BITINT_TYPE_P (TREE_TYPE (lhs))
8673 8814 : || bitint_precision_kind (TREE_TYPE (lhs)) < bitint_prec_large)
8674 8550 : continue;
8675 264 : int p1 = var_to_partition (large_huge.m_map, lhs);
8676 264 : gcc_assert (large_huge.m_vars[p1] != NULL_TREE);
8677 : tree v1 = large_huge.m_vars[p1];
8678 899 : for (unsigned i = 0; i < gimple_phi_num_args (phi); ++i)
8679 : {
8680 635 : tree arg = gimple_phi_arg_def (phi, i);
8681 635 : edge e = gimple_phi_arg_edge (phi, i);
8682 635 : gimple *g;
8683 635 : switch (TREE_CODE (arg))
8684 : {
8685 130 : case INTEGER_CST:
8686 130 : if (integer_zerop (arg) && VAR_P (v1))
8687 : {
8688 60 : tree zero = build_zero_cst (TREE_TYPE (v1));
8689 60 : g = gimple_build_assign (v1, zero);
8690 60 : gsi_insert_on_edge (e, g);
8691 60 : edge_insertions = true;
8692 232 : break;
8693 : }
8694 70 : int ext;
8695 70 : unsigned int min_prec, prec, rem;
8696 70 : tree c;
8697 70 : prec = TYPE_PRECISION (TREE_TYPE (arg));
8698 70 : rem = prec % (2 * limb_prec);
8699 70 : min_prec = bitint_min_cst_precision (arg, ext);
8700 70 : if (min_prec > prec - rem - 2 * limb_prec
8701 28 : && min_prec > (unsigned) limb_prec)
8702 : /* Constant which has enough significant bits that it
8703 : isn't worth trying to save .rodata space by extending
8704 : from smaller number. */
8705 : min_prec = prec;
8706 : else
8707 : {
8708 61 : min_prec = CEIL (min_prec, limb_prec) * limb_prec;
8709 61 : if (min_prec > (unsigned) limb_prec
8710 3 : && abi_limb_prec > limb_prec)
8711 : {
8712 : /* For targets with ABI limb precision higher than
8713 : limb precision round to ABI limb precision,
8714 : otherwise c can contain padding bits. */
8715 0 : min_prec
8716 0 : = CEIL (min_prec, abi_limb_prec) * abi_limb_prec;
8717 0 : if (min_prec > prec - rem - 2 * limb_prec)
8718 9 : min_prec = prec;
8719 : }
8720 : }
8721 70 : if (min_prec == 0)
8722 : c = NULL_TREE;
8723 67 : else if (min_prec == prec)
8724 9 : c = tree_output_constant_def (arg);
8725 58 : else if (min_prec == (unsigned) limb_prec)
8726 55 : c = fold_convert (large_huge.m_limb_type, arg);
8727 : else
8728 : {
8729 3 : tree ctype = build_bitint_type (min_prec, 1);
8730 3 : c = tree_output_constant_def (fold_convert (ctype, arg));
8731 : }
8732 67 : if (c)
8733 : {
8734 67 : if (VAR_P (v1) && min_prec == prec)
8735 : {
8736 8 : tree v2 = build1 (VIEW_CONVERT_EXPR,
8737 8 : TREE_TYPE (v1), c);
8738 8 : g = gimple_build_assign (v1, v2);
8739 8 : gsi_insert_on_edge (e, g);
8740 8 : edge_insertions = true;
8741 8 : break;
8742 : }
8743 59 : if (TREE_CODE (TREE_TYPE (c)) == INTEGER_TYPE)
8744 : {
8745 55 : if (bitint_big_endian)
8746 : {
8747 0 : tree ptype = build_pointer_type (TREE_TYPE (v1));
8748 0 : tree sz1 = TYPE_SIZE_UNIT (TREE_TYPE (v1));
8749 0 : tree sz2 = TYPE_SIZE_UNIT (TREE_TYPE (c));
8750 0 : tree off = build_int_cst (ptype,
8751 0 : tree_to_uhwi (sz1)
8752 0 : - tree_to_uhwi (sz2));
8753 0 : tree vd = build2 (MEM_REF, TREE_TYPE (c),
8754 : build_fold_addr_expr (v1),
8755 : off);
8756 0 : g = gimple_build_assign (vd, c);
8757 : }
8758 : else
8759 55 : g = gimple_build_assign (build1 (VIEW_CONVERT_EXPR,
8760 55 : TREE_TYPE (c),
8761 : v1), c);
8762 : }
8763 : else
8764 : {
8765 4 : unsigned HOST_WIDE_INT nelts
8766 4 : = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (c)))
8767 4 : / limb_prec;
8768 4 : tree vtype
8769 8 : = build_array_type_nelts (large_huge.m_limb_type,
8770 4 : nelts);
8771 4 : tree vd;
8772 4 : if (bitint_big_endian)
8773 : {
8774 0 : tree ptype = build_pointer_type (TREE_TYPE (v1));
8775 0 : tree sz1 = TYPE_SIZE_UNIT (TREE_TYPE (v1));
8776 0 : tree sz2 = TYPE_SIZE_UNIT (vtype);
8777 0 : tree off = build_int_cst (ptype,
8778 0 : tree_to_uhwi (sz1)
8779 0 : - tree_to_uhwi (sz2));
8780 0 : vd = build2 (MEM_REF, vtype,
8781 : build_fold_addr_expr (v1), off);
8782 : }
8783 : else
8784 4 : vd = build1 (VIEW_CONVERT_EXPR, vtype, v1);
8785 4 : g = gimple_build_assign (vd,
8786 : build1 (VIEW_CONVERT_EXPR,
8787 : vtype, c));
8788 : }
8789 59 : gsi_insert_on_edge (e, g);
8790 59 : if (min_prec == prec)
8791 : {
8792 : edge_insertions = true;
8793 : break;
8794 : }
8795 : }
8796 61 : if (ext == 0)
8797 : {
8798 55 : unsigned HOST_WIDE_INT nelts
8799 55 : = (tree_to_uhwi (TYPE_SIZE (TREE_TYPE (v1)))
8800 55 : - min_prec) / limb_prec;
8801 55 : tree vtype
8802 110 : = build_array_type_nelts (large_huge.m_limb_type,
8803 55 : nelts);
8804 55 : tree ptype = build_pointer_type (TREE_TYPE (v1));
8805 55 : tree off;
8806 55 : if (c && !bitint_big_endian)
8807 54 : off = fold_convert (ptype,
8808 : TYPE_SIZE_UNIT (TREE_TYPE (c)));
8809 : else
8810 1 : off = build_zero_cst (ptype);
8811 55 : tree vd = build2 (MEM_REF, vtype,
8812 : build_fold_addr_expr (v1), off);
8813 55 : g = gimple_build_assign (vd, build_zero_cst (vtype));
8814 : }
8815 : else
8816 : {
8817 6 : tree vd = v1;
8818 6 : if (c && !bitint_big_endian)
8819 : {
8820 4 : tree ptype = build_pointer_type (TREE_TYPE (v1));
8821 4 : tree off
8822 4 : = fold_convert (ptype,
8823 : TYPE_SIZE_UNIT (TREE_TYPE (c)));
8824 4 : vd = build2 (MEM_REF, large_huge.m_limb_type,
8825 : build_fold_addr_expr (v1), off);
8826 : }
8827 6 : vd = build_fold_addr_expr (vd);
8828 6 : unsigned HOST_WIDE_INT nbytes
8829 6 : = tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (v1)));
8830 6 : if (c)
8831 4 : nbytes
8832 4 : -= tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (c)));
8833 6 : tree fn = builtin_decl_implicit (BUILT_IN_MEMSET);
8834 6 : g = gimple_build_call (fn, 3, vd,
8835 : integer_minus_one_node,
8836 : build_int_cst (sizetype,
8837 6 : nbytes));
8838 : }
8839 61 : gsi_insert_on_edge (e, g);
8840 61 : edge_insertions = true;
8841 61 : break;
8842 0 : default:
8843 0 : gcc_unreachable ();
8844 505 : case SSA_NAME:
8845 505 : if (gimple_code (SSA_NAME_DEF_STMT (arg)) == GIMPLE_NOP)
8846 : {
8847 49 : if (large_huge.m_names == NULL
8848 98 : || !bitmap_bit_p (large_huge.m_names,
8849 49 : SSA_NAME_VERSION (arg)))
8850 463 : continue;
8851 : }
8852 505 : int p2 = var_to_partition (large_huge.m_map, arg);
8853 505 : if (p1 == p2)
8854 463 : continue;
8855 42 : gcc_assert (large_huge.m_vars[p2] != NULL_TREE);
8856 42 : tree v2 = large_huge.m_vars[p2];
8857 42 : if (VAR_P (v1) && VAR_P (v2))
8858 18 : g = gimple_build_assign (v1, v2);
8859 24 : else if (VAR_P (v1))
8860 16 : g = gimple_build_assign (v1, build1 (VIEW_CONVERT_EXPR,
8861 16 : TREE_TYPE (v1), v2));
8862 8 : else if (VAR_P (v2))
8863 0 : g = gimple_build_assign (build1 (VIEW_CONVERT_EXPR,
8864 0 : TREE_TYPE (v2), v1), v2);
8865 : else
8866 : {
8867 8 : if (atype == NULL_TREE
8868 8 : || !tree_int_cst_equal (TYPE_SIZE (atype),
8869 0 : TYPE_SIZE (TREE_TYPE (lhs))))
8870 : {
8871 8 : unsigned HOST_WIDE_INT nelts
8872 8 : = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (lhs)))
8873 8 : / limb_prec;
8874 8 : atype
8875 8 : = build_array_type_nelts (large_huge.m_limb_type,
8876 8 : nelts);
8877 : }
8878 8 : g = gimple_build_assign (build1 (VIEW_CONVERT_EXPR,
8879 : atype, v1),
8880 : build1 (VIEW_CONVERT_EXPR,
8881 : atype, v2));
8882 : }
8883 42 : gsi_insert_on_edge (e, g);
8884 42 : edge_insertions = true;
8885 42 : break;
8886 : }
8887 : }
8888 : }
8889 : }
8890 :
8891 7486 : if (large_huge.m_names || has_large_huge)
8892 : {
8893 5988 : gimple *nop = NULL;
8894 393588 : for (i = 0; i < num_ssa_names; ++i)
8895 : {
8896 387600 : tree s = ssa_name (i);
8897 387600 : if (s == NULL_TREE)
8898 15924 : continue;
8899 371676 : tree type = TREE_TYPE (s);
8900 371676 : if (TREE_CODE (type) == COMPLEX_TYPE)
8901 17757 : type = TREE_TYPE (type);
8902 331200 : if (BITINT_TYPE_P (type)
8903 371703 : && bitint_precision_kind (type) >= bitint_prec_large)
8904 : {
8905 41326 : if (large_huge.m_preserved
8906 45824 : && bitmap_bit_p (large_huge.m_preserved,
8907 6841 : SSA_NAME_VERSION (s)))
8908 2343 : continue;
8909 36640 : gimple *g = SSA_NAME_DEF_STMT (s);
8910 36640 : if (gimple_code (g) == GIMPLE_NOP)
8911 : {
8912 9924 : if (SSA_NAME_VAR (s))
8913 5547 : set_ssa_default_def (cfun, SSA_NAME_VAR (s), NULL_TREE);
8914 9924 : release_ssa_name (s);
8915 9924 : continue;
8916 : }
8917 26716 : if (gimple_bb (g) == NULL)
8918 : {
8919 0 : release_ssa_name (s);
8920 0 : continue;
8921 : }
8922 26716 : if (gimple_code (g) != GIMPLE_ASM)
8923 : {
8924 26715 : gimple_stmt_iterator gsi = gsi_for_stmt (g);
8925 26715 : bool save_vta = flag_var_tracking_assignments;
8926 26715 : flag_var_tracking_assignments = false;
8927 26715 : gsi_remove (&gsi, true);
8928 26715 : flag_var_tracking_assignments = save_vta;
8929 : }
8930 26716 : if (nop == NULL)
8931 4976 : nop = gimple_build_nop ();
8932 26716 : SSA_NAME_DEF_STMT (s) = nop;
8933 26716 : release_ssa_name (s);
8934 : }
8935 : }
8936 5988 : if (optimize)
8937 3192 : disable_ranger (cfun);
8938 : }
8939 :
8940 7486 : if (edge_insertions)
8941 80 : gsi_commit_edge_inserts ();
8942 :
8943 : /* Fix up arguments of ECF_RETURNS_TWICE calls. Those were temporarily
8944 : inserted before the call, but that is invalid IL, so move them to the
8945 : right place and add corresponding PHIs. */
8946 7486 : if (!large_huge.m_returns_twice_calls.is_empty ())
8947 : {
8948 9 : auto_vec<gimple *, 16> arg_stmts;
8949 29 : while (!large_huge.m_returns_twice_calls.is_empty ())
8950 : {
8951 11 : gimple *stmt = large_huge.m_returns_twice_calls.pop ();
8952 11 : gimple_stmt_iterator gsi = gsi_after_labels (gimple_bb (stmt));
8953 36 : while (gsi_stmt (gsi) != stmt)
8954 : {
8955 25 : if (is_gimple_debug (gsi_stmt (gsi)))
8956 2 : gsi_next (&gsi);
8957 : else
8958 : {
8959 23 : arg_stmts.safe_push (gsi_stmt (gsi));
8960 23 : gsi_remove (&gsi, false);
8961 : }
8962 : }
8963 11 : gimple *g;
8964 11 : basic_block bb = NULL;
8965 11 : edge e = NULL, ead = NULL;
8966 34 : FOR_EACH_VEC_ELT (arg_stmts, i, g)
8967 : {
8968 23 : gsi_safe_insert_before (&gsi, g);
8969 23 : if (i == 0)
8970 : {
8971 11 : bb = gimple_bb (stmt);
8972 11 : gcc_checking_assert (EDGE_COUNT (bb->preds) == 2);
8973 11 : e = EDGE_PRED (bb, 0);
8974 11 : ead = EDGE_PRED (bb, 1);
8975 11 : if ((ead->flags & EDGE_ABNORMAL) == 0)
8976 0 : std::swap (e, ead);
8977 11 : gcc_checking_assert ((e->flags & EDGE_ABNORMAL) == 0
8978 : && (ead->flags & EDGE_ABNORMAL));
8979 : }
8980 23 : tree lhs = gimple_assign_lhs (g);
8981 23 : tree arg = lhs;
8982 23 : gphi *phi = create_phi_node (copy_ssa_name (arg), bb);
8983 23 : add_phi_arg (phi, arg, e, UNKNOWN_LOCATION);
8984 23 : tree var = create_tmp_reg (TREE_TYPE (arg));
8985 23 : suppress_warning (var, OPT_Wuninitialized);
8986 23 : arg = get_or_create_ssa_default_def (cfun, var);
8987 23 : SSA_NAME_OCCURS_IN_ABNORMAL_PHI (arg) = 1;
8988 23 : add_phi_arg (phi, arg, ead, UNKNOWN_LOCATION);
8989 23 : arg = gimple_phi_result (phi);
8990 23 : SSA_NAME_OCCURS_IN_ABNORMAL_PHI (arg) = 1;
8991 23 : imm_use_iterator iter;
8992 23 : gimple *use_stmt;
8993 69 : FOR_EACH_IMM_USE_STMT (use_stmt, iter, lhs)
8994 : {
8995 46 : if (use_stmt == phi)
8996 23 : continue;
8997 23 : gcc_checking_assert (use_stmt == stmt);
8998 23 : use_operand_p use_p;
8999 46 : FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
9000 23 : SET_USE (use_p, arg);
9001 23 : }
9002 : }
9003 11 : update_stmt (stmt);
9004 11 : arg_stmts.truncate (0);
9005 : }
9006 9 : }
9007 :
9008 7486 : if (loop_init_done)
9009 4218 : loop_optimizer_finalize ();
9010 :
9011 7486 : return ret;
9012 7486 : }
9013 :
9014 : namespace {
9015 :
9016 : const pass_data pass_data_lower_bitint =
9017 : {
9018 : GIMPLE_PASS, /* type */
9019 : "bitintlower", /* name */
9020 : OPTGROUP_NONE, /* optinfo_flags */
9021 : TV_NONE, /* tv_id */
9022 : PROP_ssa, /* properties_required */
9023 : PROP_gimple_lbitint, /* properties_provided */
9024 : 0, /* properties_destroyed */
9025 : 0, /* todo_flags_start */
9026 : 0, /* todo_flags_finish */
9027 : };
9028 :
9029 : class pass_lower_bitint : public gimple_opt_pass
9030 : {
9031 : public:
9032 594360 : pass_lower_bitint (gcc::context *ctxt)
9033 1188720 : : gimple_opt_pass (pass_data_lower_bitint, ctxt)
9034 : {}
9035 :
9036 : /* opt_pass methods: */
9037 297180 : opt_pass * clone () final override { return new pass_lower_bitint (m_ctxt); }
9038 1065442 : unsigned int execute (function *) final override
9039 : {
9040 1065442 : return gimple_lower_bitint ();
9041 : }
9042 :
9043 : }; // class pass_lower_bitint
9044 :
9045 : } // anon namespace
9046 :
9047 : gimple_opt_pass *
9048 297180 : make_pass_lower_bitint (gcc::context *ctxt)
9049 : {
9050 297180 : return new pass_lower_bitint (ctxt);
9051 : }
9052 :
9053 :
9054 : namespace {
9055 :
9056 : const pass_data pass_data_lower_bitint_O0 =
9057 : {
9058 : GIMPLE_PASS, /* type */
9059 : "bitintlower0", /* name */
9060 : OPTGROUP_NONE, /* optinfo_flags */
9061 : TV_NONE, /* tv_id */
9062 : PROP_cfg, /* properties_required */
9063 : PROP_gimple_lbitint, /* properties_provided */
9064 : 0, /* properties_destroyed */
9065 : 0, /* todo_flags_start */
9066 : 0, /* todo_flags_finish */
9067 : };
9068 :
9069 : class pass_lower_bitint_O0 : public gimple_opt_pass
9070 : {
9071 : public:
9072 297180 : pass_lower_bitint_O0 (gcc::context *ctxt)
9073 594360 : : gimple_opt_pass (pass_data_lower_bitint_O0, ctxt)
9074 : {}
9075 :
9076 : /* opt_pass methods: */
9077 1517917 : bool gate (function *fun) final override
9078 : {
9079 : /* With errors, normal optimization passes are not run. If we don't
9080 : lower bitint operations at all, rtl expansion will abort. */
9081 1517917 : return !(fun->curr_properties & PROP_gimple_lbitint);
9082 : }
9083 :
9084 452614 : unsigned int execute (function *) final override
9085 : {
9086 452614 : return gimple_lower_bitint ();
9087 : }
9088 :
9089 : }; // class pass_lower_bitint_O0
9090 :
9091 : } // anon namespace
9092 :
9093 : gimple_opt_pass *
9094 297180 : make_pass_lower_bitint_O0 (gcc::context *ctxt)
9095 : {
9096 297180 : return new pass_lower_bitint_O0 (ctxt);
9097 : }
|