Line data Source code
1 : /* Lower _BitInt(N) operations to scalar operations.
2 : Copyright (C) 2023-2026 Free Software Foundation, Inc.
3 : Contributed by Jakub Jelinek <jakub@redhat.com>.
4 :
5 : This file is part of GCC.
6 :
7 : GCC is free software; you can redistribute it and/or modify it
8 : under the terms of the GNU General Public License as published by the
9 : Free Software Foundation; either version 3, or (at your option) any
10 : later version.
11 :
12 : GCC is distributed in the hope that it will be useful, but WITHOUT
13 : ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 : for more details.
16 :
17 : You should have received a copy of the GNU General Public License
18 : along with GCC; see the file COPYING3. If not see
19 : <http://www.gnu.org/licenses/>. */
20 :
21 : #include "config.h"
22 : #include "system.h"
23 : #include "coretypes.h"
24 : #include "backend.h"
25 : #include "rtl.h"
26 : #include "tree.h"
27 : #include "gimple.h"
28 : #include "cfghooks.h"
29 : #include "tree-pass.h"
30 : #include "ssa.h"
31 : #include "fold-const.h"
32 : #include "gimplify.h"
33 : #include "gimple-iterator.h"
34 : #include "tree-cfg.h"
35 : #include "tree-dfa.h"
36 : #include "cfgloop.h"
37 : #include "cfganal.h"
38 : #include "target.h"
39 : #include "tree-ssa-live.h"
40 : #include "tree-ssa-coalesce.h"
41 : #include "domwalk.h"
42 : #include "memmodel.h"
43 : #include "optabs.h"
44 : #include "varasm.h"
45 : #include "gimple-range.h"
46 : #include "value-range.h"
47 : #include "langhooks.h"
48 : #include "gimplify-me.h"
49 : #include "diagnostic-core.h"
50 : #include "tree-eh.h"
51 : #include "tree-pretty-print.h"
52 : #include "alloc-pool.h"
53 : #include "tree-into-ssa.h"
54 : #include "tree-cfgcleanup.h"
55 : #include "tree-switch-conversion.h"
56 : #include "ubsan.h"
57 : #include "stor-layout.h"
58 : #include "gimple-lower-bitint.h"
59 : #include "attribs.h"
60 : #include "asan.h"
61 :
62 : /* Split BITINT_TYPE precisions in 4 categories. Small _BitInt, where
63 : target hook says it is a single limb, middle _BitInt which per ABI
64 : does not, but there is some INTEGER_TYPE in which arithmetics can be
65 : performed (operations on such _BitInt are lowered to casts to that
66 : arithmetic type and cast back; e.g. on x86_64 limb is DImode, but
67 : target supports TImode, so _BitInt(65) to _BitInt(128) are middle
68 : ones), large _BitInt which should by straight line code and
69 : finally huge _BitInt which should be handled by loops over the limbs. */
70 :
71 : enum bitint_prec_kind {
72 : bitint_prec_small,
73 : bitint_prec_middle,
74 : bitint_prec_large,
75 : bitint_prec_huge
76 : };
77 :
78 : /* Caches to speed up bitint_precision_kind. */
79 :
80 : static int small_max_prec, mid_min_prec, large_min_prec, huge_min_prec;
81 : static int limb_prec, abi_limb_prec;
82 : static bool bitint_big_endian;
83 : static enum bitint_ext bitint_extended;
84 :
85 : /* Categorize _BitInt(PREC) as small, middle, large or huge. */
86 :
87 : static bitint_prec_kind
88 489455 : bitint_precision_kind (int prec)
89 : {
90 489455 : if (prec <= small_max_prec)
91 : return bitint_prec_small;
92 470970 : if (huge_min_prec && prec >= huge_min_prec)
93 : return bitint_prec_huge;
94 265647 : if (large_min_prec && prec >= large_min_prec)
95 : return bitint_prec_large;
96 52838 : if (mid_min_prec && prec >= mid_min_prec)
97 : return bitint_prec_middle;
98 :
99 10091 : struct bitint_info info;
100 10091 : bool ok = targetm.c.bitint_type_info (prec, &info);
101 10091 : gcc_assert (ok);
102 10091 : scalar_int_mode limb_mode = as_a <scalar_int_mode> (info.limb_mode);
103 10091 : if (prec <= GET_MODE_PRECISION (limb_mode))
104 : {
105 2626 : small_max_prec = prec;
106 2626 : return bitint_prec_small;
107 : }
108 7465 : bitint_big_endian = info.big_endian;
109 7465 : bitint_extended = info.extended;
110 7465 : if (info.limb_mode == info.abi_limb_mode && bitint_extended == bitint_ext_full)
111 0 : bitint_extended = bitint_ext_partial;
112 7465 : if (!large_min_prec
113 14833 : && GET_MODE_PRECISION (limb_mode) <= MAX_FIXED_MODE_SIZE)
114 14736 : large_min_prec = MAX_FIXED_MODE_SIZE + 1;
115 7465 : if (!limb_prec)
116 7368 : limb_prec = GET_MODE_PRECISION (limb_mode);
117 7465 : if (!abi_limb_prec)
118 7368 : abi_limb_prec
119 7368 : = GET_MODE_PRECISION (as_a <scalar_int_mode> (info.abi_limb_mode));
120 : /* For bitint_ext_full with different limb_mode from abi_limb_mode we
121 : currently only support only abi_limb_mode twice the precision of
122 : limb_mode, and don't support big endian in that case either. */
123 7465 : gcc_assert (bitint_extended != bitint_ext_full
124 : || (abi_limb_prec == 2 * limb_prec
125 : && !bitint_big_endian));
126 7465 : if (!huge_min_prec)
127 : {
128 14736 : if (4 * limb_prec >= MAX_FIXED_MODE_SIZE)
129 7368 : huge_min_prec = 4 * limb_prec;
130 : else
131 0 : huge_min_prec = MAX_FIXED_MODE_SIZE + 1;
132 : }
133 14930 : if (prec <= MAX_FIXED_MODE_SIZE)
134 : {
135 1625 : if (!mid_min_prec || prec < mid_min_prec)
136 1625 : mid_min_prec = prec;
137 : return bitint_prec_middle;
138 : }
139 5840 : if (huge_min_prec && prec >= huge_min_prec)
140 2361 : return bitint_prec_huge;
141 : return bitint_prec_large;
142 : }
143 :
144 : /* Same for a TYPE. */
145 :
146 : static bitint_prec_kind
147 483989 : bitint_precision_kind (tree type)
148 : {
149 483989 : return bitint_precision_kind (TYPE_PRECISION (type));
150 : }
151 :
152 : /* Return minimum precision needed to describe INTEGER_CST
153 : CST. All bits above that precision up to precision of
154 : TREE_TYPE (CST) are cleared if EXT is set to 0, or set
155 : if EXT is set to -1. */
156 :
157 : static unsigned
158 5399 : bitint_min_cst_precision (tree cst, int &ext)
159 : {
160 5399 : ext = tree_int_cst_sgn (cst) < 0 ? -1 : 0;
161 5399 : wide_int w = wi::to_wide (cst);
162 5399 : unsigned min_prec = wi::min_precision (w, TYPE_SIGN (TREE_TYPE (cst)));
163 : /* For signed values, we don't need to count the sign bit,
164 : we'll use constant 0 or -1 for the upper bits. */
165 5399 : if (!TYPE_UNSIGNED (TREE_TYPE (cst)))
166 3281 : --min_prec;
167 : else
168 : {
169 : /* For unsigned values, also try signed min_precision
170 : in case the constant has lots of most significant bits set. */
171 2118 : unsigned min_prec2 = wi::min_precision (w, SIGNED) - 1;
172 2118 : if (min_prec2 < min_prec)
173 : {
174 1002 : ext = -1;
175 1002 : return min_prec2;
176 : }
177 : }
178 : return min_prec;
179 5399 : }
180 :
181 : namespace {
182 :
183 : /* If OP is middle _BitInt, cast it to corresponding INTEGER_TYPE
184 : cached in TYPE and return it. */
185 :
186 : tree
187 7859 : maybe_cast_middle_bitint (gimple_stmt_iterator *gsi, tree op, tree &type)
188 : {
189 7859 : if (op == NULL_TREE
190 7826 : || !BITINT_TYPE_P (TREE_TYPE (op))
191 14803 : || bitint_precision_kind (TREE_TYPE (op)) != bitint_prec_middle)
192 : return op;
193 :
194 6927 : int prec = TYPE_PRECISION (TREE_TYPE (op));
195 6927 : int uns = TYPE_UNSIGNED (TREE_TYPE (op));
196 6927 : if (type == NULL_TREE
197 2557 : || TYPE_PRECISION (type) != prec
198 9484 : || TYPE_UNSIGNED (type) != uns)
199 4370 : type = build_nonstandard_integer_type (prec, uns);
200 :
201 6927 : if (TREE_CODE (op) != SSA_NAME)
202 : {
203 2365 : tree nop = fold_convert (type, op);
204 2365 : if (is_gimple_val (nop))
205 : return nop;
206 : }
207 :
208 4562 : tree nop = make_ssa_name (type);
209 4562 : gimple *g = gimple_build_assign (nop, NOP_EXPR, op);
210 4562 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
211 4562 : return nop;
212 : }
213 :
214 : /* Return true if STMT can be handled in a loop from least to most
215 : significant limb together with its dependencies. */
216 :
217 : bool
218 48100 : mergeable_op (gimple *stmt)
219 : {
220 48100 : if (!is_gimple_assign (stmt))
221 : return false;
222 39375 : switch (gimple_assign_rhs_code (stmt))
223 : {
224 : case PLUS_EXPR:
225 : case MINUS_EXPR:
226 : case NEGATE_EXPR:
227 : case BIT_AND_EXPR:
228 : case BIT_IOR_EXPR:
229 : case BIT_XOR_EXPR:
230 : case BIT_NOT_EXPR:
231 : case SSA_NAME:
232 : case INTEGER_CST:
233 : case BIT_FIELD_REF:
234 : return true;
235 410 : case LSHIFT_EXPR:
236 410 : {
237 410 : tree cnt = gimple_assign_rhs2 (stmt);
238 410 : if (tree_fits_uhwi_p (cnt)
239 198 : && tree_to_uhwi (cnt) < (unsigned HOST_WIDE_INT) limb_prec)
240 : return true;
241 : }
242 : break;
243 6341 : CASE_CONVERT:
244 6341 : case VIEW_CONVERT_EXPR:
245 6341 : {
246 6341 : tree lhs_type = TREE_TYPE (gimple_assign_lhs (stmt));
247 6341 : tree rhs_type = TREE_TYPE (gimple_assign_rhs1 (stmt));
248 6341 : if (TREE_CODE (gimple_assign_rhs1 (stmt)) == SSA_NAME
249 6299 : && BITINT_TYPE_P (lhs_type)
250 3724 : && BITINT_TYPE_P (rhs_type)
251 3369 : && bitint_precision_kind (lhs_type) >= bitint_prec_large
252 3333 : && bitint_precision_kind (rhs_type) >= bitint_prec_large
253 9513 : && (CEIL (TYPE_PRECISION (lhs_type), limb_prec)
254 3172 : == CEIL (TYPE_PRECISION (rhs_type), limb_prec)))
255 : {
256 2198 : if (TYPE_PRECISION (rhs_type) >= TYPE_PRECISION (lhs_type))
257 : return true;
258 183 : if ((unsigned) TYPE_PRECISION (lhs_type) % (2 * limb_prec) != 0)
259 : return true;
260 17 : if (bitint_precision_kind (lhs_type) == bitint_prec_large)
261 : return true;
262 : }
263 : break;
264 : }
265 : default:
266 : break;
267 : }
268 : return false;
269 : }
270 :
271 : /* Return non-zero if stmt is .{ADD,SUB,MUL}_OVERFLOW call with
272 : _Complex large/huge _BitInt lhs which has at most two immediate uses,
273 : at most one use in REALPART_EXPR stmt in the same bb and exactly one
274 : IMAGPART_EXPR use in the same bb with a single use which casts it to
275 : non-BITINT_TYPE integral type. If there is a REALPART_EXPR use,
276 : return 2. Such cases (most common uses of those builtins) can be
277 : optimized by marking their lhs and lhs of IMAGPART_EXPR and maybe lhs
278 : of REALPART_EXPR as not needed to be backed up by a stack variable.
279 : For .UBSAN_CHECK_{ADD,SUB,MUL} return 3. */
280 :
281 : int
282 21481 : optimizable_arith_overflow (gimple *stmt)
283 : {
284 21481 : bool is_ubsan = false;
285 21481 : if (!is_gimple_call (stmt) || !gimple_call_internal_p (stmt))
286 : return false;
287 5236 : switch (gimple_call_internal_fn (stmt))
288 : {
289 : case IFN_ADD_OVERFLOW:
290 : case IFN_SUB_OVERFLOW:
291 : case IFN_MUL_OVERFLOW:
292 : break;
293 63 : case IFN_UBSAN_CHECK_ADD:
294 63 : case IFN_UBSAN_CHECK_SUB:
295 63 : case IFN_UBSAN_CHECK_MUL:
296 63 : is_ubsan = true;
297 63 : break;
298 : default:
299 : return 0;
300 : }
301 5210 : tree lhs = gimple_call_lhs (stmt);
302 5210 : if (!lhs)
303 : return 0;
304 5210 : if (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (lhs))
305 : return 0;
306 5195 : tree type = is_ubsan ? TREE_TYPE (lhs) : TREE_TYPE (TREE_TYPE (lhs));
307 0 : if (!BITINT_TYPE_P (type)
308 5195 : || bitint_precision_kind (type) < bitint_prec_large)
309 : return 0;
310 :
311 5195 : if (is_ubsan)
312 : {
313 48 : use_operand_p use_p;
314 48 : gimple *use_stmt;
315 48 : if (!single_imm_use (lhs, &use_p, &use_stmt)
316 48 : || gimple_bb (use_stmt) != gimple_bb (stmt)
317 48 : || !gimple_store_p (use_stmt)
318 48 : || !is_gimple_assign (use_stmt)
319 48 : || gimple_has_volatile_ops (use_stmt)
320 96 : || stmt_ends_bb_p (use_stmt))
321 0 : return 0;
322 : return 3;
323 : }
324 :
325 5147 : imm_use_iterator ui;
326 5147 : use_operand_p use_p;
327 5147 : int seen = 0;
328 5147 : gimple *realpart = NULL, *cast = NULL;
329 14740 : FOR_EACH_IMM_USE_FAST (use_p, ui, lhs)
330 : {
331 9807 : gimple *g = USE_STMT (use_p);
332 9807 : if (is_gimple_debug (g))
333 0 : continue;
334 9807 : if (!is_gimple_assign (g) || gimple_bb (g) != gimple_bb (stmt))
335 : return 0;
336 9807 : if (gimple_assign_rhs_code (g) == REALPART_EXPR)
337 : {
338 4660 : if ((seen & 1) != 0)
339 : return 0;
340 4660 : seen |= 1;
341 4660 : realpart = g;
342 : }
343 5147 : else if (gimple_assign_rhs_code (g) == IMAGPART_EXPR)
344 : {
345 5147 : if ((seen & 2) != 0)
346 214 : return 0;
347 5147 : seen |= 2;
348 :
349 5147 : use_operand_p use2_p;
350 5147 : gimple *use_stmt;
351 5147 : tree lhs2 = gimple_assign_lhs (g);
352 5147 : if (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (lhs2))
353 : return 0;
354 5147 : if (!single_imm_use (lhs2, &use2_p, &use_stmt)
355 5147 : || gimple_bb (use_stmt) != gimple_bb (stmt)
356 10294 : || !gimple_assign_cast_p (use_stmt))
357 : return 0;
358 :
359 4937 : lhs2 = gimple_assign_lhs (use_stmt);
360 9874 : if (!INTEGRAL_TYPE_P (TREE_TYPE (lhs2))
361 9874 : || BITINT_TYPE_P (TREE_TYPE (lhs2)))
362 : return 0;
363 4933 : cast = use_stmt;
364 : }
365 : else
366 : return 0;
367 214 : }
368 4933 : if ((seen & 2) == 0)
369 : return 0;
370 4933 : if (seen == 3)
371 : {
372 : /* Punt if the cast stmt appears before realpart stmt, because
373 : if both appear, the lowering wants to emit all the code
374 : at the location of realpart stmt. */
375 4660 : gimple_stmt_iterator gsi = gsi_for_stmt (realpart);
376 4660 : unsigned int cnt = 0;
377 4663 : do
378 : {
379 4663 : gsi_prev_nondebug (&gsi);
380 4663 : if (gsi_end_p (gsi) || gsi_stmt (gsi) == cast)
381 : return 0;
382 4660 : if (gsi_stmt (gsi) == stmt)
383 : return 2;
384 : /* If realpart is too far from stmt, punt as well.
385 : Usually it will appear right after it. */
386 3 : if (++cnt == 32)
387 : return 0;
388 : }
389 : while (1);
390 : }
391 : return 1;
392 : }
393 :
394 : /* If STMT is some kind of comparison (GIMPLE_COND, comparison assignment)
395 : comparing large/huge _BitInt types, return the comparison code and if
396 : non-NULL fill in the comparison operands to *POP1 and *POP2. */
397 :
398 : tree_code
399 36031 : comparison_op (gimple *stmt, tree *pop1, tree *pop2)
400 : {
401 36031 : tree op1 = NULL_TREE, op2 = NULL_TREE;
402 36031 : tree_code code = ERROR_MARK;
403 36031 : if (gimple_code (stmt) == GIMPLE_COND)
404 : {
405 6554 : code = gimple_cond_code (stmt);
406 6554 : op1 = gimple_cond_lhs (stmt);
407 6554 : op2 = gimple_cond_rhs (stmt);
408 : }
409 29477 : else if (is_gimple_assign (stmt))
410 : {
411 29450 : code = gimple_assign_rhs_code (stmt);
412 29450 : op1 = gimple_assign_rhs1 (stmt);
413 29450 : if (TREE_CODE_CLASS (code) == tcc_comparison
414 29450 : || TREE_CODE_CLASS (code) == tcc_binary)
415 2214 : op2 = gimple_assign_rhs2 (stmt);
416 : }
417 36031 : if (TREE_CODE_CLASS (code) != tcc_comparison)
418 : return ERROR_MARK;
419 7336 : tree type = TREE_TYPE (op1);
420 10 : if (!BITINT_TYPE_P (type)
421 7346 : || bitint_precision_kind (type) < bitint_prec_large)
422 : return ERROR_MARK;
423 7336 : if (pop1)
424 : {
425 7274 : *pop1 = op1;
426 7274 : *pop2 = op2;
427 : }
428 : return code;
429 : }
430 :
431 : /* Class used during large/huge _BitInt lowering containing all the
432 : state for the methods. */
433 :
434 : struct bitint_large_huge
435 : {
436 7366 : bitint_large_huge ()
437 7366 : : m_names (NULL), m_loads (NULL), m_preserved (NULL),
438 7366 : m_single_use_names (NULL), m_map (NULL), m_vars (NULL),
439 7366 : m_limb_type (NULL_TREE), m_data (vNULL),
440 7366 : m_returns_twice_calls (vNULL) {}
441 :
442 : ~bitint_large_huge ();
443 :
444 : void insert_before (gimple *);
445 : tree limb_access_type (tree, tree);
446 : tree limb_access (tree, tree, tree, bool, bool = false);
447 : tree build_bit_field_ref (tree, tree, unsigned HOST_WIDE_INT,
448 : unsigned HOST_WIDE_INT);
449 : void if_then (gimple *, profile_probability, edge &, edge &);
450 : void if_then_else (gimple *, profile_probability, edge &, edge &);
451 : void if_then_if_then_else (gimple *g, gimple *,
452 : profile_probability, profile_probability,
453 : edge &, edge &, edge &);
454 : tree handle_operand (tree, tree);
455 : tree prepare_data_in_out (tree, tree, tree *, tree = NULL_TREE);
456 : tree add_cast (tree, tree);
457 : tree handle_plus_minus (tree_code, tree, tree, tree);
458 : tree handle_lshift (tree, tree, tree);
459 : tree handle_cast (tree, tree, tree);
460 : tree handle_bit_field_ref (tree, tree);
461 : tree handle_load (gimple *, tree);
462 : tree handle_stmt (gimple *, tree);
463 : tree handle_operand_addr (tree, gimple *, int *, int *);
464 : tree create_loop (tree, tree *);
465 : tree lower_mergeable_stmt (gimple *, tree_code &, tree, tree);
466 : tree lower_comparison_stmt (gimple *, tree_code &, tree, tree);
467 : void lower_shift_stmt (tree, gimple *);
468 : void lower_muldiv_stmt (tree, gimple *);
469 : void lower_float_conv_stmt (tree, gimple *);
470 : tree arith_overflow_extract_bits (unsigned int, unsigned int, tree,
471 : unsigned int, bool);
472 : void finish_arith_overflow (tree, tree, tree, tree, tree, tree, gimple *,
473 : unsigned, tree_code);
474 : void lower_addsub_overflow (tree, gimple *);
475 : void lower_mul_overflow (tree, gimple *);
476 : void lower_cplxpart_stmt (tree, gimple *);
477 : void lower_complexexpr_stmt (gimple *);
478 : void lower_bit_query (gimple *);
479 : void lower_bswap_bitreverse (tree, gimple *);
480 : void lower_call (tree, gimple *);
481 : void lower_asm (gimple *);
482 : void lower_stmt (gimple *);
483 :
484 : /* Bitmap of large/huge _BitInt SSA_NAMEs except those can be
485 : merged with their uses. */
486 : bitmap m_names;
487 : /* Subset of those for lhs of load statements. These will be
488 : cleared in m_names if the loads will be mergeable with all
489 : their uses. */
490 : bitmap m_loads;
491 : /* Bitmap of large/huge _BitInt SSA_NAMEs that should survive
492 : to later passes (arguments or return values of calls). */
493 : bitmap m_preserved;
494 : /* Subset of m_names which have a single use. As the lowering
495 : can replace various original statements with their lowered
496 : form even before it is done iterating over all basic blocks,
497 : testing has_single_use for the purpose of emitting clobbers
498 : doesn't work properly. */
499 : bitmap m_single_use_names;
500 : /* Used for coalescing/partitioning of large/huge _BitInt SSA_NAMEs
501 : set in m_names. */
502 : var_map m_map;
503 : /* Mapping of the partitions to corresponding decls. */
504 : tree *m_vars;
505 : /* Unsigned integer type with limb precision. */
506 : tree m_limb_type;
507 : /* Its TYPE_SIZE_UNIT. */
508 : unsigned HOST_WIDE_INT m_limb_size;
509 : /* Location of a gimple stmt which is being currently lowered. */
510 : location_t m_loc;
511 : /* Current stmt iterator where code is being lowered currently. */
512 : gimple_stmt_iterator m_gsi;
513 : /* Statement after which any clobbers should be added if non-NULL. */
514 : gimple *m_after_stmt;
515 : /* Set when creating loops to the loop header bb and its preheader. */
516 : basic_block m_bb, m_preheader_bb;
517 : /* Stmt iterator after which initialization statements should be emitted. */
518 : gimple_stmt_iterator m_init_gsi;
519 : /* Decl into which a mergeable statement stores result. */
520 : tree m_lhs;
521 : /* handle_operand/handle_stmt can be invoked in various ways.
522 :
523 : lower_mergeable_stmt for large _BitInt calls those with constant
524 : idx only, expanding to straight line code, for huge _BitInt
525 : emits a loop from least significant limb upwards, where each loop
526 : iteration handles 2 limbs, plus there can be up to one full limb
527 : and one partial limb processed after the loop, where handle_operand
528 : and/or handle_stmt are called with constant idx. m_upwards_2limb
529 : is set for this case, false otherwise. m_upwards is true if it
530 : is either large or huge _BitInt handled by lower_mergeable_stmt,
531 : i.e. indexes always increase.
532 :
533 : Another way is used by lower_comparison_stmt, which walks limbs
534 : from most significant to least significant, partial limb if any
535 : processed first with constant idx and then loop processing a single
536 : limb per iteration with non-constant idx.
537 :
538 : Another way is used in lower_shift_stmt, where for LSHIFT_EXPR
539 : destination limbs are processed from most significant to least
540 : significant or for RSHIFT_EXPR the other way around, in loops or
541 : straight line code, but idx usually is non-constant (so from
542 : handle_operand/handle_stmt POV random access). The LSHIFT_EXPR
543 : handling there can access even partial limbs using non-constant
544 : idx (then m_var_msb should be true, for all the other cases
545 : including lower_mergeable_stmt/lower_comparison_stmt that is
546 : not the case and so m_var_msb should be false.
547 :
548 : m_first should be set the first time handle_operand/handle_stmt
549 : is called and clear when it is called for some other limb with
550 : the same argument. If the lowering of an operand (e.g. INTEGER_CST)
551 : or statement (e.g. +/-/<< with < limb_prec constant) needs some
552 : state between the different calls, when m_first is true it should
553 : push some trees to m_data vector and also make sure m_data_cnt is
554 : incremented by how many trees were pushed, and when m_first is
555 : false, it can use the m_data[m_data_cnt] etc. data or update them,
556 : just needs to bump m_data_cnt by the same amount as when it was
557 : called with m_first set. The toplevel calls to
558 : handle_operand/handle_stmt should set m_data_cnt to 0 and truncate
559 : m_data vector when setting m_first to true.
560 :
561 : m_cast_conditional and m_bitfld_load are used when handling a
562 : bit-field load inside of a widening cast. handle_cast sometimes
563 : needs to do runtime comparisons and handle_operand only conditionally
564 : or even in two separate conditional blocks for one idx (once with
565 : constant index after comparing the runtime one for equality with the
566 : constant). In these cases, m_cast_conditional is set to true and
567 : the bit-field load then communicates its m_data_cnt to handle_cast
568 : using m_bitfld_load. */
569 : bool m_first;
570 : bool m_var_msb;
571 : unsigned m_upwards_2limb;
572 : bool m_upwards;
573 : bool m_cast_conditional;
574 : unsigned m_bitfld_load;
575 : vec<tree> m_data;
576 : unsigned int m_data_cnt;
577 : vec<gimple *> m_returns_twice_calls;
578 : };
579 :
580 7366 : bitint_large_huge::~bitint_large_huge ()
581 : {
582 7366 : BITMAP_FREE (m_names);
583 7366 : BITMAP_FREE (m_loads);
584 7366 : BITMAP_FREE (m_preserved);
585 7366 : BITMAP_FREE (m_single_use_names);
586 7366 : if (m_map)
587 5691 : delete_var_map (m_map);
588 7366 : XDELETEVEC (m_vars);
589 7366 : m_data.release ();
590 7366 : m_returns_twice_calls.release ();
591 7366 : }
592 :
593 : /* Insert gimple statement G before current location
594 : and set its gimple_location. */
595 :
596 : void
597 364137 : bitint_large_huge::insert_before (gimple *g)
598 : {
599 364137 : gimple_set_location (g, m_loc);
600 364137 : gsi_insert_before (&m_gsi, g, GSI_SAME_STMT);
601 364137 : }
602 :
603 : /* Return type for accessing limb IDX of BITINT_TYPE TYPE.
604 : This is normally m_limb_type, except for a partial most
605 : significant limb if any. */
606 :
607 : tree
608 130868 : bitint_large_huge::limb_access_type (tree type, tree idx)
609 : {
610 130868 : if (type == NULL_TREE)
611 5913 : return m_limb_type;
612 124955 : unsigned HOST_WIDE_INT i = tree_to_uhwi (idx);
613 124955 : unsigned int prec = TYPE_PRECISION (type);
614 124955 : gcc_assert (i * limb_prec < prec
615 : || (bitint_extended == bitint_ext_full
616 : && abi_limb_prec > limb_prec
617 : && i * limb_prec
618 : < CEIL (prec, abi_limb_prec) * abi_limb_prec));
619 249910 : if (bitint_big_endian
620 124955 : ? (i != 0 || (prec % limb_prec) == 0)
621 124955 : : (i + 1) * limb_prec <= prec)
622 81548 : return m_limb_type;
623 : else
624 86814 : return build_nonstandard_integer_type (prec % limb_prec,
625 43407 : TYPE_UNSIGNED (type));
626 : }
627 :
628 : /* Return a tree how to access limb IDX of VAR corresponding to BITINT_TYPE
629 : TYPE. If WRITE_P is true, it will be a store, otherwise a read. */
630 :
631 : tree
632 156189 : bitint_large_huge::limb_access (tree type, tree var, tree idx, bool write_p,
633 : bool abi_load_p)
634 : {
635 156189 : tree atype = (tree_fits_uhwi_p (idx)
636 156189 : ? limb_access_type (type, idx) : m_limb_type);
637 :
638 156189 : tree ltype = (bitint_extended && abi_load_p) ? atype : m_limb_type;
639 :
640 156189 : addr_space_t as = TYPE_ADDR_SPACE (TREE_TYPE (var));
641 156189 : tree ret;
642 156189 : if (DECL_P (var) && tree_fits_uhwi_p (idx))
643 : {
644 96025 : if (as != TYPE_ADDR_SPACE (ltype))
645 0 : ltype = build_qualified_type (ltype, TYPE_QUALS (ltype)
646 0 : | ENCODE_QUAL_ADDR_SPACE (as));
647 96025 : tree ptype = build_pointer_type (strip_array_types (TREE_TYPE (var)));
648 96025 : unsigned HOST_WIDE_INT off = tree_to_uhwi (idx) * m_limb_size;
649 96025 : if (bitint_big_endian)
650 0 : off += m_limb_size - tree_to_uhwi (TYPE_SIZE_UNIT (ltype));
651 96025 : ret = build2 (MEM_REF, ltype,
652 : build_fold_addr_expr (var),
653 96025 : build_int_cst (ptype, off));
654 96025 : TREE_THIS_VOLATILE (ret) = TREE_THIS_VOLATILE (var);
655 96025 : TREE_SIDE_EFFECTS (ret) = TREE_SIDE_EFFECTS (var);
656 96025 : }
657 60164 : else if (TREE_CODE (var) == MEM_REF && tree_fits_uhwi_p (idx))
658 : {
659 4316 : if (as != TYPE_ADDR_SPACE (ltype))
660 0 : ltype = build_qualified_type (ltype, TYPE_QUALS (ltype)
661 0 : | ENCODE_QUAL_ADDR_SPACE (as));
662 4316 : unsigned HOST_WIDE_INT off = tree_to_uhwi (idx) * m_limb_size;
663 4316 : if (bitint_big_endian)
664 0 : off += m_limb_size - tree_to_uhwi (TYPE_SIZE_UNIT (ltype));
665 4316 : ret
666 8632 : = build2 (MEM_REF, ltype, unshare_expr (TREE_OPERAND (var, 0)),
667 8632 : size_binop (PLUS_EXPR, TREE_OPERAND (var, 1),
668 : build_int_cst (TREE_TYPE (TREE_OPERAND (var, 1)),
669 : off)));
670 4316 : TREE_THIS_VOLATILE (ret) = TREE_THIS_VOLATILE (var);
671 4316 : TREE_SIDE_EFFECTS (ret) = TREE_SIDE_EFFECTS (var);
672 4316 : TREE_THIS_NOTRAP (ret) = TREE_THIS_NOTRAP (var);
673 4316 : }
674 : else
675 : {
676 55848 : ltype = m_limb_type;
677 55848 : if (as != TYPE_ADDR_SPACE (ltype))
678 17 : ltype = build_qualified_type (ltype, TYPE_QUALS (ltype)
679 17 : | ENCODE_QUAL_ADDR_SPACE (as));
680 55848 : var = unshare_expr (var);
681 55848 : if (TREE_CODE (TREE_TYPE (var)) != ARRAY_TYPE
682 83858 : || !useless_type_conversion_p (m_limb_type,
683 28010 : TREE_TYPE (TREE_TYPE (var))))
684 : {
685 28780 : unsigned HOST_WIDE_INT nelts
686 28780 : = CEIL (tree_to_uhwi (TYPE_SIZE (TREE_TYPE (var))), limb_prec);
687 28780 : tree atype = build_array_type_nelts (ltype, nelts);
688 28780 : var = build1 (VIEW_CONVERT_EXPR, atype, var);
689 : }
690 55848 : ret = build4 (ARRAY_REF, ltype, var, idx, NULL_TREE, NULL_TREE);
691 : }
692 156189 : if (!write_p && !useless_type_conversion_p (atype, ltype))
693 : {
694 19040 : gimple *g = gimple_build_assign (make_ssa_name (m_limb_type), ret);
695 19040 : insert_before (g);
696 19040 : ret = gimple_assign_lhs (g);
697 19040 : ret = build1 (NOP_EXPR, atype, ret);
698 : }
699 156189 : return ret;
700 : }
701 :
702 : /* Build a BIT_FIELD_REF to access BITSIZE bits with FTYPE type at
703 : offset BITPOS inside of OBJ. */
704 :
705 : tree
706 273 : bitint_large_huge::build_bit_field_ref (tree ftype, tree obj,
707 : unsigned HOST_WIDE_INT bitsize,
708 : unsigned HOST_WIDE_INT bitpos)
709 : {
710 546 : if (INTEGRAL_TYPE_P (TREE_TYPE (obj))
711 282 : && !type_has_mode_precision_p (TREE_TYPE (obj)))
712 : {
713 9 : unsigned HOST_WIDE_INT nelts
714 9 : = CEIL (tree_to_uhwi (TYPE_SIZE (TREE_TYPE (obj))), limb_prec);
715 9 : tree ltype = m_limb_type;
716 9 : addr_space_t as = TYPE_ADDR_SPACE (TREE_TYPE (obj));
717 9 : if (as != TYPE_ADDR_SPACE (ltype))
718 0 : ltype = build_qualified_type (ltype, TYPE_QUALS (ltype)
719 0 : | ENCODE_QUAL_ADDR_SPACE (as));
720 9 : tree atype = build_array_type_nelts (ltype, nelts);
721 9 : obj = build1 (VIEW_CONVERT_EXPR, atype, obj);
722 : }
723 273 : return build3 (BIT_FIELD_REF, ftype, obj, bitsize_int (bitsize),
724 273 : bitsize_int (bitpos));
725 : }
726 :
727 : /* Emit a half diamond,
728 : if (COND)
729 : |\
730 : | \
731 : | \
732 : | new_bb1
733 : | /
734 : | /
735 : |/
736 : or if (COND) new_bb1;
737 : PROB is the probability that the condition is true.
738 : Updates m_gsi to start of new_bb1.
739 : Sets EDGE_TRUE to edge from new_bb1 to successor and
740 : EDGE_FALSE to the EDGE_FALSE_VALUE edge from if (COND) bb. */
741 :
742 : void
743 4318 : bitint_large_huge::if_then (gimple *cond, profile_probability prob,
744 : edge &edge_true, edge &edge_false)
745 : {
746 4318 : insert_before (cond);
747 4318 : edge e1 = split_block (gsi_bb (m_gsi), cond);
748 4318 : edge e2 = split_block (e1->dest, (gimple *) NULL);
749 4318 : edge e3 = make_edge (e1->src, e2->dest, EDGE_FALSE_VALUE);
750 4318 : e1->flags = EDGE_TRUE_VALUE;
751 4318 : e1->probability = prob;
752 4318 : e3->probability = prob.invert ();
753 4318 : set_immediate_dominator (CDI_DOMINATORS, e2->dest, e1->src);
754 4318 : edge_true = e2;
755 4318 : edge_false = e3;
756 4318 : m_gsi = gsi_after_labels (e1->dest);
757 4318 : }
758 :
759 : /* Emit a full diamond,
760 : if (COND)
761 : /\
762 : / \
763 : / \
764 : new_bb1 new_bb2
765 : \ /
766 : \ /
767 : \/
768 : or if (COND) new_bb2; else new_bb1;
769 : PROB is the probability that the condition is true.
770 : Updates m_gsi to start of new_bb2.
771 : Sets EDGE_TRUE to edge from new_bb1 to successor and
772 : EDGE_FALSE to the EDGE_FALSE_VALUE edge from if (COND) bb. */
773 :
774 : void
775 112 : bitint_large_huge::if_then_else (gimple *cond, profile_probability prob,
776 : edge &edge_true, edge &edge_false)
777 : {
778 112 : insert_before (cond);
779 112 : edge e1 = split_block (gsi_bb (m_gsi), cond);
780 112 : edge e2 = split_block (e1->dest, (gimple *) NULL);
781 112 : basic_block bb = create_empty_bb (e1->dest);
782 112 : add_bb_to_loop (bb, e1->dest->loop_father);
783 112 : edge e3 = make_edge (e1->src, bb, EDGE_TRUE_VALUE);
784 112 : e1->flags = EDGE_FALSE_VALUE;
785 112 : e3->probability = prob;
786 112 : e1->probability = prob.invert ();
787 112 : bb->count = e1->src->count.apply_probability (prob);
788 112 : set_immediate_dominator (CDI_DOMINATORS, bb, e1->src);
789 112 : set_immediate_dominator (CDI_DOMINATORS, e2->dest, e1->src);
790 112 : edge_true = make_single_succ_edge (bb, e2->dest, EDGE_FALLTHRU);
791 112 : edge_false = e2;
792 112 : m_gsi = gsi_after_labels (bb);
793 112 : }
794 :
795 : /* Emit a half diamond with full diamond in it
796 : if (COND1)
797 : |\
798 : | \
799 : | \
800 : | if (COND2)
801 : | / \
802 : | / \
803 : |new_bb1 new_bb2
804 : | | /
805 : \ | /
806 : \ | /
807 : \ | /
808 : \|/
809 : or if (COND1) { if (COND2) new_bb2; else new_bb1; }
810 : PROB1 is the probability that the condition 1 is true.
811 : PROB2 is the probability that the condition 2 is true.
812 : Updates m_gsi to start of new_bb1.
813 : Sets EDGE_TRUE_TRUE to edge from new_bb2 to successor,
814 : EDGE_TRUE_FALSE to edge from new_bb1 to successor and
815 : EDGE_FALSE to the EDGE_FALSE_VALUE edge from if (COND1) bb.
816 : If COND2 is NULL, this is equivalent to
817 : if_then (COND1, PROB1, EDGE_TRUE_FALSE, EDGE_FALSE);
818 : EDGE_TRUE_TRUE = NULL; */
819 :
820 : void
821 2098 : bitint_large_huge::if_then_if_then_else (gimple *cond1, gimple *cond2,
822 : profile_probability prob1,
823 : profile_probability prob2,
824 : edge &edge_true_true,
825 : edge &edge_true_false,
826 : edge &edge_false)
827 : {
828 2098 : edge e2, e3, e4 = NULL;
829 2098 : if_then (cond1, prob1, e2, e3);
830 2098 : if (cond2 == NULL)
831 : {
832 1257 : edge_true_true = NULL;
833 1257 : edge_true_false = e2;
834 1257 : edge_false = e3;
835 1257 : return;
836 : }
837 841 : insert_before (cond2);
838 841 : e2 = split_block (gsi_bb (m_gsi), cond2);
839 841 : basic_block bb = create_empty_bb (e2->dest);
840 841 : add_bb_to_loop (bb, e2->dest->loop_father);
841 841 : e4 = make_edge (e2->src, bb, EDGE_TRUE_VALUE);
842 841 : set_immediate_dominator (CDI_DOMINATORS, bb, e2->src);
843 841 : e4->probability = prob2;
844 841 : e2->flags = EDGE_FALSE_VALUE;
845 841 : e2->probability = prob2.invert ();
846 841 : bb->count = e2->src->count.apply_probability (prob2);
847 841 : e4 = make_single_succ_edge (bb, e3->dest, EDGE_FALLTHRU);
848 841 : e2 = find_edge (e2->dest, e3->dest);
849 841 : edge_true_true = e4;
850 841 : edge_true_false = e2;
851 841 : edge_false = e3;
852 841 : m_gsi = gsi_after_labels (e2->src);
853 : }
854 :
855 : /* Emit code to access limb IDX from OP. */
856 :
857 : tree
858 111511 : bitint_large_huge::handle_operand (tree op, tree idx)
859 : {
860 111511 : switch (TREE_CODE (op))
861 : {
862 77073 : case SSA_NAME:
863 77073 : if (m_names == NULL
864 77073 : || !bitmap_bit_p (m_names, SSA_NAME_VERSION (op)))
865 : {
866 14521 : if (SSA_NAME_IS_DEFAULT_DEF (op))
867 : {
868 47 : if (m_first)
869 : {
870 16 : tree v = create_tmp_reg (m_limb_type);
871 16 : if (SSA_NAME_VAR (op) && VAR_P (SSA_NAME_VAR (op)))
872 : {
873 16 : DECL_NAME (v) = DECL_NAME (SSA_NAME_VAR (op));
874 16 : DECL_SOURCE_LOCATION (v)
875 16 : = DECL_SOURCE_LOCATION (SSA_NAME_VAR (op));
876 : }
877 16 : v = get_or_create_ssa_default_def (cfun, v);
878 16 : m_data.safe_push (v);
879 : }
880 47 : tree ret = m_data[m_data_cnt];
881 47 : m_data_cnt++;
882 47 : if (tree_fits_uhwi_p (idx))
883 : {
884 45 : tree type = limb_access_type (TREE_TYPE (op), idx);
885 45 : ret = add_cast (type, ret);
886 : }
887 : return ret;
888 : }
889 14474 : location_t loc_save = m_loc;
890 14474 : m_loc = gimple_location (SSA_NAME_DEF_STMT (op));
891 14474 : tree ret = handle_stmt (SSA_NAME_DEF_STMT (op), idx);
892 14474 : m_loc = loc_save;
893 14474 : return ret;
894 : }
895 62552 : int p;
896 62552 : gimple *g;
897 62552 : tree t;
898 62552 : p = var_to_partition (m_map, op);
899 62552 : gcc_assert (m_vars[p] != NULL_TREE);
900 62552 : t = limb_access (TREE_TYPE (op), m_vars[p], idx, false);
901 62552 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (t)), t);
902 62552 : insert_before (g);
903 62552 : t = gimple_assign_lhs (g);
904 62552 : if (m_first
905 22645 : && m_single_use_names
906 21773 : && m_vars[p] != m_lhs
907 21673 : && m_after_stmt
908 71318 : && bitmap_bit_p (m_single_use_names, SSA_NAME_VERSION (op)))
909 : {
910 8450 : tree clobber = build_clobber (TREE_TYPE (m_vars[p]),
911 : CLOBBER_STORAGE_END);
912 8450 : g = gimple_build_assign (m_vars[p], clobber);
913 8450 : gimple_stmt_iterator gsi = gsi_for_stmt (m_after_stmt);
914 8450 : gsi_insert_after (&gsi, g, GSI_SAME_STMT);
915 : }
916 : return t;
917 34438 : case INTEGER_CST:
918 34438 : if (tree_fits_uhwi_p (idx))
919 : {
920 23839 : tree c, type = limb_access_type (TREE_TYPE (op), idx);
921 23839 : unsigned HOST_WIDE_INT i = tree_to_uhwi (idx);
922 23839 : if (m_first)
923 : {
924 6214 : m_data.safe_push (NULL_TREE);
925 6214 : m_data.safe_push (NULL_TREE);
926 : }
927 23839 : if (bitint_big_endian)
928 0 : i = CEIL (TYPE_PRECISION (TREE_TYPE (op)), limb_prec) - 1 - i;
929 23839 : if (limb_prec != HOST_BITS_PER_WIDE_INT)
930 : {
931 0 : wide_int w = wi::rshift (wi::to_wide (op), i * limb_prec,
932 0 : TYPE_SIGN (TREE_TYPE (op)));
933 0 : c = wide_int_to_tree (type,
934 0 : wide_int::from (w, TYPE_PRECISION (type),
935 : UNSIGNED));
936 0 : }
937 23839 : else if (i >= TREE_INT_CST_EXT_NUNITS (op))
938 7511 : c = build_int_cst (type,
939 13334 : tree_int_cst_sgn (op) < 0 ? -1 : 0);
940 : else
941 16328 : c = build_int_cst (type, TREE_INT_CST_ELT (op, i));
942 23839 : m_data_cnt += 2;
943 23839 : return c;
944 : }
945 10599 : if (m_first
946 10599 : || (m_data[m_data_cnt] == NULL_TREE
947 159 : && m_data[m_data_cnt + 1] == NULL_TREE))
948 : {
949 5345 : unsigned int prec = TYPE_PRECISION (TREE_TYPE (op));
950 5345 : unsigned int rem = prec % ((m_upwards_2limb ? 2 : 1) * limb_prec);
951 5345 : int ext;
952 5345 : unsigned min_prec = bitint_min_cst_precision (op, ext);
953 5345 : if (m_first)
954 : {
955 5186 : m_data.safe_push (NULL_TREE);
956 5186 : m_data.safe_push (NULL_TREE);
957 : }
958 5345 : if (integer_zerop (op))
959 : {
960 837 : tree c = build_zero_cst (m_limb_type);
961 837 : m_data[m_data_cnt] = c;
962 837 : m_data[m_data_cnt + 1] = c;
963 : }
964 4508 : else if (integer_all_onesp (op))
965 : {
966 679 : tree c = build_all_ones_cst (m_limb_type);
967 679 : m_data[m_data_cnt] = c;
968 679 : m_data[m_data_cnt + 1] = c;
969 : }
970 3829 : else if (m_upwards_2limb && min_prec <= (unsigned) limb_prec)
971 : {
972 : /* Single limb constant. Use a phi with that limb from
973 : the preheader edge and 0 or -1 constant from the other edge
974 : and for the second limb in the loop. */
975 895 : tree out;
976 895 : gcc_assert (m_first);
977 895 : m_data.pop ();
978 895 : m_data.pop ();
979 895 : prepare_data_in_out (fold_convert (m_limb_type, op), idx, &out,
980 895 : build_int_cst (m_limb_type, ext));
981 895 : }
982 2934 : else if (min_prec > prec - rem - 2 * limb_prec)
983 : {
984 : /* Constant which has enough significant bits that it isn't
985 : worth trying to save .rodata space by extending from smaller
986 : number. */
987 2415 : tree type;
988 2415 : if (m_var_msb)
989 25 : type = TREE_TYPE (op);
990 : else
991 : /* If we have a guarantee the most significant partial limb
992 : (if any) will be only accessed through handle_operand
993 : with INTEGER_CST idx, we don't need to include the partial
994 : limb in .rodata. */
995 2390 : type = build_bitint_type (prec - rem, 1);
996 2415 : tree c = tree_output_constant_def (fold_convert (type, op));
997 2415 : m_data[m_data_cnt] = c;
998 2415 : m_data[m_data_cnt + 1] = NULL_TREE;
999 : }
1000 519 : else if (m_upwards_2limb)
1001 : {
1002 : /* Constant with smaller number of bits. Trade conditional
1003 : code for .rodata space by extending from smaller number. */
1004 444 : min_prec = CEIL (min_prec, 2 * limb_prec) * (2 * limb_prec);
1005 444 : tree type = build_bitint_type (min_prec, 1);
1006 444 : tree c = tree_output_constant_def (fold_convert (type, op));
1007 444 : tree ridx = idx;
1008 444 : if (bitint_big_endian)
1009 : {
1010 0 : ridx = make_ssa_name (sizetype);
1011 0 : g = gimple_build_assign (ridx, PLUS_EXPR, idx,
1012 0 : size_int (min_prec / limb_prec
1013 : - ((HOST_WIDE_INT)
1014 : CEIL (prec,
1015 : limb_prec))));
1016 0 : insert_before (g);
1017 : }
1018 444 : tree ridx2 = make_ssa_name (sizetype);
1019 444 : g = gimple_build_assign (ridx2, PLUS_EXPR, ridx,
1020 : bitint_big_endian
1021 0 : ? size_int (-1) : size_one_node);
1022 444 : insert_before (g);
1023 444 : if (bitint_big_endian)
1024 0 : g = gimple_build_cond (GE_EXPR, idx,
1025 0 : size_int (CEIL (prec, limb_prec)
1026 : - min_prec / limb_prec),
1027 : NULL_TREE, NULL_TREE);
1028 : else
1029 444 : g = gimple_build_cond (LT_EXPR, idx,
1030 444 : size_int (min_prec / limb_prec),
1031 : NULL_TREE, NULL_TREE);
1032 444 : edge edge_true, edge_false;
1033 888 : if_then (g, (min_prec >= (prec - rem) / 2
1034 312 : ? profile_probability::likely ()
1035 132 : : profile_probability::unlikely ()),
1036 : edge_true, edge_false);
1037 444 : tree c1 = limb_access (TREE_TYPE (op), c, ridx, false);
1038 444 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (c1)), c1);
1039 444 : insert_before (g);
1040 444 : c1 = gimple_assign_lhs (g);
1041 444 : tree c2 = limb_access (TREE_TYPE (op), c, ridx2, false);
1042 444 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (c2)), c2);
1043 444 : insert_before (g);
1044 444 : c2 = gimple_assign_lhs (g);
1045 444 : tree c3 = build_int_cst (m_limb_type, ext);
1046 444 : m_gsi = gsi_after_labels (edge_true->dest);
1047 444 : m_data[m_data_cnt] = make_ssa_name (m_limb_type);
1048 444 : m_data[m_data_cnt + 1] = make_ssa_name (m_limb_type);
1049 444 : gphi *phi = create_phi_node (m_data[m_data_cnt],
1050 : edge_true->dest);
1051 444 : add_phi_arg (phi, c1, edge_true, UNKNOWN_LOCATION);
1052 444 : add_phi_arg (phi, c3, edge_false, UNKNOWN_LOCATION);
1053 444 : phi = create_phi_node (m_data[m_data_cnt + 1], edge_true->dest);
1054 444 : add_phi_arg (phi, c2, edge_true, UNKNOWN_LOCATION);
1055 444 : add_phi_arg (phi, c3, edge_false, UNKNOWN_LOCATION);
1056 : }
1057 : else
1058 : {
1059 : /* Constant with smaller number of bits. Trade conditional
1060 : code for .rodata space by extending from smaller number.
1061 : Version for loops with random access to the limbs or
1062 : downwards loops. */
1063 75 : min_prec = CEIL (min_prec, limb_prec) * limb_prec;
1064 75 : tree c;
1065 75 : if (min_prec <= (unsigned) limb_prec)
1066 21 : c = fold_convert (m_limb_type, op);
1067 : else
1068 : {
1069 54 : tree type = build_bitint_type (min_prec, 1);
1070 54 : c = tree_output_constant_def (fold_convert (type, op));
1071 : }
1072 75 : m_data[m_data_cnt] = c;
1073 75 : m_data[m_data_cnt + 1] = integer_type_node;
1074 : }
1075 5345 : t = m_data[m_data_cnt];
1076 : }
1077 : else
1078 5254 : t = m_data[m_data_cnt + 1];
1079 10599 : if (m_data[m_data_cnt + 1] == NULL_TREE)
1080 : {
1081 4774 : tree ridx = idx;
1082 4774 : unsigned int prec = TYPE_PRECISION (TREE_TYPE (op));
1083 4774 : tree c = m_data[m_data_cnt];
1084 4774 : unsigned int min_prec = TYPE_PRECISION (TREE_TYPE (c));
1085 4774 : if (bitint_big_endian
1086 0 : && CEIL (min_prec, limb_prec) != CEIL (prec, limb_prec))
1087 : {
1088 0 : ridx = make_ssa_name (sizetype);
1089 0 : g = gimple_build_assign (ridx, PLUS_EXPR, idx,
1090 0 : size_int (CEIL (min_prec, limb_prec)
1091 : - ((HOST_WIDE_INT)
1092 : CEIL (prec, limb_prec))));
1093 0 : insert_before (g);
1094 : }
1095 4774 : t = limb_access (TREE_TYPE (op), c, ridx, false);
1096 4774 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (t)), t);
1097 4774 : insert_before (g);
1098 4774 : t = gimple_assign_lhs (g);
1099 : }
1100 5825 : else if (m_data[m_data_cnt + 1] == integer_type_node)
1101 : {
1102 115 : unsigned int prec = TYPE_PRECISION (TREE_TYPE (op));
1103 115 : unsigned rem = prec % ((m_upwards_2limb ? 2 : 1) * limb_prec);
1104 115 : int ext = wi::neg_p (wi::to_wide (op)) ? -1 : 0;
1105 115 : tree c = m_data[m_data_cnt];
1106 115 : unsigned min_prec = TYPE_PRECISION (TREE_TYPE (c));
1107 115 : if (bitint_big_endian)
1108 0 : g = gimple_build_cond (GE_EXPR, idx,
1109 0 : size_int (CEIL (prec, limb_prec)
1110 : - min_prec / limb_prec),
1111 : NULL_TREE, NULL_TREE);
1112 : else
1113 115 : g = gimple_build_cond (LT_EXPR, idx,
1114 115 : size_int (min_prec / limb_prec),
1115 : NULL_TREE, NULL_TREE);
1116 115 : edge edge_true, edge_false;
1117 230 : if_then (g, (min_prec >= (prec - rem) / 2
1118 29 : ? profile_probability::likely ()
1119 86 : : profile_probability::unlikely ()),
1120 : edge_true, edge_false);
1121 115 : if (min_prec > (unsigned) limb_prec)
1122 : {
1123 70 : tree ridx = idx;
1124 70 : if (bitint_big_endian)
1125 : {
1126 0 : ridx = make_ssa_name (sizetype);
1127 0 : g = gimple_build_assign (ridx, PLUS_EXPR, idx,
1128 0 : size_int (min_prec / limb_prec
1129 : - ((HOST_WIDE_INT)
1130 : CEIL (prec,
1131 : limb_prec))));
1132 0 : insert_before (g);
1133 : }
1134 70 : c = limb_access (TREE_TYPE (op), c, ridx, false);
1135 70 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (c)), c);
1136 70 : insert_before (g);
1137 70 : c = gimple_assign_lhs (g);
1138 : }
1139 115 : tree c2 = build_int_cst (m_limb_type, ext);
1140 115 : m_gsi = gsi_after_labels (edge_true->dest);
1141 115 : t = make_ssa_name (m_limb_type);
1142 115 : gphi *phi = create_phi_node (t, edge_true->dest);
1143 115 : add_phi_arg (phi, c, edge_true, UNKNOWN_LOCATION);
1144 115 : add_phi_arg (phi, c2, edge_false, UNKNOWN_LOCATION);
1145 : }
1146 10599 : m_data_cnt += 2;
1147 10599 : return t;
1148 0 : default:
1149 0 : gcc_unreachable ();
1150 : }
1151 : }
1152 :
1153 : /* Helper method, add a PHI node with VAL from preheader edge if
1154 : inside of a loop and m_first. Keep state in a pair of m_data
1155 : elements. If VAL_OUT is non-NULL, use that as PHI argument from
1156 : the latch edge, otherwise create a new SSA_NAME for it and let
1157 : caller initialize it. */
1158 :
1159 : tree
1160 15859 : bitint_large_huge::prepare_data_in_out (tree val, tree idx, tree *data_out,
1161 : tree val_out)
1162 : {
1163 15859 : if (!m_first)
1164 : {
1165 9430 : *data_out = tree_fits_uhwi_p (idx) ? NULL_TREE : m_data[m_data_cnt + 1];
1166 9430 : return m_data[m_data_cnt];
1167 : }
1168 :
1169 6429 : *data_out = NULL_TREE;
1170 6429 : if (tree_fits_uhwi_p (idx))
1171 : {
1172 2006 : m_data.safe_push (val);
1173 2006 : m_data.safe_push (NULL_TREE);
1174 2006 : return val;
1175 : }
1176 :
1177 4423 : tree in = make_ssa_name (TREE_TYPE (val));
1178 4423 : gphi *phi = create_phi_node (in, m_bb);
1179 4423 : edge e1 = find_edge (m_preheader_bb, m_bb);
1180 4423 : edge e2 = EDGE_PRED (m_bb, 0);
1181 4423 : if (e1 == e2)
1182 4423 : e2 = EDGE_PRED (m_bb, 1);
1183 4423 : add_phi_arg (phi, val, e1, UNKNOWN_LOCATION);
1184 4423 : tree out = val_out ? val_out : make_ssa_name (TREE_TYPE (val));
1185 4423 : add_phi_arg (phi, out, e2, UNKNOWN_LOCATION);
1186 4423 : m_data.safe_push (in);
1187 4423 : m_data.safe_push (out);
1188 4423 : return in;
1189 : }
1190 :
1191 : /* Return VAL cast to TYPE. If VAL is INTEGER_CST, just
1192 : convert it without emitting any code, otherwise emit
1193 : the conversion statement before the current location. */
1194 :
1195 : tree
1196 38746 : bitint_large_huge::add_cast (tree type, tree val)
1197 : {
1198 38746 : if (TREE_CODE (val) == INTEGER_CST)
1199 4577 : return fold_convert (type, val);
1200 :
1201 34169 : tree lhs = make_ssa_name (type);
1202 34169 : gimple *g = gimple_build_assign (lhs, NOP_EXPR, val);
1203 34169 : insert_before (g);
1204 34169 : return lhs;
1205 : }
1206 :
1207 : /* Helper of handle_stmt method, handle PLUS_EXPR or MINUS_EXPR. */
1208 :
1209 : tree
1210 13360 : bitint_large_huge::handle_plus_minus (tree_code code, tree rhs1, tree rhs2,
1211 : tree idx)
1212 : {
1213 13360 : tree lhs, data_out, ctype;
1214 13360 : tree rhs1_type = TREE_TYPE (rhs1);
1215 13360 : gimple *g;
1216 13360 : tree data_in = prepare_data_in_out (build_zero_cst (m_limb_type), idx,
1217 : &data_out);
1218 :
1219 19146 : if (optab_handler (code == PLUS_EXPR ? uaddc5_optab : usubc5_optab,
1220 13360 : TYPE_MODE (m_limb_type)) != CODE_FOR_nothing)
1221 : {
1222 13360 : ctype = build_complex_type (m_limb_type);
1223 13360 : if (!types_compatible_p (rhs1_type, m_limb_type))
1224 : {
1225 1073 : if (!TYPE_UNSIGNED (rhs1_type))
1226 : {
1227 360 : tree type = unsigned_type_for (rhs1_type);
1228 360 : rhs1 = add_cast (type, rhs1);
1229 360 : rhs2 = add_cast (type, rhs2);
1230 : }
1231 1073 : rhs1 = add_cast (m_limb_type, rhs1);
1232 1073 : rhs2 = add_cast (m_limb_type, rhs2);
1233 : }
1234 13360 : lhs = make_ssa_name (ctype);
1235 19146 : g = gimple_build_call_internal (code == PLUS_EXPR
1236 : ? IFN_UADDC : IFN_USUBC,
1237 : 3, rhs1, rhs2, data_in);
1238 13360 : gimple_call_set_lhs (g, lhs);
1239 13360 : insert_before (g);
1240 13360 : if (data_out == NULL_TREE)
1241 11141 : data_out = make_ssa_name (m_limb_type);
1242 13360 : g = gimple_build_assign (data_out, IMAGPART_EXPR,
1243 : build1 (IMAGPART_EXPR, m_limb_type, lhs));
1244 13360 : insert_before (g);
1245 : }
1246 0 : else if (types_compatible_p (rhs1_type, m_limb_type))
1247 : {
1248 0 : ctype = build_complex_type (m_limb_type);
1249 0 : lhs = make_ssa_name (ctype);
1250 0 : g = gimple_build_call_internal (code == PLUS_EXPR
1251 : ? IFN_ADD_OVERFLOW : IFN_SUB_OVERFLOW,
1252 : 2, rhs1, rhs2);
1253 0 : gimple_call_set_lhs (g, lhs);
1254 0 : insert_before (g);
1255 0 : if (data_out == NULL_TREE)
1256 0 : data_out = make_ssa_name (m_limb_type);
1257 0 : if (!integer_zerop (data_in))
1258 : {
1259 0 : rhs1 = make_ssa_name (m_limb_type);
1260 0 : g = gimple_build_assign (rhs1, REALPART_EXPR,
1261 : build1 (REALPART_EXPR, m_limb_type, lhs));
1262 0 : insert_before (g);
1263 0 : rhs2 = make_ssa_name (m_limb_type);
1264 0 : g = gimple_build_assign (rhs2, IMAGPART_EXPR,
1265 : build1 (IMAGPART_EXPR, m_limb_type, lhs));
1266 0 : insert_before (g);
1267 0 : lhs = make_ssa_name (ctype);
1268 0 : g = gimple_build_call_internal (code == PLUS_EXPR
1269 : ? IFN_ADD_OVERFLOW
1270 : : IFN_SUB_OVERFLOW,
1271 : 2, rhs1, data_in);
1272 0 : gimple_call_set_lhs (g, lhs);
1273 0 : insert_before (g);
1274 0 : data_in = make_ssa_name (m_limb_type);
1275 0 : g = gimple_build_assign (data_in, IMAGPART_EXPR,
1276 : build1 (IMAGPART_EXPR, m_limb_type, lhs));
1277 0 : insert_before (g);
1278 0 : g = gimple_build_assign (data_out, PLUS_EXPR, rhs2, data_in);
1279 0 : insert_before (g);
1280 : }
1281 : else
1282 : {
1283 0 : g = gimple_build_assign (data_out, IMAGPART_EXPR,
1284 : build1 (IMAGPART_EXPR, m_limb_type, lhs));
1285 0 : insert_before (g);
1286 : }
1287 : }
1288 : else
1289 : {
1290 : /* Always perform the two additions or subtractions in
1291 : unsigned type, avoid introducing a temporary UB. While
1292 : the end result of the 2 additions or 2 subtractions in
1293 : a valid program should not overflow, temporarily it can.
1294 : See PR126503. */
1295 0 : tree utype = unsigned_type_for (rhs1_type);
1296 0 : tree in = add_cast (utype, data_in);
1297 0 : lhs = make_ssa_name (utype);
1298 0 : if (utype != rhs1_type)
1299 : {
1300 0 : rhs1 = add_cast (utype, rhs1);
1301 0 : rhs2 = add_cast (utype, rhs2);
1302 : }
1303 0 : g = gimple_build_assign (lhs, code, rhs1, rhs2);
1304 0 : insert_before (g);
1305 0 : rhs1 = make_ssa_name (utype);
1306 0 : g = gimple_build_assign (rhs1, code, lhs, in);
1307 0 : insert_before (g);
1308 0 : m_data[m_data_cnt] = NULL_TREE;
1309 0 : m_data_cnt += 2;
1310 0 : return utype != rhs1_type ? add_cast (rhs1_type, rhs1) : rhs1;
1311 : }
1312 13360 : rhs1 = make_ssa_name (m_limb_type);
1313 13360 : g = gimple_build_assign (rhs1, REALPART_EXPR,
1314 : build1 (REALPART_EXPR, m_limb_type, lhs));
1315 13360 : insert_before (g);
1316 13360 : if (!types_compatible_p (rhs1_type, m_limb_type))
1317 1073 : rhs1 = add_cast (rhs1_type, rhs1);
1318 13360 : m_data[m_data_cnt] = data_out;
1319 13360 : m_data_cnt += 2;
1320 13360 : return rhs1;
1321 : }
1322 :
1323 : /* Helper function for handle_stmt method, handle LSHIFT_EXPR by
1324 : count in [0, limb_prec - 1] range. */
1325 :
1326 : tree
1327 152 : bitint_large_huge::handle_lshift (tree rhs1, tree rhs2, tree idx)
1328 : {
1329 152 : unsigned HOST_WIDE_INT cnt = tree_to_uhwi (rhs2);
1330 152 : gcc_checking_assert (cnt < (unsigned) limb_prec);
1331 152 : if (cnt == 0)
1332 : return rhs1;
1333 :
1334 152 : tree lhs, data_out, rhs1_type = TREE_TYPE (rhs1);
1335 152 : gimple *g;
1336 152 : tree data_in = prepare_data_in_out (build_zero_cst (m_limb_type), idx,
1337 : &data_out);
1338 :
1339 152 : if (!integer_zerop (data_in))
1340 : {
1341 136 : lhs = make_ssa_name (m_limb_type);
1342 136 : g = gimple_build_assign (lhs, RSHIFT_EXPR, data_in,
1343 : build_int_cst (unsigned_type_node,
1344 136 : limb_prec - cnt));
1345 136 : insert_before (g);
1346 136 : if (!types_compatible_p (rhs1_type, m_limb_type))
1347 35 : lhs = add_cast (rhs1_type, lhs);
1348 : data_in = lhs;
1349 : }
1350 152 : if (types_compatible_p (rhs1_type, m_limb_type))
1351 : {
1352 117 : if (data_out == NULL_TREE)
1353 82 : data_out = make_ssa_name (m_limb_type);
1354 117 : g = gimple_build_assign (data_out, rhs1);
1355 117 : insert_before (g);
1356 : }
1357 152 : if (cnt < (unsigned) TYPE_PRECISION (rhs1_type))
1358 : {
1359 137 : lhs = make_ssa_name (rhs1_type);
1360 137 : g = gimple_build_assign (lhs, LSHIFT_EXPR, rhs1, rhs2);
1361 137 : insert_before (g);
1362 137 : if (!integer_zerop (data_in))
1363 : {
1364 121 : rhs1 = lhs;
1365 121 : lhs = make_ssa_name (rhs1_type);
1366 121 : g = gimple_build_assign (lhs, BIT_IOR_EXPR, rhs1, data_in);
1367 121 : insert_before (g);
1368 : }
1369 : }
1370 : else
1371 : lhs = data_in;
1372 152 : m_data[m_data_cnt] = data_out;
1373 152 : m_data_cnt += 2;
1374 152 : return lhs;
1375 : }
1376 :
1377 : /* Helper function for handle_stmt method, handle an integral
1378 : to integral conversion. */
1379 :
1380 : tree
1381 7561 : bitint_large_huge::handle_cast (tree lhs_type, tree rhs1, tree idx)
1382 : {
1383 7561 : tree rhs_type = TREE_TYPE (rhs1);
1384 7561 : gimple *g;
1385 7561 : if ((TREE_CODE (rhs1) == SSA_NAME || TREE_CODE (rhs1) == INTEGER_CST)
1386 7561 : && BITINT_TYPE_P (lhs_type)
1387 7561 : && BITINT_TYPE_P (rhs_type)
1388 6568 : && bitint_precision_kind (lhs_type) >= bitint_prec_large
1389 14129 : && bitint_precision_kind (rhs_type) >= bitint_prec_large)
1390 : {
1391 5929 : if (TYPE_PRECISION (rhs_type) >= TYPE_PRECISION (lhs_type)
1392 : /* If lhs has bigger precision than rhs, we can use
1393 : the simple case only if there is a guarantee that
1394 : the most significant limb is handled in straight
1395 : line code. If m_var_msb (on left shifts) or
1396 : if m_upwards_2limb * limb_prec is equal to
1397 : lhs precision or if not m_upwards_2limb and lhs_type
1398 : has precision which is multiple of limb_prec that is
1399 : not the case. */
1400 5929 : || (!m_var_msb
1401 1524 : && (CEIL (TYPE_PRECISION (lhs_type), limb_prec)
1402 1524 : == CEIL (TYPE_PRECISION (rhs_type), limb_prec))
1403 370 : && ((!m_upwards_2limb
1404 182 : && (TYPE_PRECISION (lhs_type) % limb_prec != 0))
1405 267 : || (m_upwards_2limb
1406 376 : && (m_upwards_2limb * limb_prec
1407 188 : < TYPE_PRECISION (lhs_type))))))
1408 : {
1409 4670 : tree ridx = idx;
1410 4670 : if (bitint_big_endian
1411 4670 : && (CEIL (TYPE_PRECISION (lhs_type), limb_prec)
1412 0 : != CEIL (TYPE_PRECISION (rhs_type), limb_prec)))
1413 : {
1414 0 : HOST_WIDE_INT diff = CEIL (TYPE_PRECISION (rhs_type), limb_prec);
1415 0 : diff -= CEIL (TYPE_PRECISION (lhs_type), limb_prec);
1416 0 : if (tree_fits_uhwi_p (idx))
1417 0 : ridx = size_int (tree_to_uhwi (idx) + diff);
1418 : else
1419 : {
1420 0 : tree t = make_ssa_name (sizetype);
1421 0 : g = gimple_build_assign (t, PLUS_EXPR, idx, size_int (diff));
1422 0 : insert_before (g);
1423 0 : ridx = t;
1424 : }
1425 : }
1426 4670 : rhs1 = handle_operand (rhs1, ridx);
1427 4670 : if (tree_fits_uhwi_p (idx))
1428 : {
1429 2389 : tree type = limb_access_type (lhs_type, idx);
1430 2389 : if (!types_compatible_p (type, TREE_TYPE (rhs1)))
1431 1250 : rhs1 = add_cast (type, rhs1);
1432 : }
1433 : return rhs1;
1434 : }
1435 1259 : tree t;
1436 : /* Indexes lower than this don't need any special processing. */
1437 1259 : unsigned low = ((unsigned) TYPE_PRECISION (rhs_type)
1438 1259 : - !TYPE_UNSIGNED (rhs_type)) / limb_prec;
1439 : /* Indexes >= than this always contain an extension. */
1440 1259 : unsigned high = CEIL ((unsigned) TYPE_PRECISION (rhs_type), limb_prec);
1441 1259 : unsigned lcnt = CEIL ((unsigned) TYPE_PRECISION (lhs_type), limb_prec);
1442 1259 : unsigned lowe = bitint_big_endian ? lcnt - 1 - low : low;
1443 1259 : bool save_first = m_first;
1444 1259 : if (m_first)
1445 : {
1446 414 : m_data.safe_push (NULL_TREE);
1447 414 : m_data.safe_push (NULL_TREE);
1448 414 : m_data.safe_push (NULL_TREE);
1449 414 : if (TYPE_UNSIGNED (rhs_type))
1450 : /* No need to keep state between iterations. */
1451 : ;
1452 193 : else if (m_upwards && !m_upwards_2limb)
1453 : /* We need to keep state between iterations, but
1454 : not within any loop, everything is straight line
1455 : code with only increasing indexes. */
1456 : ;
1457 153 : else if (!m_upwards_2limb)
1458 : {
1459 3 : unsigned save_data_cnt = m_data_cnt;
1460 3 : gimple_stmt_iterator save_gsi = m_gsi;
1461 3 : m_gsi = m_init_gsi;
1462 3 : if (gsi_end_p (m_gsi))
1463 0 : m_gsi = gsi_after_labels (gsi_bb (m_gsi));
1464 : else
1465 3 : gsi_next (&m_gsi);
1466 3 : m_data_cnt = save_data_cnt + 3;
1467 3 : t = handle_operand (rhs1, size_int (bitint_big_endian
1468 : ? high - 1 - low : low));
1469 3 : m_first = false;
1470 3 : m_data[save_data_cnt + 2]
1471 3 : = build_int_cst (integer_type_node, m_data_cnt);
1472 3 : m_data_cnt = save_data_cnt;
1473 3 : t = add_cast (signed_type_for (m_limb_type), t);
1474 3 : tree lpm1 = build_int_cst (unsigned_type_node, limb_prec - 1);
1475 3 : tree n = make_ssa_name (TREE_TYPE (t));
1476 3 : g = gimple_build_assign (n, RSHIFT_EXPR, t, lpm1);
1477 3 : insert_before (g);
1478 3 : m_data[save_data_cnt + 1] = add_cast (m_limb_type, n);
1479 3 : m_init_gsi = m_gsi;
1480 3 : if (gsi_end_p (m_init_gsi))
1481 0 : m_init_gsi = gsi_last_bb (gsi_bb (m_init_gsi));
1482 : else
1483 3 : gsi_prev (&m_init_gsi);
1484 3 : m_gsi = save_gsi;
1485 : }
1486 150 : else if (m_upwards_2limb * limb_prec < TYPE_PRECISION (rhs_type))
1487 : /* We need to keep state between iterations, but
1488 : fortunately not within the loop, only afterwards. */
1489 : ;
1490 : else
1491 : {
1492 146 : tree out;
1493 146 : m_data.truncate (m_data_cnt);
1494 146 : prepare_data_in_out (build_zero_cst (m_limb_type), idx, &out);
1495 146 : m_data.safe_push (NULL_TREE);
1496 : }
1497 : }
1498 :
1499 1259 : unsigned save_data_cnt = m_data_cnt;
1500 1259 : m_data_cnt += 3;
1501 1259 : if (!tree_fits_uhwi_p (idx))
1502 : {
1503 688 : if (m_upwards_2limb
1504 668 : && low >= m_upwards_2limb - m_first)
1505 : {
1506 158 : if (bitint_big_endian
1507 158 : && (CEIL (TYPE_PRECISION (lhs_type), limb_prec)
1508 0 : != CEIL (TYPE_PRECISION (rhs_type), limb_prec)))
1509 : {
1510 0 : HOST_WIDE_INT diff
1511 0 : = CEIL (TYPE_PRECISION (rhs_type), limb_prec);
1512 0 : diff -= CEIL (TYPE_PRECISION (lhs_type), limb_prec);
1513 0 : tree t = make_ssa_name (sizetype);
1514 0 : g = gimple_build_assign (t, PLUS_EXPR, idx, size_int (diff));
1515 0 : insert_before (g);
1516 0 : idx = t;
1517 : }
1518 158 : rhs1 = handle_operand (rhs1, idx);
1519 158 : if (m_first)
1520 131 : m_data[save_data_cnt + 2]
1521 262 : = build_int_cst (integer_type_node, m_data_cnt);
1522 158 : m_first = save_first;
1523 158 : return rhs1;
1524 : }
1525 1326 : bool single_comparison
1526 530 : = low == high || (m_upwards_2limb && (low & 1) == m_first);
1527 266 : tree idxc = idx;
1528 266 : if (!single_comparison
1529 266 : && m_upwards_2limb
1530 246 : && !m_first
1531 112 : && low + 1 == m_upwards_2limb)
1532 : /* In this case we know that idx <= low always,
1533 : so effectively we just needs a single comparison,
1534 : idx < low or idx == low, but we'd need to emit different
1535 : code for the 2 branches than single_comparison normally
1536 : emits. So, instead of special-casing that, emit a
1537 : low <= low comparison which cfg cleanup will clean up
1538 : at the end of the pass. */
1539 89 : idxc = size_int (lowe);
1540 530 : if (bitint_big_endian)
1541 0 : g = gimple_build_cond (single_comparison ? GT_EXPR : GE_EXPR,
1542 0 : idxc, size_int (lowe),
1543 : NULL_TREE, NULL_TREE);
1544 : else
1545 796 : g = gimple_build_cond (single_comparison ? LT_EXPR : LE_EXPR,
1546 530 : idxc, size_int (low), NULL_TREE, NULL_TREE);
1547 530 : edge edge_true_true, edge_true_false, edge_false;
1548 796 : if_then_if_then_else (g, (single_comparison ? NULL
1549 266 : : gimple_build_cond (EQ_EXPR, idx,
1550 266 : size_int (lowe),
1551 : NULL_TREE,
1552 : NULL_TREE)),
1553 : profile_probability::likely (),
1554 : profile_probability::unlikely (),
1555 : edge_true_true, edge_true_false, edge_false);
1556 530 : bool save_cast_conditional = m_cast_conditional;
1557 530 : m_cast_conditional = true;
1558 530 : m_bitfld_load = 0;
1559 530 : tree t1 = idx, t2 = NULL_TREE;
1560 530 : if (bitint_big_endian
1561 530 : && (CEIL (TYPE_PRECISION (lhs_type), limb_prec)
1562 0 : != CEIL (TYPE_PRECISION (rhs_type), limb_prec)))
1563 : {
1564 0 : HOST_WIDE_INT diff = CEIL (TYPE_PRECISION (rhs_type), limb_prec);
1565 0 : diff -= CEIL (TYPE_PRECISION (lhs_type), limb_prec);
1566 0 : t1 = make_ssa_name (sizetype);
1567 0 : g = gimple_build_assign (t1, PLUS_EXPR, idx, size_int (diff));
1568 0 : insert_before (g);
1569 : }
1570 530 : t1 = handle_operand (rhs1, t1);
1571 530 : if (m_first)
1572 208 : m_data[save_data_cnt + 2]
1573 416 : = build_int_cst (integer_type_node, m_data_cnt);
1574 530 : tree ext = NULL_TREE;
1575 530 : tree bitfld = NULL_TREE;
1576 530 : if (!single_comparison)
1577 : {
1578 266 : m_gsi = gsi_after_labels (edge_true_true->src);
1579 266 : m_first = false;
1580 266 : m_data_cnt = save_data_cnt + 3;
1581 266 : if (m_bitfld_load)
1582 : {
1583 4 : bitfld = m_data[m_bitfld_load];
1584 4 : m_data[m_bitfld_load] = m_data[m_bitfld_load + 2];
1585 4 : m_bitfld_load = 0;
1586 : }
1587 266 : t2 = handle_operand (rhs1, size_int (bitint_big_endian
1588 : ? high - 1 - low : low));
1589 266 : if (!useless_type_conversion_p (m_limb_type, TREE_TYPE (t2)))
1590 221 : t2 = add_cast (m_limb_type, t2);
1591 266 : if (!TYPE_UNSIGNED (rhs_type) && m_upwards_2limb)
1592 : {
1593 146 : ext = add_cast (signed_type_for (m_limb_type), t2);
1594 292 : tree lpm1 = build_int_cst (unsigned_type_node,
1595 146 : limb_prec - 1);
1596 146 : tree n = make_ssa_name (TREE_TYPE (ext));
1597 146 : g = gimple_build_assign (n, RSHIFT_EXPR, ext, lpm1);
1598 146 : insert_before (g);
1599 146 : ext = add_cast (m_limb_type, n);
1600 : }
1601 : }
1602 530 : tree t3;
1603 530 : if (TYPE_UNSIGNED (rhs_type))
1604 278 : t3 = build_zero_cst (m_limb_type);
1605 252 : else if (m_upwards_2limb && (save_first || ext != NULL_TREE))
1606 168 : t3 = m_data[save_data_cnt];
1607 : else
1608 84 : t3 = m_data[save_data_cnt + 1];
1609 530 : m_gsi = gsi_after_labels (edge_true_false->dest);
1610 530 : t = make_ssa_name (m_limb_type);
1611 530 : gphi *phi = create_phi_node (t, edge_true_false->dest);
1612 530 : add_phi_arg (phi, t1, edge_true_false, UNKNOWN_LOCATION);
1613 530 : add_phi_arg (phi, t3, edge_false, UNKNOWN_LOCATION);
1614 530 : if (edge_true_true)
1615 266 : add_phi_arg (phi, t2, edge_true_true, UNKNOWN_LOCATION);
1616 530 : if (ext)
1617 : {
1618 146 : tree t4 = make_ssa_name (m_limb_type);
1619 146 : phi = create_phi_node (t4, edge_true_false->dest);
1620 146 : add_phi_arg (phi, build_zero_cst (m_limb_type), edge_true_false,
1621 : UNKNOWN_LOCATION);
1622 146 : add_phi_arg (phi, m_data[save_data_cnt], edge_false,
1623 : UNKNOWN_LOCATION);
1624 146 : add_phi_arg (phi, ext, edge_true_true, UNKNOWN_LOCATION);
1625 146 : if (!save_cast_conditional)
1626 : {
1627 136 : g = gimple_build_assign (m_data[save_data_cnt + 1], t4);
1628 136 : insert_before (g);
1629 : }
1630 : else
1631 10 : for (basic_block bb = gsi_bb (m_gsi);;)
1632 : {
1633 10 : edge e1 = single_succ_edge (bb);
1634 10 : edge e2 = find_edge (e1->dest, m_bb), e3;
1635 10 : tree t5 = (e2 ? m_data[save_data_cnt + 1]
1636 10 : : make_ssa_name (m_limb_type));
1637 10 : phi = create_phi_node (t5, e1->dest);
1638 10 : edge_iterator ei;
1639 30 : FOR_EACH_EDGE (e3, ei, e1->dest->preds)
1640 30 : add_phi_arg (phi, (e3 == e1 ? t4
1641 10 : : build_zero_cst (m_limb_type)),
1642 : e3, UNKNOWN_LOCATION);
1643 10 : if (e2)
1644 : break;
1645 0 : t4 = t5;
1646 0 : bb = e1->dest;
1647 0 : }
1648 : }
1649 530 : if (m_bitfld_load)
1650 : {
1651 8 : tree t4;
1652 8 : if (!save_first && !save_cast_conditional)
1653 2 : t4 = m_data[m_bitfld_load + 1];
1654 : else
1655 6 : t4 = make_ssa_name (m_limb_type);
1656 8 : phi = create_phi_node (t4, edge_true_false->dest);
1657 12 : add_phi_arg (phi,
1658 4 : edge_true_true ? bitfld : m_data[m_bitfld_load],
1659 : edge_true_false, UNKNOWN_LOCATION);
1660 8 : add_phi_arg (phi, m_data[m_bitfld_load + 2],
1661 : edge_false, UNKNOWN_LOCATION);
1662 8 : if (edge_true_true)
1663 4 : add_phi_arg (phi, m_data[m_bitfld_load], edge_true_true,
1664 : UNKNOWN_LOCATION);
1665 8 : if (save_cast_conditional)
1666 4 : for (basic_block bb = gsi_bb (m_gsi);;)
1667 : {
1668 4 : edge e1 = single_succ_edge (bb);
1669 4 : edge e2 = find_edge (e1->dest, m_bb), e3;
1670 4 : tree t5 = ((e2 && !save_first) ? m_data[m_bitfld_load + 1]
1671 4 : : make_ssa_name (m_limb_type));
1672 4 : phi = create_phi_node (t5, e1->dest);
1673 4 : edge_iterator ei;
1674 14 : FOR_EACH_EDGE (e3, ei, e1->dest->preds)
1675 16 : add_phi_arg (phi, (e3 == e1 ? t4
1676 6 : : build_zero_cst (m_limb_type)),
1677 : e3, UNKNOWN_LOCATION);
1678 4 : t4 = t5;
1679 4 : if (e2)
1680 : break;
1681 0 : bb = e1->dest;
1682 0 : }
1683 8 : m_data[m_bitfld_load] = t4;
1684 8 : m_data[m_bitfld_load + 2] = t4;
1685 8 : m_bitfld_load = 0;
1686 : }
1687 530 : m_cast_conditional = save_cast_conditional;
1688 530 : m_first = save_first;
1689 530 : return t;
1690 : }
1691 : else
1692 : {
1693 571 : unsigned tidx = tree_to_uhwi (idx);
1694 571 : if (bitint_big_endian)
1695 0 : tidx = lcnt - 1 - tidx;
1696 571 : if (tidx < low)
1697 : {
1698 152 : t = handle_operand (rhs1, (bitint_big_endian
1699 0 : ? size_int (high - 1 - tidx) : idx));
1700 152 : if (m_first)
1701 71 : m_data[save_data_cnt + 2]
1702 142 : = build_int_cst (integer_type_node, m_data_cnt);
1703 : }
1704 419 : else if (tidx < high)
1705 : {
1706 68 : t = handle_operand (rhs1, size_int (bitint_big_endian
1707 : ? high - 1 - low : low));
1708 68 : if (m_first)
1709 1 : m_data[save_data_cnt + 2]
1710 2 : = build_int_cst (integer_type_node, m_data_cnt);
1711 68 : if (!useless_type_conversion_p (m_limb_type, TREE_TYPE (t)))
1712 60 : t = add_cast (m_limb_type, t);
1713 68 : tree ext = NULL_TREE;
1714 68 : if (!TYPE_UNSIGNED (rhs_type) && m_upwards)
1715 : {
1716 44 : ext = add_cast (signed_type_for (m_limb_type), t);
1717 88 : tree lpm1 = build_int_cst (unsigned_type_node,
1718 44 : limb_prec - 1);
1719 44 : tree n = make_ssa_name (TREE_TYPE (ext));
1720 44 : g = gimple_build_assign (n, RSHIFT_EXPR, ext, lpm1);
1721 44 : insert_before (g);
1722 44 : ext = add_cast (m_limb_type, n);
1723 44 : m_data[save_data_cnt + 1] = ext;
1724 : }
1725 : }
1726 : else
1727 : {
1728 351 : if (TYPE_UNSIGNED (rhs_type) && m_first)
1729 : {
1730 0 : handle_operand (rhs1, (bitint_big_endian
1731 0 : ? size_int (high - 1)
1732 : : size_zero_node));
1733 0 : m_data[save_data_cnt + 2]
1734 0 : = build_int_cst (integer_type_node, m_data_cnt);
1735 : }
1736 : else
1737 351 : m_data_cnt = tree_to_uhwi (m_data[save_data_cnt + 2]);
1738 351 : if (TYPE_UNSIGNED (rhs_type))
1739 184 : t = build_zero_cst (m_limb_type);
1740 167 : else if (m_bb
1741 16 : && m_data[save_data_cnt]
1742 180 : && ((tidx & 1) == 0 || tidx != low + 1))
1743 : t = m_data[save_data_cnt];
1744 : else
1745 161 : t = m_data[save_data_cnt + 1];
1746 : }
1747 571 : tree type = limb_access_type (lhs_type, idx);
1748 571 : if (!useless_type_conversion_p (type, m_limb_type))
1749 292 : t = add_cast (type, t);
1750 571 : m_first = save_first;
1751 571 : return t;
1752 : }
1753 : }
1754 0 : else if (BITINT_TYPE_P (lhs_type)
1755 1632 : && bitint_precision_kind (lhs_type) >= bitint_prec_large
1756 3264 : && INTEGRAL_TYPE_P (rhs_type))
1757 : {
1758 : /* Add support for 3 or more limbs filled in from normal integral
1759 : type if this assert fails. If no target chooses limb mode smaller
1760 : than half of largest supported normal integral type, this will not
1761 : be needed. */
1762 1632 : gcc_assert (TYPE_PRECISION (rhs_type) <= 2 * limb_prec);
1763 1632 : tree r1 = NULL_TREE, r2 = NULL_TREE, rext = NULL_TREE;
1764 1632 : if (m_first)
1765 : {
1766 594 : gimple_stmt_iterator save_gsi = m_gsi;
1767 594 : m_gsi = m_init_gsi;
1768 594 : if (gsi_end_p (m_gsi))
1769 58 : m_gsi = gsi_after_labels (gsi_bb (m_gsi));
1770 : else
1771 536 : gsi_next (&m_gsi);
1772 375 : if (BITINT_TYPE_P (rhs_type)
1773 594 : && bitint_precision_kind (rhs_type) == bitint_prec_middle)
1774 : {
1775 63 : tree type = NULL_TREE;
1776 63 : rhs1 = maybe_cast_middle_bitint (&m_gsi, rhs1, type);
1777 63 : rhs_type = TREE_TYPE (rhs1);
1778 : }
1779 594 : r1 = rhs1;
1780 594 : if (!useless_type_conversion_p (m_limb_type, TREE_TYPE (rhs1)))
1781 526 : r1 = add_cast (m_limb_type, rhs1);
1782 594 : if (TYPE_PRECISION (rhs_type) > limb_prec)
1783 : {
1784 109 : g = gimple_build_assign (make_ssa_name (rhs_type),
1785 : RSHIFT_EXPR, rhs1,
1786 : build_int_cst (unsigned_type_node,
1787 109 : limb_prec));
1788 109 : insert_before (g);
1789 109 : r2 = add_cast (m_limb_type, gimple_assign_lhs (g));
1790 : }
1791 594 : if (TYPE_UNSIGNED (rhs_type))
1792 287 : rext = build_zero_cst (m_limb_type);
1793 : else
1794 : {
1795 307 : rext = add_cast (signed_type_for (m_limb_type), r2 ? r2 : r1);
1796 307 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (rext)),
1797 : RSHIFT_EXPR, rext,
1798 : build_int_cst (unsigned_type_node,
1799 307 : limb_prec - 1));
1800 307 : insert_before (g);
1801 307 : rext = add_cast (m_limb_type, gimple_assign_lhs (g));
1802 : }
1803 594 : m_init_gsi = m_gsi;
1804 594 : if (gsi_end_p (m_init_gsi))
1805 598 : m_init_gsi = gsi_last_bb (gsi_bb (m_init_gsi));
1806 : else
1807 295 : gsi_prev (&m_init_gsi);
1808 594 : m_gsi = save_gsi;
1809 : }
1810 1632 : tree t;
1811 1632 : if (m_upwards_2limb)
1812 : {
1813 776 : if (m_first)
1814 : {
1815 297 : tree out1, out2;
1816 297 : prepare_data_in_out (r1, idx, &out1, rext);
1817 297 : if (TYPE_PRECISION (rhs_type) > limb_prec)
1818 : {
1819 70 : prepare_data_in_out (r2, idx, &out2, rext);
1820 70 : m_data.pop ();
1821 70 : t = m_data.pop ();
1822 70 : m_data[m_data_cnt + 1] = t;
1823 : }
1824 : else
1825 227 : m_data[m_data_cnt + 1] = rext;
1826 297 : m_data.safe_push (rext);
1827 297 : t = m_data[m_data_cnt];
1828 : }
1829 479 : else if (!tree_fits_uhwi_p (idx))
1830 297 : t = m_data[m_data_cnt + 1];
1831 : else
1832 : {
1833 182 : tree type = limb_access_type (lhs_type, idx);
1834 182 : t = m_data[m_data_cnt + 2];
1835 182 : if (!useless_type_conversion_p (type, m_limb_type))
1836 153 : t = add_cast (type, t);
1837 : }
1838 776 : m_data_cnt += 3;
1839 776 : return t;
1840 : }
1841 856 : else if (m_first)
1842 : {
1843 297 : m_data.safe_push (r1);
1844 297 : m_data.safe_push (r2);
1845 297 : m_data.safe_push (rext);
1846 : }
1847 856 : unsigned lcnt = CEIL ((unsigned) TYPE_PRECISION (lhs_type), limb_prec);
1848 856 : if (tree_fits_uhwi_p (idx))
1849 : {
1850 812 : tree type = limb_access_type (lhs_type, idx);
1851 812 : if (bitint_big_endian
1852 812 : ? tree_to_uhwi (idx) == lcnt - 1 : integer_zerop (idx))
1853 269 : t = m_data[m_data_cnt];
1854 543 : else if (TYPE_PRECISION (rhs_type) > limb_prec
1855 543 : && (bitint_big_endian
1856 72 : ? tree_to_uhwi (idx) == lcnt - 2
1857 72 : : integer_onep (idx)))
1858 33 : t = m_data[m_data_cnt + 1];
1859 : else
1860 510 : t = m_data[m_data_cnt + 2];
1861 812 : if (!useless_type_conversion_p (type, m_limb_type))
1862 250 : t = add_cast (type, t);
1863 812 : m_data_cnt += 3;
1864 812 : return t;
1865 : }
1866 44 : g = gimple_build_cond (NE_EXPR, idx,
1867 : bitint_big_endian
1868 0 : ? size_int (lcnt - 1) : size_zero_node,
1869 : NULL_TREE, NULL_TREE);
1870 44 : edge e2, e3, e4 = NULL;
1871 44 : if_then (g, profile_probability::likely (), e2, e3);
1872 44 : if (m_data[m_data_cnt + 1])
1873 : {
1874 14 : g = gimple_build_cond (EQ_EXPR, idx,
1875 : bitint_big_endian
1876 0 : ? size_int (lcnt - 2) : size_one_node,
1877 : NULL_TREE, NULL_TREE);
1878 14 : insert_before (g);
1879 14 : edge e5 = split_block (gsi_bb (m_gsi), g);
1880 14 : e4 = make_edge (e5->src, e2->dest, EDGE_TRUE_VALUE);
1881 14 : e2 = find_edge (e5->dest, e2->dest);
1882 14 : e4->probability = profile_probability::unlikely ();
1883 14 : e5->flags = EDGE_FALSE_VALUE;
1884 14 : e5->probability = e4->probability.invert ();
1885 : }
1886 44 : m_gsi = gsi_after_labels (e2->dest);
1887 44 : t = make_ssa_name (m_limb_type);
1888 44 : gphi *phi = create_phi_node (t, e2->dest);
1889 44 : add_phi_arg (phi, m_data[m_data_cnt + 2], e2, UNKNOWN_LOCATION);
1890 44 : add_phi_arg (phi, m_data[m_data_cnt], e3, UNKNOWN_LOCATION);
1891 44 : if (e4)
1892 14 : add_phi_arg (phi, m_data[m_data_cnt + 1], e4, UNKNOWN_LOCATION);
1893 44 : m_data_cnt += 3;
1894 44 : return t;
1895 : }
1896 : return NULL_TREE;
1897 : }
1898 :
1899 : /* Helper function for handle_stmt method, handle a BIT_FIELD_REF. */
1900 :
1901 : tree
1902 23 : bitint_large_huge::handle_bit_field_ref (tree op, tree idx)
1903 : {
1904 23 : if (tree_fits_uhwi_p (idx))
1905 : {
1906 13 : if (m_first)
1907 4 : m_data.safe_push (NULL);
1908 13 : ++m_data_cnt;
1909 13 : unsigned HOST_WIDE_INT sz = tree_to_uhwi (TYPE_SIZE (m_limb_type));
1910 13 : unsigned i = tree_to_uhwi (idx);
1911 13 : if (bitint_big_endian)
1912 0 : i = CEIL (TYPE_PRECISION (TREE_TYPE (op)), limb_prec) - 1 - i;
1913 26 : tree bfr = build3 (BIT_FIELD_REF, m_limb_type,
1914 13 : TREE_OPERAND (op, 0),
1915 13 : TYPE_SIZE (m_limb_type),
1916 13 : size_binop (PLUS_EXPR, TREE_OPERAND (op, 2),
1917 : bitsize_int (i * sz)));
1918 13 : tree r = make_ssa_name (m_limb_type);
1919 13 : gimple *g = gimple_build_assign (r, bfr);
1920 13 : insert_before (g);
1921 13 : tree type = limb_access_type (TREE_TYPE (op), idx);
1922 13 : if (!useless_type_conversion_p (type, m_limb_type))
1923 0 : r = add_cast (type, r);
1924 : return r;
1925 : }
1926 10 : tree var;
1927 10 : if (m_first)
1928 : {
1929 5 : unsigned HOST_WIDE_INT sz = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (op)));
1930 5 : machine_mode mode;
1931 5 : tree type, bfr;
1932 5 : if (bitwise_mode_for_size (sz).exists (&mode)
1933 0 : && known_eq (GET_MODE_BITSIZE (mode), sz))
1934 0 : type = bitwise_type_for_mode (mode);
1935 : else
1936 : {
1937 5 : mode = VOIDmode;
1938 5 : type = TYPE_MAIN_VARIANT (TREE_TYPE (TREE_OPERAND (op, 0)));
1939 : }
1940 5 : if (TYPE_ALIGN (type) < TYPE_ALIGN (TREE_TYPE (op)))
1941 0 : type = build_aligned_type (type, TYPE_ALIGN (TREE_TYPE (op)));
1942 5 : var = create_tmp_var (type);
1943 5 : TREE_ADDRESSABLE (var) = 1;
1944 5 : gimple *g;
1945 5 : if (mode != VOIDmode)
1946 : {
1947 0 : bfr = build3 (BIT_FIELD_REF, type, TREE_OPERAND (op, 0),
1948 0 : TYPE_SIZE (type), TREE_OPERAND (op, 2));
1949 0 : g = gimple_build_assign (make_ssa_name (type),
1950 : BIT_FIELD_REF, bfr);
1951 0 : gimple_set_location (g, m_loc);
1952 0 : gsi_insert_after (&m_init_gsi, g, GSI_NEW_STMT);
1953 0 : bfr = gimple_assign_lhs (g);
1954 : }
1955 : else
1956 5 : bfr = TREE_OPERAND (op, 0);
1957 5 : g = gimple_build_assign (var, bfr);
1958 5 : gimple_set_location (g, m_loc);
1959 5 : gsi_insert_after (&m_init_gsi, g, GSI_NEW_STMT);
1960 5 : if (mode == VOIDmode)
1961 : {
1962 5 : unsigned HOST_WIDE_INT nelts
1963 5 : = CEIL (tree_to_uhwi (TYPE_SIZE (TREE_TYPE (op))), limb_prec);
1964 5 : tree atype = build_array_type_nelts (m_limb_type, nelts);
1965 5 : var = build2 (MEM_REF, atype, build_fold_addr_expr (var),
1966 : build_int_cst (build_pointer_type (type),
1967 5 : tree_to_uhwi (TREE_OPERAND (op, 2))
1968 5 : / BITS_PER_UNIT));
1969 : }
1970 5 : m_data.safe_push (var);
1971 : }
1972 : else
1973 5 : var = unshare_expr (m_data[m_data_cnt]);
1974 10 : ++m_data_cnt;
1975 10 : var = limb_access (TREE_TYPE (op), var, idx, false);
1976 10 : tree r = make_ssa_name (m_limb_type);
1977 10 : gimple *g = gimple_build_assign (r, var);
1978 10 : insert_before (g);
1979 10 : return r;
1980 : }
1981 :
1982 : /* Add a new EH edge from SRC to EH_EDGE->dest, where EH_EDGE
1983 : is an older EH edge, and except for virtual PHIs duplicate the
1984 : PHI argument from the EH_EDGE to the new EH edge. */
1985 :
1986 : static void
1987 20 : add_eh_edge (basic_block src, edge eh_edge)
1988 : {
1989 20 : edge e = make_edge (src, eh_edge->dest, EDGE_EH);
1990 20 : e->probability = profile_probability::very_unlikely ();
1991 20 : for (gphi_iterator gsi = gsi_start_phis (eh_edge->dest);
1992 27 : !gsi_end_p (gsi); gsi_next (&gsi))
1993 : {
1994 7 : gphi *phi = gsi.phi ();
1995 7 : tree lhs = gimple_phi_result (phi);
1996 14 : if (virtual_operand_p (lhs))
1997 4 : continue;
1998 3 : const phi_arg_d *arg = gimple_phi_arg (phi, eh_edge->dest_idx);
1999 3 : add_phi_arg (phi, arg->def, e, arg->locus);
2000 : }
2001 20 : }
2002 :
2003 : /* Helper function for handle_stmt method, handle a load from memory. */
2004 :
2005 : tree
2006 21362 : bitint_large_huge::handle_load (gimple *stmt, tree idx)
2007 : {
2008 21362 : tree rhs1 = gimple_assign_rhs1 (stmt);
2009 21362 : tree rhs_type = TREE_TYPE (rhs1);
2010 21362 : bool eh = stmt_ends_bb_p (stmt);
2011 21362 : bool load_bitfield_p = false;
2012 21362 : edge eh_edge = NULL;
2013 21362 : gimple *g;
2014 :
2015 21362 : if (TREE_CODE (rhs1) == BIT_FIELD_REF
2016 21362 : && integer_zerop (TREE_OPERAND (rhs1, 2)))
2017 2 : rhs1 = TREE_OPERAND (rhs1, 0);
2018 :
2019 21362 : if (eh)
2020 : {
2021 10 : edge_iterator ei;
2022 10 : basic_block bb = gimple_bb (stmt);
2023 :
2024 10 : FOR_EACH_EDGE (eh_edge, ei, bb->succs)
2025 10 : if (eh_edge->flags & EDGE_EH)
2026 : break;
2027 : }
2028 :
2029 21362 : if (TREE_CODE (rhs1) == COMPONENT_REF
2030 21362 : && DECL_BIT_FIELD_TYPE (TREE_OPERAND (rhs1, 1)))
2031 : {
2032 1225 : tree fld = TREE_OPERAND (rhs1, 1);
2033 : /* For little-endian, we can allow as inputs bit-fields
2034 : which start at a limb boundary. */
2035 1225 : gcc_assert (tree_fits_uhwi_p (DECL_FIELD_BIT_OFFSET (fld)));
2036 1225 : if (!bitint_big_endian
2037 1225 : && DECL_OFFSET_ALIGN (fld) >= TYPE_ALIGN (TREE_TYPE (rhs1))
2038 2450 : && (tree_to_uhwi (DECL_FIELD_BIT_OFFSET (fld)) % limb_prec) == 0)
2039 : {
2040 739 : load_bitfield_p = true;
2041 751 : goto normal_load;
2042 : }
2043 : /* Even if DECL_FIELD_BIT_OFFSET (fld) is a multiple of BITS_PER_UNIT,
2044 : handle it normally for now. */
2045 486 : if (!bitint_big_endian
2046 486 : && (tree_to_uhwi (DECL_FIELD_BIT_OFFSET (fld)) % BITS_PER_UNIT) == 0)
2047 : {
2048 12 : load_bitfield_p = true;
2049 12 : goto normal_load;
2050 : }
2051 474 : tree repr = DECL_BIT_FIELD_REPRESENTATIVE (fld);
2052 474 : poly_int64 bitoffset;
2053 474 : poly_uint64 field_offset, repr_offset;
2054 474 : bool var_field_off = false;
2055 474 : if (poly_int_tree_p (DECL_FIELD_OFFSET (fld), &field_offset)
2056 948 : && poly_int_tree_p (DECL_FIELD_OFFSET (repr), &repr_offset))
2057 474 : bitoffset = (field_offset - repr_offset) * BITS_PER_UNIT;
2058 : else
2059 : {
2060 : bitoffset = 0;
2061 : var_field_off = true;
2062 : }
2063 474 : bitoffset += (tree_to_uhwi (DECL_FIELD_BIT_OFFSET (fld))
2064 474 : - tree_to_uhwi (DECL_FIELD_BIT_OFFSET (repr)));
2065 948 : tree nrhs1 = build3 (COMPONENT_REF, TREE_TYPE (repr),
2066 474 : TREE_OPERAND (rhs1, 0), repr,
2067 0 : var_field_off ? TREE_OPERAND (rhs1, 2) : NULL_TREE);
2068 474 : HOST_WIDE_INT bo = bitoffset.to_constant ();
2069 474 : unsigned bo_idx = (unsigned HOST_WIDE_INT) bo / limb_prec;
2070 474 : unsigned bo_bit = (unsigned HOST_WIDE_INT) bo % limb_prec;
2071 474 : unsigned bo_last = 0;
2072 474 : unsigned bo_shift = bo_bit;
2073 474 : unsigned nelts = CEIL (TYPE_PRECISION (rhs_type), limb_prec);
2074 474 : if (bitint_big_endian)
2075 : {
2076 0 : bo_last = CEIL (TYPE_PRECISION (rhs_type) + bo_bit, limb_prec) - 1;
2077 0 : bo_shift = (TYPE_PRECISION (rhs_type) + bo_bit) % limb_prec;
2078 0 : if (bo_shift)
2079 0 : bo_shift = limb_prec - bo_shift;
2080 : }
2081 474 : if (m_first)
2082 : {
2083 141 : if (m_upwards && bo_shift)
2084 : {
2085 138 : gimple_stmt_iterator save_gsi = m_gsi;
2086 138 : m_gsi = m_init_gsi;
2087 138 : if (gsi_end_p (m_gsi))
2088 57 : m_gsi = gsi_after_labels (gsi_bb (m_gsi));
2089 : else
2090 81 : gsi_next (&m_gsi);
2091 138 : tree t = limb_access (NULL_TREE, nrhs1,
2092 138 : size_int (bo_idx + bo_last), true);
2093 138 : tree iv = make_ssa_name (m_limb_type);
2094 138 : g = gimple_build_assign (iv, t);
2095 138 : insert_before (g);
2096 138 : if (eh)
2097 : {
2098 2 : maybe_duplicate_eh_stmt (g, stmt);
2099 2 : if (eh_edge)
2100 : {
2101 2 : edge e = split_block (gsi_bb (m_gsi), g);
2102 2 : add_eh_edge (e->src, eh_edge);
2103 2 : m_gsi = gsi_after_labels (e->dest);
2104 2 : if (gsi_bb (save_gsi) == e->src)
2105 : {
2106 1 : if (gsi_end_p (save_gsi))
2107 0 : save_gsi = gsi_end_bb (e->dest);
2108 : else
2109 1 : save_gsi = gsi_for_stmt (gsi_stmt (save_gsi));
2110 : }
2111 2 : if (m_preheader_bb == e->src)
2112 1 : m_preheader_bb = e->dest;
2113 : }
2114 : }
2115 138 : m_init_gsi = m_gsi;
2116 138 : if (gsi_end_p (m_init_gsi))
2117 222 : m_init_gsi = gsi_last_bb (gsi_bb (m_init_gsi));
2118 : else
2119 28 : gsi_prev (&m_init_gsi);
2120 138 : m_gsi = save_gsi;
2121 138 : tree out;
2122 138 : prepare_data_in_out (iv, idx, &out);
2123 138 : out = m_data[m_data_cnt];
2124 138 : m_data.safe_push (out);
2125 138 : }
2126 : else
2127 : {
2128 3 : m_data.safe_push (NULL_TREE);
2129 3 : m_data.safe_push (NULL_TREE);
2130 3 : m_data.safe_push (NULL_TREE);
2131 : }
2132 : }
2133 :
2134 474 : tree nidx0 = NULL_TREE, nidx1 = NULL_TREE;
2135 474 : tree iv = m_data[m_data_cnt];
2136 474 : if (m_cast_conditional && iv)
2137 : {
2138 12 : gcc_assert (!m_bitfld_load);
2139 12 : m_bitfld_load = m_data_cnt;
2140 : }
2141 474 : if (tree_fits_uhwi_p (idx))
2142 : {
2143 272 : unsigned prec = TYPE_PRECISION (rhs_type);
2144 272 : unsigned HOST_WIDE_INT i = tree_to_uhwi (idx);
2145 272 : if (bitint_big_endian)
2146 0 : i = nelts - 1 - i;
2147 272 : gcc_assert (i * limb_prec < prec);
2148 272 : if (bo_shift)
2149 272 : nidx1 = size_int (bo_idx + (bitint_big_endian
2150 : ? bo_last - i - 1 : i + 1));
2151 272 : if ((i + 1) * limb_prec > prec)
2152 : {
2153 100 : prec %= limb_prec;
2154 100 : if (prec + bo_bit <= (unsigned) limb_prec)
2155 272 : nidx1 = NULL_TREE;
2156 : }
2157 272 : if (!iv)
2158 4 : nidx0 = size_int (bo_idx + (bitint_big_endian ? bo_last - i : i));
2159 : }
2160 : else
2161 : {
2162 202 : HOST_WIDE_INT adj = bo_idx;
2163 202 : if (bitint_big_endian)
2164 0 : adj += (HOST_WIDE_INT) bo_last + 1 - nelts;
2165 202 : if (!iv)
2166 : {
2167 4 : if (adj == 0)
2168 : nidx0 = idx;
2169 : else
2170 : {
2171 0 : nidx0 = make_ssa_name (sizetype);
2172 0 : g = gimple_build_assign (nidx0, PLUS_EXPR, idx,
2173 0 : size_int (adj));
2174 0 : insert_before (g);
2175 : }
2176 : }
2177 202 : if (bo_shift)
2178 : {
2179 202 : if (bitint_big_endian && adj == 1)
2180 : nidx1 = idx;
2181 : else
2182 : {
2183 202 : nidx1 = make_ssa_name (sizetype);
2184 202 : g = gimple_build_assign (nidx1, PLUS_EXPR, idx,
2185 404 : size_int (adj + (bitint_big_endian
2186 : ? -1 : 1)));
2187 202 : insert_before (g);
2188 : }
2189 : }
2190 : }
2191 :
2192 680 : tree iv2 = NULL_TREE;
2193 206 : if (nidx0)
2194 : {
2195 8 : tree t = limb_access (NULL_TREE, nrhs1, nidx0, true);
2196 8 : iv = make_ssa_name (m_limb_type);
2197 8 : g = gimple_build_assign (iv, t);
2198 8 : insert_before (g);
2199 8 : if (eh)
2200 : {
2201 0 : maybe_duplicate_eh_stmt (g, stmt);
2202 0 : if (eh_edge)
2203 : {
2204 0 : edge e = split_block (gsi_bb (m_gsi), g);
2205 0 : m_gsi = gsi_after_labels (e->dest);
2206 0 : add_eh_edge (e->src, eh_edge);
2207 : }
2208 : }
2209 : }
2210 474 : if (nidx1)
2211 : {
2212 385 : bool conditional = m_var_msb && !tree_fits_uhwi_p (idx);
2213 385 : unsigned prec = TYPE_PRECISION (rhs_type);
2214 385 : if (conditional)
2215 : {
2216 3 : if ((prec % limb_prec) == 0
2217 3 : || ((prec % limb_prec) + bo_bit > (unsigned) limb_prec))
2218 382 : conditional = false;
2219 : }
2220 385 : edge edge_true = NULL, edge_false = NULL;
2221 385 : if (conditional)
2222 : {
2223 6 : g = gimple_build_cond (NE_EXPR, idx,
2224 : bitint_big_endian
2225 : ? size_zero_node
2226 3 : : size_int (prec / limb_prec),
2227 : NULL_TREE, NULL_TREE);
2228 3 : if_then (g, profile_probability::likely (),
2229 : edge_true, edge_false);
2230 : }
2231 385 : tree t = limb_access (NULL_TREE, nrhs1, nidx1, true);
2232 385 : if (m_upwards_2limb
2233 283 : && !m_first
2234 184 : && !m_bitfld_load
2235 178 : && !tree_fits_uhwi_p (idx))
2236 95 : iv2 = m_data[m_data_cnt + 1];
2237 : else
2238 290 : iv2 = make_ssa_name (m_limb_type);
2239 385 : g = gimple_build_assign (iv2, t);
2240 385 : insert_before (g);
2241 385 : if (eh)
2242 : {
2243 5 : maybe_duplicate_eh_stmt (g, stmt);
2244 5 : if (eh_edge)
2245 : {
2246 5 : edge e = split_block (gsi_bb (m_gsi), g);
2247 5 : m_gsi = gsi_after_labels (e->dest);
2248 5 : add_eh_edge (e->src, eh_edge);
2249 : }
2250 : }
2251 385 : if (conditional)
2252 : {
2253 3 : tree iv3 = make_ssa_name (m_limb_type);
2254 3 : if (eh)
2255 0 : edge_true = find_edge (gsi_bb (m_gsi), edge_false->dest);
2256 3 : gphi *phi = create_phi_node (iv3, edge_true->dest);
2257 3 : add_phi_arg (phi, iv2, edge_true, UNKNOWN_LOCATION);
2258 3 : add_phi_arg (phi, build_zero_cst (m_limb_type),
2259 : edge_false, UNKNOWN_LOCATION);
2260 3 : m_gsi = gsi_after_labels (edge_true->dest);
2261 3 : iv2 = iv3;
2262 : }
2263 : }
2264 474 : if (bo_shift)
2265 : {
2266 474 : g = gimple_build_assign (make_ssa_name (m_limb_type), RSHIFT_EXPR,
2267 : iv, build_int_cst (unsigned_type_node,
2268 474 : bo_shift));
2269 474 : insert_before (g);
2270 474 : iv = gimple_assign_lhs (g);
2271 : }
2272 474 : if (iv2)
2273 : {
2274 385 : g = gimple_build_assign (make_ssa_name (m_limb_type), LSHIFT_EXPR,
2275 : iv2, build_int_cst (unsigned_type_node,
2276 385 : limb_prec - bo_shift));
2277 385 : insert_before (g);
2278 385 : g = gimple_build_assign (make_ssa_name (m_limb_type), BIT_IOR_EXPR,
2279 : gimple_assign_lhs (g), iv);
2280 385 : insert_before (g);
2281 385 : iv = gimple_assign_lhs (g);
2282 385 : if (m_data[m_data_cnt])
2283 379 : m_data[m_data_cnt] = iv2;
2284 : }
2285 474 : if (tree_fits_uhwi_p (idx))
2286 : {
2287 272 : tree atype = limb_access_type (rhs_type, idx);
2288 272 : if (!useless_type_conversion_p (atype, TREE_TYPE (iv)))
2289 100 : iv = add_cast (atype, iv);
2290 : }
2291 474 : m_data_cnt += 3;
2292 474 : return iv;
2293 : }
2294 :
2295 20888 : normal_load:
2296 : /* Use write_p = true for loads with EH edges to make
2297 : sure limb_access doesn't add a cast as separate
2298 : statement after it. */
2299 20888 : rhs1 = limb_access (rhs_type, rhs1, idx, eh, !load_bitfield_p);
2300 20888 : tree ret = make_ssa_name (TREE_TYPE (rhs1));
2301 20888 : g = gimple_build_assign (ret, rhs1);
2302 20888 : insert_before (g);
2303 20888 : if (eh)
2304 : {
2305 3 : maybe_duplicate_eh_stmt (g, stmt);
2306 3 : if (eh_edge)
2307 : {
2308 3 : edge e = split_block (gsi_bb (m_gsi), g);
2309 3 : m_gsi = gsi_after_labels (e->dest);
2310 3 : add_eh_edge (e->src, eh_edge);
2311 : }
2312 3 : if (tree_fits_uhwi_p (idx))
2313 : {
2314 1 : tree atype = limb_access_type (rhs_type, idx);
2315 1 : if (!useless_type_conversion_p (atype, TREE_TYPE (rhs1)))
2316 1 : ret = add_cast (atype, ret);
2317 : }
2318 : }
2319 : return ret;
2320 : }
2321 :
2322 : /* Return a limb IDX from a mergeable statement STMT. */
2323 :
2324 : tree
2325 39200 : bitint_large_huge::handle_stmt (gimple *stmt, tree idx)
2326 : {
2327 39200 : tree lhs, rhs1, rhs2 = NULL_TREE;
2328 39200 : gimple *g;
2329 39200 : switch (gimple_code (stmt))
2330 : {
2331 39200 : case GIMPLE_ASSIGN:
2332 39200 : if (gimple_assign_load_p (stmt))
2333 21362 : return handle_load (stmt, idx);
2334 17838 : switch (gimple_assign_rhs_code (stmt))
2335 : {
2336 990 : case BIT_AND_EXPR:
2337 990 : case BIT_IOR_EXPR:
2338 990 : case BIT_XOR_EXPR:
2339 990 : rhs2 = handle_operand (gimple_assign_rhs2 (stmt), idx);
2340 : /* FALLTHRU */
2341 1410 : case BIT_NOT_EXPR:
2342 1410 : rhs1 = handle_operand (gimple_assign_rhs1 (stmt), idx);
2343 1410 : lhs = make_ssa_name (TREE_TYPE (rhs1));
2344 1410 : g = gimple_build_assign (lhs, gimple_assign_rhs_code (stmt),
2345 : rhs1, rhs2);
2346 1410 : insert_before (g);
2347 1410 : return lhs;
2348 4026 : case PLUS_EXPR:
2349 4026 : case MINUS_EXPR:
2350 4026 : rhs1 = handle_operand (gimple_assign_rhs1 (stmt), idx);
2351 4026 : rhs2 = handle_operand (gimple_assign_rhs2 (stmt), idx);
2352 4026 : return handle_plus_minus (gimple_assign_rhs_code (stmt),
2353 4026 : rhs1, rhs2, idx);
2354 163 : case NEGATE_EXPR:
2355 163 : rhs2 = handle_operand (gimple_assign_rhs1 (stmt), idx);
2356 163 : rhs1 = build_zero_cst (TREE_TYPE (rhs2));
2357 163 : return handle_plus_minus (MINUS_EXPR, rhs1, rhs2, idx);
2358 152 : case LSHIFT_EXPR:
2359 152 : return handle_lshift (handle_operand (gimple_assign_rhs1 (stmt),
2360 : idx),
2361 152 : gimple_assign_rhs2 (stmt), idx);
2362 5087 : case SSA_NAME:
2363 5087 : case PAREN_EXPR:
2364 5087 : case INTEGER_CST:
2365 5087 : return handle_operand (gimple_assign_rhs1 (stmt), idx);
2366 6969 : CASE_CONVERT:
2367 6969 : return handle_cast (TREE_TYPE (gimple_assign_lhs (stmt)),
2368 6969 : gimple_assign_rhs1 (stmt), idx);
2369 8 : case VIEW_CONVERT_EXPR:
2370 8 : return handle_cast (TREE_TYPE (gimple_assign_lhs (stmt)),
2371 8 : TREE_OPERAND (gimple_assign_rhs1 (stmt), 0),
2372 8 : idx);
2373 23 : case BIT_FIELD_REF:
2374 23 : return handle_bit_field_ref (gimple_assign_rhs1 (stmt), idx);
2375 : default:
2376 : break;
2377 : }
2378 : break;
2379 : default:
2380 : break;
2381 : }
2382 0 : gcc_unreachable ();
2383 : }
2384 :
2385 : /* Return minimum precision of OP at STMT.
2386 : Positive value is minimum precision above which all bits
2387 : are zero, negative means all bits above negation of the
2388 : value are copies of the sign bit. */
2389 :
2390 : static int
2391 8446 : range_to_prec (tree op, gimple *stmt)
2392 : {
2393 8446 : int_range_max r;
2394 8446 : wide_int w;
2395 8446 : tree type = TREE_TYPE (op);
2396 8446 : unsigned int prec = TYPE_PRECISION (type);
2397 :
2398 8446 : if (!optimize
2399 7510 : || !get_range_query (cfun)->range_of_expr (r, op, stmt)
2400 12201 : || r.undefined_p ())
2401 : {
2402 4692 : if (TYPE_UNSIGNED (type))
2403 1902 : return prec;
2404 : else
2405 2790 : return MIN ((int) -prec, -2);
2406 : }
2407 :
2408 3754 : if (!TYPE_UNSIGNED (TREE_TYPE (op)))
2409 : {
2410 2339 : w = r.lower_bound ();
2411 2339 : if (wi::neg_p (w))
2412 : {
2413 1934 : int min_prec1 = wi::min_precision (w, SIGNED);
2414 1934 : w = r.upper_bound ();
2415 1934 : int min_prec2 = wi::min_precision (w, SIGNED);
2416 1934 : int min_prec = MAX (min_prec1, min_prec2);
2417 1934 : return MIN (-min_prec, -2);
2418 : }
2419 : }
2420 :
2421 1820 : w = r.upper_bound ();
2422 1820 : int min_prec = wi::min_precision (w, UNSIGNED);
2423 1820 : return MAX (min_prec, 1);
2424 8446 : }
2425 :
2426 : /* Return address of the first limb of OP and write into *PREC
2427 : its precision. If positive, the operand is zero extended
2428 : from that precision, if it is negative, the operand is sign-extended
2429 : from -*PREC. If PREC_STORED is NULL, it is the toplevel call,
2430 : otherwise *PREC_STORED is prec from the innermost call without
2431 : range optimizations (0 for uninitialized SSA_NAME). */
2432 :
2433 : tree
2434 3864 : bitint_large_huge::handle_operand_addr (tree op, gimple *stmt,
2435 : int *prec_stored, int *prec)
2436 : {
2437 3864 : wide_int w;
2438 3864 : location_t loc_save = m_loc;
2439 3864 : tree ret = NULL_TREE;
2440 3864 : int precs = 0;
2441 3899 : if ((!BITINT_TYPE_P (TREE_TYPE (op))
2442 3835 : || bitint_precision_kind (TREE_TYPE (op)) < bitint_prec_large)
2443 3988 : && TREE_CODE (op) != INTEGER_CST)
2444 : {
2445 124 : do_int:
2446 124 : *prec = range_to_prec (op, stmt);
2447 124 : bitint_prec_kind kind = bitint_prec_small;
2448 124 : gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (op)));
2449 124 : if (BITINT_TYPE_P (TREE_TYPE (op)))
2450 104 : kind = bitint_precision_kind (TREE_TYPE (op));
2451 104 : if (kind == bitint_prec_middle)
2452 : {
2453 12 : tree type = NULL_TREE;
2454 12 : op = maybe_cast_middle_bitint (&m_gsi, op, type);
2455 : }
2456 124 : tree op_type = TREE_TYPE (op);
2457 124 : unsigned HOST_WIDE_INT nelts
2458 124 : = CEIL (TYPE_PRECISION (op_type), limb_prec);
2459 : /* Add support for 3 or more limbs filled in from normal
2460 : integral type if this assert fails. If no target chooses
2461 : limb mode smaller than half of largest supported normal
2462 : integral type, this will not be needed. */
2463 124 : gcc_assert (nelts <= 2);
2464 124 : precs = (TYPE_UNSIGNED (op_type)
2465 124 : ? TYPE_PRECISION (op_type) : -TYPE_PRECISION (op_type));
2466 124 : if (*prec <= limb_prec && *prec >= -limb_prec)
2467 : {
2468 111 : nelts = 1;
2469 111 : if (TYPE_UNSIGNED (op_type))
2470 : {
2471 26 : if (precs > limb_prec)
2472 124 : precs = limb_prec;
2473 : }
2474 85 : else if (precs < -limb_prec)
2475 124 : precs = -limb_prec;
2476 : }
2477 124 : if (prec_stored)
2478 0 : *prec_stored = precs;
2479 124 : tree atype = build_array_type_nelts (m_limb_type, nelts);
2480 124 : tree var = create_tmp_var (atype);
2481 124 : tree t1 = op;
2482 124 : if (!useless_type_conversion_p (m_limb_type, op_type))
2483 124 : t1 = add_cast (m_limb_type, t1);
2484 124 : tree v = build4 (ARRAY_REF, m_limb_type, var,
2485 0 : bitint_big_endian && nelts > 1
2486 : ? size_one_node : size_zero_node,
2487 : NULL_TREE, NULL_TREE);
2488 124 : gimple *g = gimple_build_assign (v, t1);
2489 124 : insert_before (g);
2490 124 : if (nelts > 1)
2491 : {
2492 13 : tree lp = build_int_cst (unsigned_type_node, limb_prec);
2493 13 : g = gimple_build_assign (make_ssa_name (op_type),
2494 : RSHIFT_EXPR, op, lp);
2495 13 : insert_before (g);
2496 13 : tree t2 = gimple_assign_lhs (g);
2497 13 : t2 = add_cast (m_limb_type, t2);
2498 13 : v = build4 (ARRAY_REF, m_limb_type, var,
2499 : bitint_big_endian ? size_zero_node : size_one_node,
2500 : NULL_TREE, NULL_TREE);
2501 13 : g = gimple_build_assign (v, t2);
2502 13 : insert_before (g);
2503 : }
2504 124 : ret = build_fold_addr_expr (var);
2505 124 : if (!stmt_ends_bb_p (gsi_stmt (m_gsi)))
2506 : {
2507 123 : tree clobber = build_clobber (atype, CLOBBER_STORAGE_END);
2508 123 : g = gimple_build_assign (var, clobber);
2509 123 : gsi_insert_after (&m_gsi, g, GSI_SAME_STMT);
2510 : }
2511 124 : m_loc = loc_save;
2512 124 : goto do_ret;
2513 : }
2514 3788 : switch (TREE_CODE (op))
2515 : {
2516 2922 : case SSA_NAME:
2517 2922 : if (m_names == NULL
2518 2922 : || !bitmap_bit_p (m_names, SSA_NAME_VERSION (op)))
2519 : {
2520 102 : gimple *g = SSA_NAME_DEF_STMT (op);
2521 102 : m_loc = gimple_location (g);
2522 102 : if (gimple_assign_load_p (g))
2523 : {
2524 36 : *prec = range_to_prec (op, NULL);
2525 36 : precs = (TYPE_UNSIGNED (TREE_TYPE (op))
2526 36 : ? TYPE_PRECISION (TREE_TYPE (op))
2527 25 : : -TYPE_PRECISION (TREE_TYPE (op)));
2528 36 : if (prec_stored)
2529 6 : *prec_stored = precs;
2530 36 : ret = build_fold_addr_expr (gimple_assign_rhs1 (g));
2531 36 : ret = force_gimple_operand_gsi (&m_gsi, ret, true,
2532 : NULL_TREE, true, GSI_SAME_STMT);
2533 : }
2534 66 : else if (gimple_code (g) == GIMPLE_NOP)
2535 : {
2536 2 : *prec = TYPE_UNSIGNED (TREE_TYPE (op)) ? limb_prec : -limb_prec;
2537 2 : precs = *prec;
2538 2 : if (prec_stored)
2539 1 : *prec_stored = 0;
2540 2 : tree var = create_tmp_var (m_limb_type);
2541 2 : TREE_ADDRESSABLE (var) = 1;
2542 2 : ret = build_fold_addr_expr (var);
2543 2 : if (!stmt_ends_bb_p (gsi_stmt (m_gsi)))
2544 : {
2545 2 : tree clobber = build_clobber (m_limb_type,
2546 : CLOBBER_STORAGE_END);
2547 2 : g = gimple_build_assign (var, clobber);
2548 2 : gsi_insert_after (&m_gsi, g, GSI_SAME_STMT);
2549 : }
2550 : }
2551 : else
2552 : {
2553 64 : gcc_assert (gimple_assign_cast_p (g));
2554 64 : tree rhs1 = gimple_assign_rhs1 (g);
2555 64 : bitint_prec_kind kind = bitint_prec_small;
2556 64 : if (TREE_CODE (rhs1) == VIEW_CONVERT_EXPR)
2557 1 : rhs1 = TREE_OPERAND (rhs1, 0);
2558 64 : gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (rhs1)));
2559 64 : if (BITINT_TYPE_P (TREE_TYPE (rhs1)))
2560 49 : kind = bitint_precision_kind (TREE_TYPE (rhs1));
2561 49 : if (kind >= bitint_prec_large)
2562 : {
2563 16 : tree lhs_type = TREE_TYPE (op);
2564 16 : tree rhs_type = TREE_TYPE (rhs1);
2565 16 : int prec_stored_val = 0;
2566 16 : ret = handle_operand_addr (rhs1, g, &prec_stored_val, prec);
2567 16 : precs = prec_stored_val;
2568 16 : if (prec_stored)
2569 0 : *prec_stored = prec_stored_val;
2570 16 : if (precs == 0)
2571 : {
2572 1 : gcc_assert (*prec == limb_prec || *prec == -limb_prec);
2573 : precs = *prec;
2574 : }
2575 16 : if (TYPE_PRECISION (lhs_type) > TYPE_PRECISION (rhs_type))
2576 : {
2577 4 : if (TYPE_UNSIGNED (lhs_type)
2578 4 : && !TYPE_UNSIGNED (rhs_type))
2579 1 : gcc_assert (*prec >= 0 || prec_stored == NULL);
2580 : }
2581 : else
2582 : {
2583 12 : if (prec_stored_val == 0)
2584 : /* Non-widening cast of uninitialized value. */;
2585 11 : else if (*prec > 0 && *prec < TYPE_PRECISION (lhs_type))
2586 : ;
2587 11 : else if (TYPE_UNSIGNED (lhs_type))
2588 : {
2589 8 : gcc_assert (*prec > 0
2590 : || prec_stored_val > 0
2591 : || (-prec_stored_val
2592 : >= TYPE_PRECISION (lhs_type)));
2593 8 : *prec = TYPE_PRECISION (lhs_type);
2594 : }
2595 3 : else if (*prec < 0 && -*prec < TYPE_PRECISION (lhs_type))
2596 : ;
2597 : else
2598 3 : *prec = -TYPE_PRECISION (lhs_type);
2599 : }
2600 : }
2601 : else
2602 : {
2603 48 : op = rhs1;
2604 48 : stmt = g;
2605 48 : goto do_int;
2606 : }
2607 : }
2608 54 : m_loc = loc_save;
2609 54 : goto do_ret;
2610 : }
2611 : else
2612 : {
2613 2820 : int p = var_to_partition (m_map, op);
2614 2820 : gcc_assert (m_vars[p] != NULL_TREE);
2615 2820 : *prec = range_to_prec (op, stmt);
2616 2820 : precs = (TYPE_UNSIGNED (TREE_TYPE (op))
2617 2820 : ? TYPE_PRECISION (TREE_TYPE (op))
2618 1669 : : -TYPE_PRECISION (TREE_TYPE (op)));
2619 2820 : if (prec_stored)
2620 9 : *prec_stored = precs;
2621 2820 : ret = build_fold_addr_expr (m_vars[p]);
2622 2820 : goto do_ret;
2623 : }
2624 866 : case INTEGER_CST:
2625 866 : unsigned int min_prec, mp;
2626 866 : tree type;
2627 866 : w = wi::to_wide (op);
2628 866 : if (tree_int_cst_sgn (op) >= 0)
2629 : {
2630 651 : min_prec = wi::min_precision (w, UNSIGNED);
2631 651 : *prec = MAX (min_prec, 1);
2632 : }
2633 : else
2634 : {
2635 215 : min_prec = wi::min_precision (w, SIGNED);
2636 215 : *prec = MIN ((int) -min_prec, -2);
2637 : }
2638 866 : mp = CEIL (min_prec, limb_prec) * limb_prec;
2639 866 : if (mp == 0)
2640 : mp = 1;
2641 866 : if (mp >= (unsigned) TYPE_PRECISION (TREE_TYPE (op))
2642 866 : && (BITINT_TYPE_P (TREE_TYPE (op))
2643 24 : || TYPE_PRECISION (TREE_TYPE (op)) <= limb_prec))
2644 316 : type = TREE_TYPE (op);
2645 : else
2646 550 : type = build_bitint_type (mp, 1);
2647 20 : if (!BITINT_TYPE_P (type)
2648 866 : || bitint_precision_kind (type) == bitint_prec_small)
2649 : {
2650 587 : if (TYPE_PRECISION (type) <= limb_prec)
2651 587 : type = m_limb_type;
2652 : else
2653 : {
2654 0 : while (bitint_precision_kind (mp) == bitint_prec_small)
2655 0 : mp += limb_prec;
2656 : /* This case is for targets which e.g. have 64-bit
2657 : limb but categorize up to 128-bits _BitInts as
2658 : small. We could use type of m_limb_type[2] and
2659 : similar instead to save space. */
2660 0 : type = build_bitint_type (mp, 1);
2661 : }
2662 : }
2663 866 : if (tree_int_cst_sgn (op) >= 0)
2664 651 : precs = MAX (TYPE_PRECISION (type), 1);
2665 : else
2666 215 : precs = MIN ((int) -TYPE_PRECISION (type), -2);
2667 866 : if (prec_stored)
2668 0 : *prec_stored = precs;
2669 866 : op = tree_output_constant_def (fold_convert (type, op));
2670 866 : ret = build_fold_addr_expr (op);
2671 866 : goto do_ret;
2672 0 : default:
2673 0 : gcc_unreachable ();
2674 : }
2675 3864 : do_ret:
2676 3864 : if (bitint_big_endian && prec_stored == NULL)
2677 : {
2678 0 : int p1 = *prec < 0 ? -*prec : *prec;
2679 0 : int p2 = precs < 0 ? -precs : precs;
2680 0 : int c1 = CEIL (p1, limb_prec);
2681 0 : int c2 = CEIL (p2, limb_prec);
2682 0 : gcc_assert (c1 <= c2);
2683 0 : if (c1 != c2)
2684 : {
2685 0 : gimple *g
2686 0 : = gimple_build_assign (make_ssa_name (TREE_TYPE (ret)),
2687 : POINTER_PLUS_EXPR, ret,
2688 0 : size_int ((c2 - c1) * m_limb_size));
2689 0 : insert_before (g);
2690 0 : ret = gimple_assign_lhs (g);
2691 : }
2692 : }
2693 3864 : return ret;
2694 3864 : }
2695 :
2696 : /* Helper function, create a loop before the current location,
2697 : start with sizetype INIT value from the preheader edge. Return
2698 : a PHI result and set *IDX_NEXT to SSA_NAME it creates and uses
2699 : from the latch edge. */
2700 :
2701 : tree
2702 14674 : bitint_large_huge::create_loop (tree init, tree *idx_next)
2703 : {
2704 14674 : if (!gsi_end_p (m_gsi))
2705 15776 : gsi_prev (&m_gsi);
2706 : else
2707 4336 : m_gsi = gsi_last_bb (gsi_bb (m_gsi));
2708 14674 : edge e1 = split_block (gsi_bb (m_gsi), gsi_stmt (m_gsi));
2709 14674 : edge e2 = split_block (e1->dest, (gimple *) NULL);
2710 14674 : edge e3 = make_edge (e1->dest, e1->dest, EDGE_TRUE_VALUE);
2711 14674 : e3->probability = profile_probability::very_unlikely ();
2712 14674 : e2->flags = EDGE_FALSE_VALUE;
2713 14674 : e2->probability = e3->probability.invert ();
2714 14674 : tree idx = make_ssa_name (sizetype);
2715 14674 : gphi *phi = create_phi_node (idx, e1->dest);
2716 14674 : add_phi_arg (phi, init, e1, UNKNOWN_LOCATION);
2717 14674 : *idx_next = make_ssa_name (sizetype);
2718 14674 : add_phi_arg (phi, *idx_next, e3, UNKNOWN_LOCATION);
2719 14674 : m_gsi = gsi_after_labels (e1->dest);
2720 14674 : m_bb = e1->dest;
2721 14674 : m_preheader_bb = e1->src;
2722 14674 : class loop *loop = alloc_loop ();
2723 14674 : loop->header = e1->dest;
2724 14674 : add_loop (loop, e1->src->loop_father);
2725 14674 : return idx;
2726 : }
2727 :
2728 : /* Lower large/huge _BitInt statement mergeable or similar STMT which can be
2729 : lowered using iteration from the least significant limb up to the most
2730 : significant limb. For large _BitInt it is emitted as straight line code
2731 : before current location, for huge _BitInt as a loop handling two limbs
2732 : at once, followed by handling up to limbs in straight line code (at most
2733 : one full and one partial limb). It can also handle EQ_EXPR/NE_EXPR
2734 : comparisons, in that case CMP_CODE should be the comparison code and
2735 : CMP_OP1/CMP_OP2 the comparison operands. */
2736 :
2737 : tree
2738 23385 : bitint_large_huge::lower_mergeable_stmt (gimple *stmt, tree_code &cmp_code,
2739 : tree cmp_op1, tree cmp_op2)
2740 : {
2741 23385 : bool eq_p = cmp_code != ERROR_MARK;
2742 23385 : tree type;
2743 23385 : if (eq_p)
2744 6552 : type = TREE_TYPE (cmp_op1);
2745 : else
2746 16833 : type = TREE_TYPE (gimple_assign_lhs (stmt));
2747 23385 : gcc_assert (BITINT_TYPE_P (type));
2748 23385 : bitint_prec_kind kind = bitint_precision_kind (type);
2749 23385 : gcc_assert (kind >= bitint_prec_large);
2750 23385 : gimple *g;
2751 23385 : tree lhs = gimple_get_lhs (stmt);
2752 23385 : tree rhs1, lhs_type = lhs ? TREE_TYPE (lhs) : NULL_TREE;
2753 17273 : if (lhs
2754 17273 : && TREE_CODE (lhs) == SSA_NAME
2755 8749 : && BITINT_TYPE_P (TREE_TYPE (lhs))
2756 8309 : && bitint_precision_kind (TREE_TYPE (lhs)) >= bitint_prec_large)
2757 : {
2758 8309 : int p = var_to_partition (m_map, lhs);
2759 8309 : gcc_assert (m_vars[p] != NULL_TREE);
2760 8309 : m_lhs = lhs = m_vars[p];
2761 : }
2762 23385 : unsigned cnt, rem = 0, end = 0, prec = TYPE_PRECISION (type);
2763 23385 : bool sext = false;
2764 23385 : tree ext = NULL_TREE, store_operand = NULL_TREE;
2765 23385 : bool eh = false;
2766 23385 : basic_block eh_pad = NULL;
2767 23385 : tree nlhs = NULL_TREE;
2768 23385 : unsigned HOST_WIDE_INT bo_idx = 0;
2769 23385 : unsigned HOST_WIDE_INT bo_bit = 0;
2770 23385 : unsigned bo_shift = 0;
2771 23385 : unsigned bo_last = 0;
2772 23385 : bool bo_be_p = false;
2773 23385 : tree bf_cur = NULL_TREE, bf_next = NULL_TREE;
2774 23385 : if (gimple_store_p (stmt))
2775 : {
2776 8524 : store_operand = gimple_assign_rhs1 (stmt);
2777 8524 : eh = stmt_ends_bb_p (stmt);
2778 8524 : if (eh)
2779 : {
2780 2 : edge e;
2781 2 : edge_iterator ei;
2782 2 : basic_block bb = gimple_bb (stmt);
2783 :
2784 2 : FOR_EACH_EDGE (e, ei, bb->succs)
2785 2 : if (e->flags & EDGE_EH)
2786 : {
2787 2 : eh_pad = e->dest;
2788 2 : break;
2789 : }
2790 : }
2791 8524 : if (TREE_CODE (lhs) == COMPONENT_REF
2792 8524 : && DECL_BIT_FIELD_TYPE (TREE_OPERAND (lhs, 1)))
2793 : {
2794 162 : tree fld = TREE_OPERAND (lhs, 1);
2795 162 : gcc_assert (tree_fits_uhwi_p (DECL_FIELD_BIT_OFFSET (fld)));
2796 162 : tree repr = DECL_BIT_FIELD_REPRESENTATIVE (fld);
2797 162 : poly_int64 bitoffset;
2798 162 : poly_uint64 field_offset, repr_offset;
2799 162 : if (!bitint_big_endian
2800 162 : && (tree_to_uhwi (DECL_FIELD_BIT_OFFSET (fld))
2801 162 : % BITS_PER_UNIT) == 0)
2802 : nlhs = lhs;
2803 : else
2804 : {
2805 143 : bool var_field_off = false;
2806 143 : if (poly_int_tree_p (DECL_FIELD_OFFSET (fld), &field_offset)
2807 286 : && poly_int_tree_p (DECL_FIELD_OFFSET (repr), &repr_offset))
2808 143 : bitoffset = (field_offset - repr_offset) * BITS_PER_UNIT;
2809 : else
2810 : {
2811 : bitoffset = 0;
2812 : var_field_off = true;
2813 : }
2814 143 : bitoffset += (tree_to_uhwi (DECL_FIELD_BIT_OFFSET (fld))
2815 143 : - tree_to_uhwi (DECL_FIELD_BIT_OFFSET (repr)));
2816 286 : nlhs = build3 (COMPONENT_REF, TREE_TYPE (repr),
2817 143 : TREE_OPERAND (lhs, 0), repr,
2818 : var_field_off
2819 0 : ? TREE_OPERAND (lhs, 2) : NULL_TREE);
2820 143 : HOST_WIDE_INT bo = bitoffset.to_constant ();
2821 143 : bo_idx = (unsigned HOST_WIDE_INT) bo / limb_prec;
2822 143 : bo_bit = (unsigned HOST_WIDE_INT) bo % limb_prec;
2823 143 : bo_shift = bo_bit;
2824 143 : if (bitint_big_endian)
2825 : {
2826 0 : bo_last = CEIL (prec + bo_bit, limb_prec) - 1;
2827 0 : bo_shift = (prec + bo_bit) % limb_prec;
2828 0 : bo_be_p = true;
2829 0 : if (bo_shift)
2830 0 : bo_shift = limb_prec - bo_shift;
2831 : }
2832 : }
2833 : }
2834 : }
2835 8524 : if ((store_operand
2836 8524 : && TREE_CODE (store_operand) == SSA_NAME
2837 7546 : && (m_names == NULL
2838 7516 : || !bitmap_bit_p (m_names, SSA_NAME_VERSION (store_operand)))
2839 1156 : && gimple_assign_cast_p (SSA_NAME_DEF_STMT (store_operand)))
2840 31353 : || gimple_assign_cast_p (stmt))
2841 : {
2842 2159 : rhs1 = gimple_assign_rhs1 (store_operand
2843 556 : ? SSA_NAME_DEF_STMT (store_operand)
2844 : : stmt);
2845 1603 : if (TREE_CODE (rhs1) == VIEW_CONVERT_EXPR)
2846 2 : rhs1 = TREE_OPERAND (rhs1, 0);
2847 : /* Optimize mergeable ops ending with widening cast to _BitInt
2848 : (or followed by store). We can lower just the limbs of the
2849 : cast operand and widen afterwards. */
2850 1603 : if (TREE_CODE (rhs1) == SSA_NAME
2851 1603 : && (m_names == NULL
2852 1597 : || !bitmap_bit_p (m_names, SSA_NAME_VERSION (rhs1)))
2853 619 : && BITINT_TYPE_P (TREE_TYPE (rhs1))
2854 480 : && bitint_precision_kind (TREE_TYPE (rhs1)) >= bitint_prec_large
2855 2034 : && (CEIL ((unsigned) TYPE_PRECISION (TREE_TYPE (rhs1)),
2856 431 : limb_prec) < CEIL (prec, limb_prec)
2857 306 : || (kind == bitint_prec_huge
2858 262 : && TYPE_PRECISION (TREE_TYPE (rhs1)) < prec)))
2859 : {
2860 131 : store_operand = rhs1;
2861 131 : prec = TYPE_PRECISION (TREE_TYPE (rhs1));
2862 131 : kind = bitint_precision_kind (TREE_TYPE (rhs1));
2863 131 : if (!TYPE_UNSIGNED (TREE_TYPE (rhs1)))
2864 54 : sext = true;
2865 : }
2866 : }
2867 23385 : tree idx = NULL_TREE, idx_first = NULL_TREE, idx_next = NULL_TREE;
2868 23385 : if (kind == bitint_prec_large)
2869 12207 : cnt = CEIL (prec, limb_prec);
2870 : else
2871 : {
2872 11178 : rem = (prec % (2 * limb_prec));
2873 11178 : end = (prec - rem) / limb_prec;
2874 11178 : cnt = 2 + CEIL (rem, limb_prec);
2875 11178 : idx = idx_first = create_loop (bitint_big_endian
2876 11178 : ? size_int (cnt - 2 + end - 1)
2877 : : size_zero_node, &idx_next);
2878 : }
2879 :
2880 23385 : basic_block edge_bb = NULL;
2881 23385 : if (eq_p)
2882 : {
2883 6552 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
2884 6552 : gsi_prev (&gsi);
2885 6552 : edge e = split_block (gsi_bb (gsi), gsi_stmt (gsi));
2886 6552 : edge_bb = e->src;
2887 6552 : if (kind == bitint_prec_large)
2888 7186 : m_gsi = gsi_end_bb (edge_bb);
2889 : }
2890 : else
2891 16833 : m_after_stmt = stmt;
2892 23385 : if (kind != bitint_prec_large)
2893 11178 : m_upwards_2limb = end;
2894 23385 : m_upwards = true;
2895 :
2896 23385 : bool separate_ext
2897 23385 : = (prec != (unsigned) TYPE_PRECISION (type)
2898 23385 : && (CEIL ((unsigned) TYPE_PRECISION (type), limb_prec)
2899 131 : > CEIL (prec, limb_prec)));
2900 23385 : bool zero_ms_limb = false;
2901 23385 : if (bitint_extended == bitint_ext_full
2902 0 : && !eq_p
2903 0 : && !nlhs
2904 0 : && abi_limb_prec > limb_prec
2905 23385 : && ((CEIL ((unsigned) TYPE_PRECISION (type), abi_limb_prec)
2906 0 : * abi_limb_prec / limb_prec)
2907 0 : > CEIL ((unsigned) TYPE_PRECISION (type), limb_prec)))
2908 : {
2909 0 : if (prec == (unsigned) TYPE_PRECISION (type))
2910 : {
2911 0 : sext = !TYPE_UNSIGNED (type);
2912 0 : separate_ext = true;
2913 : }
2914 0 : else if (TYPE_UNSIGNED (type) && sext)
2915 : zero_ms_limb = true;
2916 : else
2917 : separate_ext = true;
2918 : }
2919 23510 : unsigned dst_idx_off = 0;
2920 23385 : if (separate_ext && bitint_big_endian)
2921 0 : dst_idx_off = (CEIL ((unsigned) TYPE_PRECISION (type), limb_prec)
2922 0 : - CEIL (prec, limb_prec));
2923 :
2924 93759 : for (unsigned i = 0; i < cnt; i++)
2925 : {
2926 70374 : m_data_cnt = 0;
2927 70374 : if (kind == bitint_prec_large)
2928 37235 : idx = size_int (bitint_big_endian ? cnt - 1 - i : i);
2929 33139 : else if (i >= 2)
2930 10783 : idx = size_int (bitint_big_endian ? cnt - 1 - i : end + (i > 2));
2931 70374 : if (eq_p)
2932 : {
2933 19845 : rhs1 = handle_operand (cmp_op1, idx);
2934 19845 : tree rhs2 = handle_operand (cmp_op2, idx);
2935 19845 : g = gimple_build_cond (NE_EXPR, rhs1, rhs2, NULL_TREE, NULL_TREE);
2936 19845 : insert_before (g);
2937 19845 : edge e1 = split_block (gsi_bb (m_gsi), g);
2938 19845 : e1->flags = EDGE_FALSE_VALUE;
2939 19845 : edge e2 = make_edge (e1->src, gimple_bb (stmt), EDGE_TRUE_VALUE);
2940 19845 : e1->probability = profile_probability::unlikely ();
2941 19845 : e2->probability = e1->probability.invert ();
2942 19845 : if (i == 0)
2943 6552 : set_immediate_dominator (CDI_DOMINATORS, e2->dest, e2->src);
2944 19845 : m_gsi = gsi_after_labels (e1->dest);
2945 : }
2946 : else
2947 : {
2948 50529 : if (store_operand)
2949 25803 : rhs1 = handle_operand (store_operand, idx);
2950 : else
2951 24726 : rhs1 = handle_stmt (stmt, idx);
2952 50529 : if (!useless_type_conversion_p (m_limb_type, TREE_TYPE (rhs1)))
2953 12056 : rhs1 = add_cast (m_limb_type, rhs1);
2954 50529 : if (sext && i == cnt - 1)
2955 50529 : ext = rhs1;
2956 50529 : tree nidx = idx;
2957 50529 : HOST_WIDE_INT adj = bo_idx;
2958 50529 : if (bo_be_p)
2959 0 : adj += bo_last - (CEIL (prec, limb_prec) - 1);
2960 : else
2961 50529 : adj += dst_idx_off;
2962 50529 : if (adj)
2963 : {
2964 220 : if (tree_fits_uhwi_p (idx))
2965 116 : nidx = size_int (tree_to_uhwi (idx) + adj);
2966 : else
2967 : {
2968 104 : nidx = make_ssa_name (sizetype);
2969 104 : g = gimple_build_assign (nidx, PLUS_EXPR, idx,
2970 104 : size_int (adj));
2971 104 : insert_before (g);
2972 : }
2973 : }
2974 50529 : bool done = false;
2975 50529 : basic_block new_bb = NULL;
2976 : /* Handle stores into bit-fields. */
2977 50529 : if (bo_shift)
2978 : {
2979 455 : if (i == 0)
2980 : {
2981 143 : edge e2 = NULL;
2982 143 : if (kind != bitint_prec_large)
2983 : {
2984 112 : prepare_data_in_out (build_zero_cst (m_limb_type),
2985 : idx, &bf_next);
2986 112 : bf_next = m_data.pop ();
2987 112 : bf_cur = m_data.pop ();
2988 112 : g = gimple_build_cond (EQ_EXPR, idx,
2989 : bitint_big_endian
2990 0 : ? size_int (CEIL (prec,
2991 : limb_prec) - 1)
2992 : : size_zero_node,
2993 : NULL_TREE, NULL_TREE);
2994 112 : edge edge_true;
2995 112 : if_then_else (g, profile_probability::unlikely (),
2996 : edge_true, e2);
2997 112 : new_bb = e2->dest;
2998 : }
2999 143 : tree ftype
3000 143 : = build_nonstandard_integer_type (limb_prec - bo_shift, 1);
3001 286 : tree bfr = build_bit_field_ref (ftype, unshare_expr (nlhs),
3002 143 : limb_prec - bo_shift,
3003 : bitint_big_endian
3004 0 : ? (bo_idx + bo_last)
3005 0 : * limb_prec
3006 143 : : bo_idx * limb_prec
3007 : + bo_bit);
3008 143 : tree t = add_cast (ftype, rhs1);
3009 143 : g = gimple_build_assign (bfr, t);
3010 143 : insert_before (g);
3011 143 : if (eh)
3012 : {
3013 0 : maybe_duplicate_eh_stmt (g, stmt);
3014 0 : if (eh_pad)
3015 : {
3016 0 : edge e = split_block (gsi_bb (m_gsi), g);
3017 0 : m_gsi = gsi_after_labels (e->dest);
3018 0 : add_eh_edge (e->src,
3019 : find_edge (gimple_bb (stmt), eh_pad));
3020 : }
3021 : }
3022 143 : if (kind == bitint_prec_large)
3023 : {
3024 : bf_cur = rhs1;
3025 : done = true;
3026 : }
3027 112 : else if (e2)
3028 112 : m_gsi = gsi_after_labels (e2->src);
3029 : }
3030 112 : if (!done)
3031 : {
3032 424 : tree t1 = make_ssa_name (m_limb_type);
3033 424 : tree t2 = make_ssa_name (m_limb_type);
3034 424 : tree t3 = make_ssa_name (m_limb_type);
3035 424 : g = gimple_build_assign (t1, RSHIFT_EXPR, bf_cur,
3036 : build_int_cst (unsigned_type_node,
3037 424 : limb_prec
3038 424 : - bo_shift));
3039 424 : insert_before (g);
3040 424 : g = gimple_build_assign (t2, LSHIFT_EXPR, rhs1,
3041 : build_int_cst (unsigned_type_node,
3042 424 : bo_shift));
3043 424 : insert_before (g);
3044 424 : bf_cur = rhs1;
3045 424 : g = gimple_build_assign (t3, BIT_IOR_EXPR, t1, t2);
3046 424 : insert_before (g);
3047 424 : rhs1 = t3;
3048 424 : if (bf_next && i == 1)
3049 : {
3050 112 : g = gimple_build_assign (bf_next, bf_cur);
3051 112 : insert_before (g);
3052 : }
3053 : }
3054 : }
3055 31 : if (!done)
3056 : {
3057 : /* Handle bit-field access to partial last limb if needed. */
3058 50498 : if (nlhs
3059 480 : && i == cnt - 1
3060 162 : && !separate_ext
3061 138 : && tree_fits_uhwi_p (idx))
3062 : {
3063 123 : unsigned int tprec = TYPE_PRECISION (type);
3064 123 : unsigned int rprec = (tprec - 1) % limb_prec + 1;
3065 123 : if (rprec + bo_shift < (unsigned) limb_prec)
3066 : {
3067 46 : tree ftype
3068 46 : = build_nonstandard_integer_type (rprec + bo_shift, 1);
3069 46 : tree bfr
3070 92 : = build_bit_field_ref (ftype, unshare_expr (nlhs),
3071 : rprec + bo_shift,
3072 : bitint_big_endian
3073 0 : ? bo_idx * limb_prec + bo_bit
3074 46 : : (bo_idx + tprec / limb_prec)
3075 46 : * limb_prec);
3076 46 : tree t = add_cast (ftype, rhs1);
3077 46 : g = gimple_build_assign (bfr, t);
3078 46 : done = true;
3079 46 : bf_cur = NULL_TREE;
3080 : }
3081 77 : else if (rprec + bo_shift == (unsigned) limb_prec)
3082 50452 : bf_cur = NULL_TREE;
3083 : }
3084 : /* Otherwise, stores to any other lhs. */
3085 46 : if (!done)
3086 : {
3087 50886 : tree l = limb_access (nlhs ? NULL_TREE : lhs_type,
3088 : nlhs ? nlhs : lhs, nidx, true);
3089 50452 : g = gimple_build_assign (l, rhs1);
3090 : }
3091 50498 : insert_before (g);
3092 50498 : if (eh)
3093 : {
3094 6 : maybe_duplicate_eh_stmt (g, stmt);
3095 6 : if (eh_pad)
3096 : {
3097 6 : edge e = split_block (gsi_bb (m_gsi), g);
3098 6 : m_gsi = gsi_after_labels (e->dest);
3099 6 : add_eh_edge (e->src,
3100 : find_edge (gimple_bb (stmt), eh_pad));
3101 : }
3102 : }
3103 50498 : if (new_bb)
3104 112 : m_gsi = gsi_after_labels (new_bb);
3105 : }
3106 : }
3107 70374 : m_first = false;
3108 70374 : if (kind == bitint_prec_huge && i <= 1)
3109 : {
3110 22356 : if (i == 0)
3111 : {
3112 11178 : idx = make_ssa_name (sizetype);
3113 11178 : g = gimple_build_assign (idx, PLUS_EXPR, idx_first,
3114 : bitint_big_endian
3115 0 : ? size_int (-1) : size_one_node);
3116 11178 : insert_before (g);
3117 : }
3118 : else
3119 : {
3120 11178 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx_first,
3121 22356 : size_int (bitint_big_endian ? -2 : 2));
3122 11178 : insert_before (g);
3123 11178 : if (bitint_big_endian)
3124 0 : g = gimple_build_cond (NE_EXPR, idx_first, size_int (cnt - 1),
3125 : NULL_TREE, NULL_TREE);
3126 : else
3127 11178 : g = gimple_build_cond (NE_EXPR, idx_next, size_int (end),
3128 : NULL_TREE, NULL_TREE);
3129 11178 : insert_before (g);
3130 11178 : if (eq_p)
3131 2959 : m_gsi = gsi_after_labels (edge_bb);
3132 : else
3133 8219 : m_gsi = gsi_for_stmt (stmt);
3134 11178 : m_bb = NULL;
3135 : }
3136 : }
3137 : }
3138 :
3139 23385 : if (separate_ext)
3140 : {
3141 125 : if (sext)
3142 : {
3143 51 : ext = add_cast (signed_type_for (m_limb_type), ext);
3144 102 : tree lpm1 = build_int_cst (unsigned_type_node,
3145 51 : limb_prec - 1);
3146 51 : tree n = make_ssa_name (TREE_TYPE (ext));
3147 51 : g = gimple_build_assign (n, RSHIFT_EXPR, ext, lpm1);
3148 51 : insert_before (g);
3149 51 : ext = add_cast (m_limb_type, n);
3150 : }
3151 : else
3152 74 : ext = build_zero_cst (m_limb_type);
3153 125 : kind = bitint_precision_kind (type);
3154 125 : unsigned start = CEIL (prec, limb_prec);
3155 125 : prec = TYPE_PRECISION (type);
3156 125 : if (bitint_extended == bitint_ext_full
3157 0 : && !nlhs
3158 0 : && !zero_ms_limb
3159 0 : && abi_limb_prec > limb_prec)
3160 : {
3161 0 : prec = CEIL (prec, abi_limb_prec) * abi_limb_prec;
3162 0 : kind = bitint_precision_kind (prec);
3163 : }
3164 125 : unsigned total = CEIL (prec, limb_prec);
3165 125 : idx = idx_first = idx_next = NULL_TREE;
3166 125 : if (prec <= (start + 2 + (bo_shift != 0)) * limb_prec)
3167 : kind = bitint_prec_large;
3168 80 : if (kind == bitint_prec_large)
3169 45 : cnt = total - start;
3170 : else
3171 : {
3172 80 : rem = prec % limb_prec;
3173 80 : end = (prec - rem) / limb_prec;
3174 152 : cnt = (bo_shift != 0) + 1 + (rem != 0);
3175 : }
3176 125 : if (bitint_big_endian && bo_shift != 0 && (prec % limb_prec) == 0)
3177 0 : ++total;
3178 321 : for (unsigned i = 0; i < cnt; i++)
3179 : {
3180 196 : if (kind == bitint_prec_large || (i == 0 && bo_shift != 0))
3181 62 : idx = size_int (bo_idx
3182 : + (bitint_big_endian
3183 : ? total - 1 - start - i : start + i));
3184 134 : else if (i == cnt - 1 && rem != 0)
3185 108 : idx = size_int (bo_idx + (bitint_big_endian ? 0 : end));
3186 80 : else if (i == (bo_shift != 0))
3187 80 : idx = create_loop (size_int (bo_idx
3188 : + (bitint_big_endian
3189 : ? total - 1 - start - i
3190 : : start + i)), &idx_next);
3191 196 : rhs1 = ext;
3192 196 : if (bf_cur != NULL_TREE && bf_cur != ext)
3193 : {
3194 20 : tree t1 = make_ssa_name (m_limb_type);
3195 20 : g = gimple_build_assign (t1, RSHIFT_EXPR, bf_cur,
3196 : build_int_cst (unsigned_type_node,
3197 20 : limb_prec - bo_shift));
3198 20 : insert_before (g);
3199 20 : if (integer_zerop (ext))
3200 : rhs1 = t1;
3201 : else
3202 : {
3203 10 : tree t2 = make_ssa_name (m_limb_type);
3204 10 : rhs1 = make_ssa_name (m_limb_type);
3205 10 : g = gimple_build_assign (t2, LSHIFT_EXPR, ext,
3206 : build_int_cst (unsigned_type_node,
3207 10 : bo_shift));
3208 10 : insert_before (g);
3209 10 : g = gimple_build_assign (rhs1, BIT_IOR_EXPR, t1, t2);
3210 10 : insert_before (g);
3211 : }
3212 : bf_cur = ext;
3213 : }
3214 196 : bool done = false;
3215 : /* Handle bit-field access to partial last limb if needed. */
3216 196 : if (nlhs && i == cnt - 1)
3217 : {
3218 24 : unsigned int tprec = TYPE_PRECISION (type);
3219 24 : unsigned int rprec = (tprec - 1) % limb_prec + 1;
3220 24 : if (rprec + bo_shift < (unsigned) limb_prec)
3221 : {
3222 12 : tree ftype
3223 12 : = build_nonstandard_integer_type (rprec + bo_shift, 1);
3224 12 : tree bfr
3225 24 : = build_bit_field_ref (ftype, unshare_expr (nlhs),
3226 : rprec + bo_shift,
3227 : bitint_big_endian
3228 0 : ? bo_idx * limb_prec + bo_bit
3229 12 : : (bo_idx + tprec / limb_prec)
3230 12 : * limb_prec);
3231 12 : tree t = add_cast (ftype, rhs1);
3232 12 : g = gimple_build_assign (bfr, t);
3233 12 : done = true;
3234 12 : bf_cur = NULL_TREE;
3235 : }
3236 12 : else if (rprec + bo_shift == (unsigned) limb_prec)
3237 : bf_cur = NULL_TREE;
3238 : }
3239 : /* Otherwise, stores to any other lhs. */
3240 12 : if (!done)
3241 : {
3242 214 : tree l = limb_access (nlhs ? NULL_TREE : lhs_type,
3243 : nlhs ? nlhs : lhs, idx, true);
3244 :
3245 184 : if (bitint_extended
3246 0 : && sext
3247 0 : && TYPE_UNSIGNED (lhs_type)
3248 0 : && tree_fits_uhwi_p (idx)
3249 184 : && !nlhs)
3250 : {
3251 0 : rhs1 = add_cast (limb_access_type (lhs_type, idx), rhs1);
3252 0 : rhs1 = add_cast (TREE_TYPE (l), rhs1);
3253 : }
3254 :
3255 184 : g = gimple_build_assign (l, rhs1);
3256 : }
3257 196 : insert_before (g);
3258 196 : if (eh)
3259 : {
3260 0 : maybe_duplicate_eh_stmt (g, stmt);
3261 0 : if (eh_pad)
3262 : {
3263 0 : edge e = split_block (gsi_bb (m_gsi), g);
3264 0 : m_gsi = gsi_after_labels (e->dest);
3265 0 : add_eh_edge (e->src, find_edge (gimple_bb (stmt), eh_pad));
3266 : }
3267 : }
3268 196 : if (kind == bitint_prec_huge && i == (bo_shift != 0))
3269 : {
3270 80 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
3271 : bitint_big_endian
3272 0 : ? size_int (-1) : size_one_node);
3273 80 : insert_before (g);
3274 : /* For big-endian, if bo_idx + total - 1 - end is all ones, then
3275 : compare idx (which is equal to idx_next + 1) against 0
3276 : instead. */
3277 80 : if (bitint_big_endian && bo_idx + total - end == 0)
3278 0 : g = gimple_build_cond (NE_EXPR, idx, size_zero_node,
3279 : NULL_TREE, NULL_TREE);
3280 : else
3281 80 : g = gimple_build_cond (NE_EXPR, idx_next,
3282 80 : size_int (bo_idx
3283 : + (bitint_big_endian
3284 : ? total - 1 - end
3285 : : end)),
3286 : NULL_TREE, NULL_TREE);
3287 80 : insert_before (g);
3288 80 : m_gsi = gsi_for_stmt (stmt);
3289 80 : m_bb = NULL;
3290 : }
3291 : }
3292 : }
3293 23385 : if (bf_cur != NULL_TREE)
3294 : {
3295 72 : unsigned int tprec = TYPE_PRECISION (type);
3296 72 : unsigned int rprec = (tprec + bo_shift) % limb_prec;
3297 72 : tree ftype = build_nonstandard_integer_type (rprec, 1);
3298 144 : tree bfr = build_bit_field_ref (ftype, unshare_expr (nlhs),
3299 : rprec,
3300 : bitint_big_endian
3301 0 : ? bo_idx * limb_prec + bo_bit
3302 72 : : (bo_idx + (tprec + bo_bit) / limb_prec)
3303 : * limb_prec);
3304 72 : rhs1 = bf_cur;
3305 72 : if (bf_cur != ext)
3306 : {
3307 64 : rhs1 = make_ssa_name (TREE_TYPE (rhs1));
3308 64 : g = gimple_build_assign (rhs1, RSHIFT_EXPR, bf_cur,
3309 : build_int_cst (unsigned_type_node,
3310 64 : limb_prec - bo_shift));
3311 64 : insert_before (g);
3312 : }
3313 72 : rhs1 = add_cast (ftype, rhs1);
3314 72 : g = gimple_build_assign (bfr, rhs1);
3315 72 : insert_before (g);
3316 72 : if (eh)
3317 : {
3318 0 : maybe_duplicate_eh_stmt (g, stmt);
3319 0 : if (eh_pad)
3320 : {
3321 0 : edge e = split_block (gsi_bb (m_gsi), g);
3322 0 : m_gsi = gsi_after_labels (e->dest);
3323 0 : add_eh_edge (e->src, find_edge (gimple_bb (stmt), eh_pad));
3324 : }
3325 : }
3326 : }
3327 23385 : if (zero_ms_limb)
3328 : {
3329 0 : tree p2 = build_int_cst (sizetype,
3330 0 : CEIL ((unsigned) TYPE_PRECISION (type),
3331 : abi_limb_prec)
3332 0 : * abi_limb_prec / limb_prec - 1);
3333 0 : tree l = limb_access (lhs_type, lhs, p2, true);
3334 0 : g = gimple_build_assign (l, build_zero_cst (m_limb_type));
3335 0 : insert_before (g);
3336 0 : if (eh)
3337 : {
3338 0 : maybe_duplicate_eh_stmt (g, stmt);
3339 0 : if (eh_pad)
3340 : {
3341 0 : edge e = split_block (gsi_bb (m_gsi), g);
3342 0 : m_gsi = gsi_after_labels (e->dest);
3343 0 : add_eh_edge (e->src, find_edge (gimple_bb (stmt), eh_pad));
3344 : }
3345 : }
3346 : }
3347 :
3348 23385 : if (gimple_store_p (stmt))
3349 : {
3350 8524 : unlink_stmt_vdef (stmt);
3351 17048 : release_ssa_name (gimple_vdef (stmt));
3352 8524 : gsi_remove (&m_gsi, true);
3353 : }
3354 23385 : if (eq_p)
3355 : {
3356 6552 : lhs = make_ssa_name (boolean_type_node);
3357 6552 : basic_block bb = gimple_bb (stmt);
3358 6552 : gphi *phi = create_phi_node (lhs, bb);
3359 6552 : edge e = find_edge (gsi_bb (m_gsi), bb);
3360 6552 : unsigned int n = EDGE_COUNT (bb->preds);
3361 32949 : for (unsigned int i = 0; i < n; i++)
3362 : {
3363 26397 : edge e2 = EDGE_PRED (bb, i);
3364 26397 : add_phi_arg (phi, e == e2 ? boolean_true_node : boolean_false_node,
3365 : e2, UNKNOWN_LOCATION);
3366 : }
3367 6552 : cmp_code = cmp_code == EQ_EXPR ? NE_EXPR : EQ_EXPR;
3368 6552 : return lhs;
3369 : }
3370 : else
3371 : return NULL_TREE;
3372 : }
3373 :
3374 : /* Handle a large/huge _BitInt comparison statement STMT other than
3375 : EQ_EXPR/NE_EXPR. CMP_CODE, CMP_OP1 and CMP_OP2 meaning is like in
3376 : lower_mergeable_stmt. The {GT,GE,LT,LE}_EXPR comparisons are
3377 : lowered by iteration from the most significant limb downwards to
3378 : the least significant one, for large _BitInt in straight line code,
3379 : otherwise with most significant limb handled in
3380 : straight line code followed by a loop handling one limb at a time.
3381 : Comparisons with unsigned huge _BitInt with precisions which are
3382 : multiples of limb precision can use just the loop and don't need to
3383 : handle most significant limb before the loop. The loop or straight
3384 : line code jumps to final basic block if a particular pair of limbs
3385 : is not equal. */
3386 :
3387 : tree
3388 722 : bitint_large_huge::lower_comparison_stmt (gimple *stmt, tree_code &cmp_code,
3389 : tree cmp_op1, tree cmp_op2)
3390 : {
3391 722 : tree type = TREE_TYPE (cmp_op1);
3392 722 : gcc_assert (BITINT_TYPE_P (type));
3393 722 : bitint_prec_kind kind = bitint_precision_kind (type);
3394 722 : gcc_assert (kind >= bitint_prec_large);
3395 722 : gimple *g;
3396 722 : if (!TYPE_UNSIGNED (type)
3397 441 : && integer_zerop (cmp_op2)
3398 750 : && (cmp_code == GE_EXPR || cmp_code == LT_EXPR))
3399 : {
3400 28 : unsigned end = CEIL ((unsigned) TYPE_PRECISION (type), limb_prec) - 1;
3401 56 : tree idx = size_int (bitint_big_endian ? 0 : end);
3402 28 : m_data_cnt = 0;
3403 28 : tree rhs1 = handle_operand (cmp_op1, idx);
3404 28 : if (TYPE_UNSIGNED (TREE_TYPE (rhs1)))
3405 : {
3406 24 : tree stype = signed_type_for (TREE_TYPE (rhs1));
3407 24 : rhs1 = add_cast (stype, rhs1);
3408 : }
3409 28 : tree lhs = make_ssa_name (boolean_type_node);
3410 28 : g = gimple_build_assign (lhs, cmp_code, rhs1,
3411 28 : build_zero_cst (TREE_TYPE (rhs1)));
3412 28 : insert_before (g);
3413 28 : cmp_code = NE_EXPR;
3414 28 : return lhs;
3415 : }
3416 :
3417 694 : unsigned cnt, rem = 0, end = 0;
3418 694 : tree idx = NULL_TREE, idx_next = NULL_TREE;
3419 694 : if (kind == bitint_prec_large)
3420 377 : cnt = CEIL ((unsigned) TYPE_PRECISION (type), limb_prec);
3421 : else
3422 : {
3423 317 : rem = ((unsigned) TYPE_PRECISION (type) % limb_prec);
3424 317 : if (rem == 0 && !TYPE_UNSIGNED (type))
3425 : rem = limb_prec;
3426 317 : end = ((unsigned) TYPE_PRECISION (type) - rem) / limb_prec;
3427 317 : cnt = 1 + (rem != 0);
3428 : }
3429 :
3430 694 : basic_block edge_bb = NULL;
3431 694 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
3432 694 : gsi_prev (&gsi);
3433 694 : edge e = split_block (gsi_bb (gsi), gsi_stmt (gsi));
3434 694 : edge_bb = e->src;
3435 694 : m_gsi = gsi_end_bb (edge_bb);
3436 :
3437 694 : edge *edges = XALLOCAVEC (edge, cnt * 2);
3438 2519 : for (unsigned i = 0; i < cnt; i++)
3439 : {
3440 1825 : m_data_cnt = 0;
3441 1825 : if (kind == bitint_prec_large)
3442 1246 : idx = size_int (bitint_big_endian ? i : cnt - i - 1);
3443 579 : else if (i == cnt - 1)
3444 317 : idx = create_loop (size_int (bitint_big_endian ? cnt - 1 : end - 1),
3445 : &idx_next);
3446 : else
3447 524 : idx = size_int (bitint_big_endian ? 0 : end);
3448 1825 : tree rhs1 = handle_operand (cmp_op1, idx);
3449 1825 : tree rhs2 = handle_operand (cmp_op2, idx);
3450 1825 : if (i == 0
3451 694 : && !TYPE_UNSIGNED (type)
3452 2238 : && TYPE_UNSIGNED (TREE_TYPE (rhs1)))
3453 : {
3454 112 : tree stype = signed_type_for (TREE_TYPE (rhs1));
3455 112 : rhs1 = add_cast (stype, rhs1);
3456 112 : rhs2 = add_cast (stype, rhs2);
3457 : }
3458 1825 : g = gimple_build_cond (GT_EXPR, rhs1, rhs2, NULL_TREE, NULL_TREE);
3459 1825 : insert_before (g);
3460 1825 : edge e1 = split_block (gsi_bb (m_gsi), g);
3461 1825 : e1->flags = EDGE_FALSE_VALUE;
3462 1825 : edge e2 = make_edge (e1->src, gimple_bb (stmt), EDGE_TRUE_VALUE);
3463 1825 : e1->probability = profile_probability::likely ();
3464 1825 : e2->probability = e1->probability.invert ();
3465 1825 : if (i == 0)
3466 694 : set_immediate_dominator (CDI_DOMINATORS, e2->dest, e2->src);
3467 1825 : m_gsi = gsi_after_labels (e1->dest);
3468 1825 : edges[2 * i] = e2;
3469 1825 : g = gimple_build_cond (LT_EXPR, rhs1, rhs2, NULL_TREE, NULL_TREE);
3470 1825 : insert_before (g);
3471 1825 : e1 = split_block (gsi_bb (m_gsi), g);
3472 1825 : e1->flags = EDGE_FALSE_VALUE;
3473 1825 : e2 = make_edge (e1->src, gimple_bb (stmt), EDGE_TRUE_VALUE);
3474 1825 : e1->probability = profile_probability::unlikely ();
3475 1825 : e2->probability = e1->probability.invert ();
3476 1825 : m_gsi = gsi_after_labels (e1->dest);
3477 1825 : edges[2 * i + 1] = e2;
3478 1825 : m_first = false;
3479 1825 : if (kind == bitint_prec_huge && i == cnt - 1)
3480 : {
3481 634 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
3482 : bitint_big_endian ? size_one_node
3483 317 : : size_int (-1));
3484 317 : insert_before (g);
3485 317 : g = gimple_build_cond (NE_EXPR, idx,
3486 : bitint_big_endian
3487 0 : ? size_int (end - 1 + (cnt != 1))
3488 : : size_zero_node,
3489 : NULL_TREE, NULL_TREE);
3490 317 : insert_before (g);
3491 317 : edge true_edge, false_edge;
3492 317 : extract_true_false_edges_from_block (gsi_bb (m_gsi),
3493 : &true_edge, &false_edge);
3494 317 : m_gsi = gsi_after_labels (false_edge->dest);
3495 317 : m_bb = NULL;
3496 : }
3497 : }
3498 :
3499 694 : tree lhs = make_ssa_name (boolean_type_node);
3500 694 : basic_block bb = gimple_bb (stmt);
3501 694 : gphi *phi = create_phi_node (lhs, bb);
3502 5038 : for (unsigned int i = 0; i < cnt * 2; i++)
3503 : {
3504 3650 : tree val = ((cmp_code == GT_EXPR || cmp_code == GE_EXPR)
3505 3650 : ^ (i & 1)) ? boolean_true_node : boolean_false_node;
3506 3650 : add_phi_arg (phi, val, edges[i], UNKNOWN_LOCATION);
3507 : }
3508 694 : add_phi_arg (phi, (cmp_code == GE_EXPR || cmp_code == LE_EXPR)
3509 : ? boolean_true_node : boolean_false_node,
3510 : find_edge (gsi_bb (m_gsi), bb), UNKNOWN_LOCATION);
3511 694 : cmp_code = NE_EXPR;
3512 694 : return lhs;
3513 : }
3514 :
3515 : /* Lower large/huge _BitInt left and right shift except for left
3516 : shift by < limb_prec constant. */
3517 :
3518 : void
3519 584 : bitint_large_huge::lower_shift_stmt (tree obj, gimple *stmt)
3520 : {
3521 584 : tree rhs1 = gimple_assign_rhs1 (stmt);
3522 584 : tree lhs = gimple_assign_lhs (stmt);
3523 584 : tree_code rhs_code = gimple_assign_rhs_code (stmt);
3524 584 : tree type = TREE_TYPE (rhs1);
3525 584 : gimple *final_stmt = gsi_stmt (m_gsi);
3526 584 : gcc_assert (BITINT_TYPE_P (type)
3527 : && bitint_precision_kind (type) >= bitint_prec_large);
3528 584 : int prec = TYPE_PRECISION (type);
3529 584 : tree n = gimple_assign_rhs2 (stmt), n1, n2, n3, n4;
3530 584 : gimple *g;
3531 584 : if (obj == NULL_TREE)
3532 : {
3533 465 : int part = var_to_partition (m_map, lhs);
3534 465 : gcc_assert (m_vars[part] != NULL_TREE);
3535 : obj = m_vars[part];
3536 : }
3537 : /* Preparation code common for both left and right shifts.
3538 : unsigned n1 = n % limb_prec;
3539 : size_t n2 = n / limb_prec;
3540 : size_t n3 = n1 != 0;
3541 : unsigned n4 = (limb_prec - n1) % limb_prec;
3542 : (for power of 2 limb_prec n4 can be -n1 & limb_prec). */
3543 584 : if (TREE_CODE (n) == INTEGER_CST)
3544 : {
3545 249 : tree lp = build_int_cst (TREE_TYPE (n), limb_prec);
3546 249 : n1 = int_const_binop (TRUNC_MOD_EXPR, n, lp);
3547 249 : n2 = fold_convert (sizetype, int_const_binop (TRUNC_DIV_EXPR, n, lp));
3548 249 : n3 = size_int (!integer_zerop (n1));
3549 249 : n4 = int_const_binop (TRUNC_MOD_EXPR,
3550 249 : int_const_binop (MINUS_EXPR, lp, n1), lp);
3551 : }
3552 : else
3553 : {
3554 335 : n1 = make_ssa_name (TREE_TYPE (n));
3555 335 : n2 = make_ssa_name (sizetype);
3556 335 : n3 = make_ssa_name (sizetype);
3557 335 : n4 = make_ssa_name (TREE_TYPE (n));
3558 335 : if (pow2p_hwi (limb_prec))
3559 : {
3560 335 : tree lpm1 = build_int_cst (TREE_TYPE (n), limb_prec - 1);
3561 335 : g = gimple_build_assign (n1, BIT_AND_EXPR, n, lpm1);
3562 335 : insert_before (g);
3563 989 : g = gimple_build_assign (useless_type_conversion_p (sizetype,
3564 335 : TREE_TYPE (n))
3565 319 : ? n2 : make_ssa_name (TREE_TYPE (n)),
3566 : RSHIFT_EXPR, n,
3567 335 : build_int_cst (TREE_TYPE (n),
3568 670 : exact_log2 (limb_prec)));
3569 335 : insert_before (g);
3570 335 : if (gimple_assign_lhs (g) != n2)
3571 : {
3572 319 : g = gimple_build_assign (n2, NOP_EXPR, gimple_assign_lhs (g));
3573 319 : insert_before (g);
3574 : }
3575 335 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (n)),
3576 : NEGATE_EXPR, n1);
3577 335 : insert_before (g);
3578 335 : g = gimple_build_assign (n4, BIT_AND_EXPR, gimple_assign_lhs (g),
3579 : lpm1);
3580 335 : insert_before (g);
3581 : }
3582 : else
3583 : {
3584 0 : tree lp = build_int_cst (TREE_TYPE (n), limb_prec);
3585 0 : g = gimple_build_assign (n1, TRUNC_MOD_EXPR, n, lp);
3586 0 : insert_before (g);
3587 0 : g = gimple_build_assign (useless_type_conversion_p (sizetype,
3588 0 : TREE_TYPE (n))
3589 0 : ? n2 : make_ssa_name (TREE_TYPE (n)),
3590 : TRUNC_DIV_EXPR, n, lp);
3591 0 : insert_before (g);
3592 0 : if (gimple_assign_lhs (g) != n2)
3593 : {
3594 0 : g = gimple_build_assign (n2, NOP_EXPR, gimple_assign_lhs (g));
3595 0 : insert_before (g);
3596 : }
3597 0 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (n)),
3598 : MINUS_EXPR, lp, n1);
3599 0 : insert_before (g);
3600 0 : g = gimple_build_assign (n4, TRUNC_MOD_EXPR, gimple_assign_lhs (g),
3601 : lp);
3602 0 : insert_before (g);
3603 : }
3604 335 : g = gimple_build_assign (make_ssa_name (boolean_type_node), NE_EXPR, n1,
3605 335 : build_zero_cst (TREE_TYPE (n)));
3606 335 : insert_before (g);
3607 335 : g = gimple_build_assign (n3, NOP_EXPR, gimple_assign_lhs (g));
3608 335 : insert_before (g);
3609 : }
3610 1168 : tree p = build_int_cst (sizetype,
3611 584 : prec / limb_prec - (prec % limb_prec == 0));
3612 584 : if (rhs_code == RSHIFT_EXPR)
3613 : {
3614 : /* Lower
3615 : dst = src >> n;
3616 : as
3617 : unsigned n1 = n % limb_prec;
3618 : size_t n2 = n / limb_prec;
3619 : size_t n3 = n1 != 0;
3620 : unsigned n4 = (limb_prec - n1) % limb_prec;
3621 : size_t idx;
3622 : size_t p = prec / limb_prec - (prec % limb_prec == 0);
3623 : int signed_p = (typeof (src) -1) < 0;
3624 : for (idx = n2; idx < ((!signed_p && (prec % limb_prec == 0))
3625 : ? p : p - n3); ++idx)
3626 : dst[idx - n2] = (src[idx] >> n1) | (src[idx + n3] << n4);
3627 : limb_type ext;
3628 : if (prec % limb_prec == 0)
3629 : ext = src[p];
3630 : else if (signed_p)
3631 : ext = ((signed limb_type) (src[p] << (limb_prec
3632 : - (prec % limb_prec))))
3633 : >> (limb_prec - (prec % limb_prec));
3634 : else
3635 : ext = src[p] & (((limb_type) 1 << (prec % limb_prec)) - 1);
3636 : if (!signed_p && (prec % limb_prec == 0))
3637 : ;
3638 : else if (idx < prec / 64)
3639 : {
3640 : dst[idx - n2] = (src[idx] >> n1) | (ext << n4);
3641 : ++idx;
3642 : }
3643 : idx -= n2;
3644 : if (signed_p)
3645 : {
3646 : dst[idx] = ((signed limb_type) ext) >> n1;
3647 : ext = ((signed limb_type) ext) >> (limb_prec - 1);
3648 : }
3649 : else
3650 : {
3651 : dst[idx] = ext >> n1;
3652 : ext = 0;
3653 : }
3654 : for (++idx; idx <= p; ++idx)
3655 : dst[idx] = ext; */
3656 387 : tree pmn3;
3657 387 : if (TYPE_UNSIGNED (type) && prec % limb_prec == 0)
3658 100 : pmn3 = bitint_big_endian ? size_zero_node : p;
3659 287 : else if (bitint_big_endian)
3660 : pmn3 = n3;
3661 287 : else if (TREE_CODE (n3) == INTEGER_CST)
3662 114 : pmn3 = int_const_binop (MINUS_EXPR, p, n3);
3663 : else
3664 : {
3665 173 : pmn3 = make_ssa_name (sizetype);
3666 173 : g = gimple_build_assign (pmn3, MINUS_EXPR, p, n3);
3667 173 : insert_before (g);
3668 : }
3669 387 : tree pmn2 = NULL_TREE;
3670 387 : if (bitint_big_endian)
3671 : {
3672 0 : if (TREE_CODE (n2) == INTEGER_CST)
3673 0 : pmn2 = int_const_binop (MINUS_EXPR, p, n2);
3674 : else
3675 : {
3676 0 : pmn2 = make_ssa_name (sizetype);
3677 0 : g = gimple_build_assign (pmn2, MINUS_EXPR, p, n2);
3678 0 : insert_before (g);
3679 : }
3680 0 : g = gimple_build_cond (GT_EXPR, pmn2, pmn3, NULL_TREE, NULL_TREE);
3681 : }
3682 : else
3683 387 : g = gimple_build_cond (LT_EXPR, n2, pmn3, NULL_TREE, NULL_TREE);
3684 387 : edge edge_true, edge_false;
3685 387 : if_then (g, profile_probability::likely (), edge_true, edge_false);
3686 387 : tree idx_next;
3687 774 : tree idx = create_loop (bitint_big_endian ? pmn2 : n2, &idx_next);
3688 387 : tree idxmn2 = make_ssa_name (sizetype);
3689 387 : tree idxpn3 = make_ssa_name (sizetype);
3690 774 : g = gimple_build_assign (idxmn2,
3691 : bitint_big_endian ? PLUS_EXPR : MINUS_EXPR,
3692 : idx, n2);
3693 387 : insert_before (g);
3694 774 : g = gimple_build_assign (idxpn3,
3695 : bitint_big_endian ? MINUS_EXPR : PLUS_EXPR,
3696 : idx, n3);
3697 387 : insert_before (g);
3698 387 : m_data_cnt = 0;
3699 387 : tree t1 = handle_operand (rhs1, idx);
3700 387 : m_first = false;
3701 387 : g = gimple_build_assign (make_ssa_name (m_limb_type),
3702 : RSHIFT_EXPR, t1, n1);
3703 387 : insert_before (g);
3704 387 : t1 = gimple_assign_lhs (g);
3705 387 : if (!integer_zerop (n3))
3706 : {
3707 299 : m_data_cnt = 0;
3708 299 : tree t2 = handle_operand (rhs1, idxpn3);
3709 299 : g = gimple_build_assign (make_ssa_name (m_limb_type),
3710 : LSHIFT_EXPR, t2, n4);
3711 299 : insert_before (g);
3712 299 : t2 = gimple_assign_lhs (g);
3713 299 : g = gimple_build_assign (make_ssa_name (m_limb_type),
3714 : BIT_IOR_EXPR, t1, t2);
3715 299 : insert_before (g);
3716 299 : t1 = gimple_assign_lhs (g);
3717 : }
3718 387 : tree l = limb_access (TREE_TYPE (lhs), obj, idxmn2, true);
3719 387 : g = gimple_build_assign (l, t1);
3720 387 : insert_before (g);
3721 387 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
3722 0 : bitint_big_endian ? size_int (-1)
3723 : : size_one_node);
3724 387 : insert_before (g);
3725 774 : g = gimple_build_cond (bitint_big_endian ? GT_EXPR : LT_EXPR,
3726 : idx_next, pmn3, NULL_TREE, NULL_TREE);
3727 387 : insert_before (g);
3728 387 : idx = make_ssa_name (sizetype);
3729 387 : m_gsi = gsi_for_stmt (final_stmt);
3730 387 : gphi *phi = create_phi_node (idx, gsi_bb (m_gsi));
3731 387 : edge_false = find_edge (edge_false->src, gsi_bb (m_gsi));
3732 387 : edge_true = EDGE_PRED (gsi_bb (m_gsi),
3733 : EDGE_PRED (gsi_bb (m_gsi), 0) == edge_false);
3734 774 : add_phi_arg (phi, bitint_big_endian ? pmn2 : n2, edge_false,
3735 : UNKNOWN_LOCATION);
3736 387 : add_phi_arg (phi, idx_next, edge_true, UNKNOWN_LOCATION);
3737 387 : m_data_cnt = 0;
3738 387 : tree ms = handle_operand (rhs1, bitint_big_endian ? size_zero_node : p);
3739 387 : tree ext = ms;
3740 387 : if (!types_compatible_p (TREE_TYPE (ms), m_limb_type))
3741 230 : ext = add_cast (m_limb_type, ms);
3742 587 : if (!(TYPE_UNSIGNED (type) && prec % limb_prec == 0)
3743 487 : && !integer_zerop (n3))
3744 : {
3745 249 : if (bitint_big_endian)
3746 0 : g = gimple_build_cond (GT_EXPR, idx, size_zero_node,
3747 : NULL_TREE, NULL_TREE);
3748 : else
3749 249 : g = gimple_build_cond (LT_EXPR, idx, p, NULL_TREE, NULL_TREE);
3750 249 : if_then (g, profile_probability::likely (), edge_true, edge_false);
3751 249 : m_data_cnt = 0;
3752 249 : t1 = handle_operand (rhs1, idx);
3753 249 : g = gimple_build_assign (make_ssa_name (m_limb_type),
3754 : RSHIFT_EXPR, t1, n1);
3755 249 : insert_before (g);
3756 249 : t1 = gimple_assign_lhs (g);
3757 249 : g = gimple_build_assign (make_ssa_name (m_limb_type),
3758 : LSHIFT_EXPR, ext, n4);
3759 249 : insert_before (g);
3760 249 : tree t2 = gimple_assign_lhs (g);
3761 249 : g = gimple_build_assign (make_ssa_name (m_limb_type),
3762 : BIT_IOR_EXPR, t1, t2);
3763 249 : insert_before (g);
3764 249 : t1 = gimple_assign_lhs (g);
3765 249 : idxmn2 = make_ssa_name (sizetype);
3766 498 : g = gimple_build_assign (idxmn2, bitint_big_endian
3767 : ? PLUS_EXPR : MINUS_EXPR, idx, n2);
3768 249 : insert_before (g);
3769 249 : l = limb_access (TREE_TYPE (lhs), obj, idxmn2, true);
3770 249 : g = gimple_build_assign (l, t1);
3771 249 : insert_before (g);
3772 249 : idx_next = make_ssa_name (sizetype);
3773 249 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
3774 : bitint_big_endian
3775 0 : ? size_int (-1) : size_one_node);
3776 249 : insert_before (g);
3777 249 : m_gsi = gsi_for_stmt (final_stmt);
3778 249 : tree nidx = make_ssa_name (sizetype);
3779 249 : phi = create_phi_node (nidx, gsi_bb (m_gsi));
3780 249 : edge_false = find_edge (edge_false->src, gsi_bb (m_gsi));
3781 249 : edge_true = EDGE_PRED (gsi_bb (m_gsi),
3782 : EDGE_PRED (gsi_bb (m_gsi), 0) == edge_false);
3783 249 : add_phi_arg (phi, idx, edge_false, UNKNOWN_LOCATION);
3784 249 : add_phi_arg (phi, idx_next, edge_true, UNKNOWN_LOCATION);
3785 249 : idx = nidx;
3786 : }
3787 774 : g = gimple_build_assign (make_ssa_name (sizetype),
3788 : bitint_big_endian ? PLUS_EXPR : MINUS_EXPR,
3789 : idx, n2);
3790 387 : insert_before (g);
3791 387 : idx = gimple_assign_lhs (g);
3792 387 : tree sext = ext;
3793 387 : if (!TYPE_UNSIGNED (type))
3794 187 : sext = add_cast (signed_type_for (m_limb_type), ext);
3795 387 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (sext)),
3796 : RSHIFT_EXPR, sext, n1);
3797 387 : insert_before (g);
3798 387 : t1 = gimple_assign_lhs (g);
3799 387 : if (!TYPE_UNSIGNED (type))
3800 : {
3801 187 : t1 = add_cast (m_limb_type, t1);
3802 187 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (sext)),
3803 : RSHIFT_EXPR, sext,
3804 187 : build_int_cst (TREE_TYPE (n),
3805 187 : limb_prec - 1));
3806 187 : insert_before (g);
3807 187 : ext = add_cast (m_limb_type, gimple_assign_lhs (g));
3808 : }
3809 : else
3810 200 : ext = build_zero_cst (m_limb_type);
3811 387 : l = limb_access (TREE_TYPE (lhs), obj, idx, true);
3812 387 : g = gimple_build_assign (l, t1);
3813 387 : insert_before (g);
3814 387 : g = gimple_build_assign (make_ssa_name (sizetype), PLUS_EXPR, idx,
3815 : bitint_big_endian
3816 0 : ? size_int (-1) : size_one_node);
3817 387 : insert_before (g);
3818 387 : tree p2 = p;
3819 387 : if (bitint_big_endian)
3820 : {
3821 0 : tree new_idx = gimple_assign_lhs (g);
3822 0 : g = gimple_build_cond (NE_EXPR, idx, size_zero_node,
3823 : NULL_TREE, NULL_TREE);
3824 0 : idx = new_idx;
3825 : }
3826 : else
3827 : {
3828 387 : if (bitint_extended == bitint_ext_full
3829 0 : && abi_limb_prec > limb_prec)
3830 0 : p2 = build_int_cst (sizetype,
3831 0 : CEIL (prec, abi_limb_prec)
3832 0 : * abi_limb_prec / limb_prec - 1);
3833 387 : idx = gimple_assign_lhs (g);
3834 387 : g = gimple_build_cond (LE_EXPR, idx, p2, NULL_TREE, NULL_TREE);
3835 : }
3836 387 : if_then (g, profile_probability::likely (), edge_true, edge_false);
3837 387 : idx = create_loop (idx, &idx_next);
3838 387 : l = limb_access (TREE_TYPE (lhs), obj, idx, true);
3839 387 : g = gimple_build_assign (l, ext);
3840 387 : insert_before (g);
3841 387 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
3842 : bitint_big_endian
3843 0 : ? size_int (-1) : size_one_node);
3844 387 : insert_before (g);
3845 387 : if (bitint_big_endian)
3846 0 : g = gimple_build_cond (NE_EXPR, idx, size_zero_node,
3847 : NULL_TREE, NULL_TREE);
3848 : else
3849 387 : g = gimple_build_cond (LE_EXPR, idx_next, p2, NULL_TREE, NULL_TREE);
3850 387 : insert_before (g);
3851 : }
3852 : else
3853 : {
3854 : /* Lower
3855 : dst = src << n;
3856 : as
3857 : unsigned n1 = n % limb_prec;
3858 : size_t n2 = n / limb_prec;
3859 : size_t n3 = n1 != 0;
3860 : unsigned n4 = (limb_prec - n1) % limb_prec;
3861 : size_t idx;
3862 : size_t p = prec / limb_prec - (prec % limb_prec == 0);
3863 : for (idx = p; (ssize_t) idx >= (ssize_t) (n2 + n3); --idx)
3864 : dst[idx] = (src[idx - n2] << n1) | (src[idx - n2 - n3] >> n4);
3865 : if (n1)
3866 : {
3867 : dst[idx] = src[idx - n2] << n1;
3868 : --idx;
3869 : }
3870 : for (; (ssize_t) idx >= 0; --idx)
3871 : dst[idx] = 0; */
3872 197 : tree n2pn3;
3873 197 : if (TREE_CODE (n2) == INTEGER_CST && TREE_CODE (n3) == INTEGER_CST)
3874 71 : n2pn3 = int_const_binop (PLUS_EXPR, n2, n3);
3875 : else
3876 : {
3877 126 : n2pn3 = make_ssa_name (sizetype);
3878 126 : g = gimple_build_assign (n2pn3, PLUS_EXPR, n2, n3);
3879 126 : insert_before (g);
3880 : }
3881 197 : if (bitint_big_endian)
3882 : {
3883 0 : if (TREE_CODE (n2pn3) == INTEGER_CST)
3884 0 : n2pn3 = int_const_binop (MINUS_EXPR, p, n2pn3);
3885 : else
3886 : {
3887 0 : g = gimple_build_assign (make_ssa_name (sizetype),
3888 : MINUS_EXPR, p, n2pn3);
3889 0 : insert_before (g);
3890 0 : n2pn3 = gimple_assign_lhs (g);
3891 : }
3892 : }
3893 : /* For LSHIFT_EXPR, we can use handle_operand with non-INTEGER_CST
3894 : idx even to access the most significant partial limb. */
3895 197 : m_var_msb = true;
3896 197 : if (integer_zerop (n3))
3897 : /* For n3 == 0 p >= n2 + n3 is always true for all valid shift
3898 : counts. Emit if (true) condition that can be optimized later. */
3899 45 : g = gimple_build_cond (NE_EXPR, boolean_true_node, boolean_false_node,
3900 : NULL_TREE, NULL_TREE);
3901 152 : else if (bitint_big_endian)
3902 0 : g = gimple_build_cond (NE_EXPR, n2pn3, size_int (-1), NULL_TREE,
3903 : NULL_TREE);
3904 : else
3905 152 : g = gimple_build_cond (LE_EXPR, n2pn3, p, NULL_TREE, NULL_TREE);
3906 197 : edge edge_true, edge_false;
3907 197 : if_then (g, profile_probability::likely (), edge_true, edge_false);
3908 197 : tree idx_next;
3909 197 : tree idx = create_loop (bitint_big_endian ? size_zero_node : p,
3910 : &idx_next);
3911 197 : tree idxmn2 = make_ssa_name (sizetype);
3912 197 : tree idxmn2mn3 = make_ssa_name (sizetype);
3913 394 : g = gimple_build_assign (idxmn2,
3914 : bitint_big_endian ? PLUS_EXPR : MINUS_EXPR,
3915 : idx, n2);
3916 197 : insert_before (g);
3917 394 : g = gimple_build_assign (idxmn2mn3,
3918 : bitint_big_endian ? PLUS_EXPR : MINUS_EXPR,
3919 : idxmn2, n3);
3920 197 : insert_before (g);
3921 197 : m_data_cnt = 0;
3922 197 : tree t1 = handle_operand (rhs1, idxmn2);
3923 197 : m_first = false;
3924 197 : g = gimple_build_assign (make_ssa_name (m_limb_type),
3925 : LSHIFT_EXPR, t1, n1);
3926 197 : insert_before (g);
3927 197 : t1 = gimple_assign_lhs (g);
3928 197 : if (!integer_zerop (n3))
3929 : {
3930 152 : m_data_cnt = 0;
3931 152 : tree t2 = handle_operand (rhs1, idxmn2mn3);
3932 152 : g = gimple_build_assign (make_ssa_name (m_limb_type),
3933 : RSHIFT_EXPR, t2, n4);
3934 152 : insert_before (g);
3935 152 : t2 = gimple_assign_lhs (g);
3936 152 : g = gimple_build_assign (make_ssa_name (m_limb_type),
3937 : BIT_IOR_EXPR, t1, t2);
3938 152 : insert_before (g);
3939 152 : t1 = gimple_assign_lhs (g);
3940 : }
3941 197 : tree l = limb_access (TREE_TYPE (lhs), obj, idx, true);
3942 197 : g = gimple_build_assign (l, t1);
3943 197 : insert_before (g);
3944 394 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
3945 : bitint_big_endian
3946 197 : ? size_one_node : size_int (-1));
3947 197 : insert_before (g);
3948 197 : tree sn2pn3 = add_cast (ssizetype, n2pn3);
3949 394 : g = gimple_build_cond (bitint_big_endian ? LE_EXPR : GE_EXPR,
3950 : add_cast (ssizetype, idx_next), sn2pn3,
3951 : NULL_TREE, NULL_TREE);
3952 197 : insert_before (g);
3953 197 : idx = make_ssa_name (sizetype);
3954 197 : m_gsi = gsi_for_stmt (final_stmt);
3955 197 : gphi *phi = create_phi_node (idx, gsi_bb (m_gsi));
3956 197 : edge_false = find_edge (edge_false->src, gsi_bb (m_gsi));
3957 197 : edge_true = EDGE_PRED (gsi_bb (m_gsi),
3958 : EDGE_PRED (gsi_bb (m_gsi), 0) == edge_false);
3959 197 : add_phi_arg (phi, bitint_big_endian ? size_zero_node : p,
3960 : edge_false, UNKNOWN_LOCATION);
3961 197 : add_phi_arg (phi, idx_next, edge_true, UNKNOWN_LOCATION);
3962 197 : m_data_cnt = 0;
3963 197 : if (!integer_zerop (n3))
3964 : {
3965 152 : g = gimple_build_cond (NE_EXPR, n3, size_zero_node,
3966 : NULL_TREE, NULL_TREE);
3967 152 : if_then (g, profile_probability::likely (), edge_true, edge_false);
3968 152 : idxmn2 = make_ssa_name (sizetype);
3969 304 : g = gimple_build_assign (idxmn2,
3970 : bitint_big_endian ? PLUS_EXPR : MINUS_EXPR,
3971 : idx, n2);
3972 152 : insert_before (g);
3973 152 : m_data_cnt = 0;
3974 152 : t1 = handle_operand (rhs1, idxmn2);
3975 152 : g = gimple_build_assign (make_ssa_name (m_limb_type),
3976 : LSHIFT_EXPR, t1, n1);
3977 152 : insert_before (g);
3978 152 : t1 = gimple_assign_lhs (g);
3979 152 : l = limb_access (TREE_TYPE (lhs), obj, idx, true);
3980 152 : g = gimple_build_assign (l, t1);
3981 152 : insert_before (g);
3982 152 : idx_next = make_ssa_name (sizetype);
3983 304 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
3984 : bitint_big_endian
3985 152 : ? size_one_node : size_int (-1));
3986 152 : insert_before (g);
3987 152 : m_gsi = gsi_for_stmt (final_stmt);
3988 152 : tree nidx = make_ssa_name (sizetype);
3989 152 : phi = create_phi_node (nidx, gsi_bb (m_gsi));
3990 152 : edge_false = find_edge (edge_false->src, gsi_bb (m_gsi));
3991 152 : edge_true = EDGE_PRED (gsi_bb (m_gsi),
3992 : EDGE_PRED (gsi_bb (m_gsi), 0) == edge_false);
3993 152 : add_phi_arg (phi, idx, edge_false, UNKNOWN_LOCATION);
3994 152 : add_phi_arg (phi, idx_next, edge_true, UNKNOWN_LOCATION);
3995 152 : idx = nidx;
3996 : }
3997 197 : if (bitint_big_endian)
3998 0 : g = gimple_build_cond (LE_EXPR, idx, p, NULL_TREE, NULL_TREE);
3999 : else
4000 197 : g = gimple_build_cond (GE_EXPR, add_cast (ssizetype, idx),
4001 : ssize_int (0), NULL_TREE, NULL_TREE);
4002 197 : if_then (g, profile_probability::likely (), edge_true, edge_false);
4003 197 : idx = create_loop (idx, &idx_next);
4004 197 : l = limb_access (TREE_TYPE (lhs), obj, idx, true);
4005 197 : g = gimple_build_assign (l, build_zero_cst (m_limb_type));
4006 197 : insert_before (g);
4007 394 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
4008 : bitint_big_endian
4009 197 : ? size_one_node : size_int (-1));
4010 197 : insert_before (g);
4011 197 : if (bitint_big_endian)
4012 0 : g = gimple_build_cond (LE_EXPR, idx_next, p, NULL_TREE, NULL_TREE);
4013 : else
4014 197 : g = gimple_build_cond (GE_EXPR, add_cast (ssizetype, idx_next),
4015 : ssize_int (0), NULL_TREE, NULL_TREE);
4016 197 : insert_before (g);
4017 197 : if (bitint_extended && prec % limb_prec != 0)
4018 : {
4019 : /* The most significant limb has been updated either in the
4020 : loop or in the if after it. To simplify the code, just
4021 : read it back from memory and extend. */
4022 0 : m_gsi = gsi_after_labels (edge_false->dest);
4023 0 : idx = bitint_big_endian ? size_zero_node : p;
4024 0 : tree l = limb_access (TREE_TYPE (lhs), obj, idx, true);
4025 0 : tree type = limb_access_type (TREE_TYPE (lhs), idx);
4026 0 : tree v = make_ssa_name (m_limb_type);
4027 0 : g = gimple_build_assign (v, l);
4028 0 : insert_before (g);
4029 0 : v = add_cast (type, v);
4030 0 : l = limb_access (TREE_TYPE (lhs), obj, idx, true);
4031 0 : v = add_cast (m_limb_type, v);
4032 0 : g = gimple_build_assign (l, v);
4033 0 : insert_before (g);
4034 0 : if (bitint_extended == bitint_ext_full
4035 0 : && abi_limb_prec > limb_prec
4036 0 : && (CEIL (prec, abi_limb_prec) * abi_limb_prec
4037 0 : > CEIL (prec, limb_prec) * limb_prec))
4038 : {
4039 0 : tree p2 = build_int_cst (sizetype,
4040 : CEIL (prec, abi_limb_prec)
4041 0 : * abi_limb_prec / limb_prec - 1);
4042 0 : if (TYPE_UNSIGNED (TREE_TYPE (lhs)))
4043 0 : v = build_zero_cst (m_limb_type);
4044 : else
4045 : {
4046 0 : v = add_cast (signed_type_for (m_limb_type), v);
4047 0 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (v)),
4048 : RSHIFT_EXPR, v,
4049 : build_int_cst (unsigned_type_node,
4050 0 : limb_prec - 1));
4051 0 : insert_before (g);
4052 0 : v = add_cast (m_limb_type, gimple_assign_lhs (g));
4053 : }
4054 0 : l = limb_access (TREE_TYPE (lhs), obj, p2, true);
4055 0 : g = gimple_build_assign (l, v);
4056 0 : insert_before (g);
4057 : }
4058 : }
4059 197 : else if (bitint_extended == bitint_ext_full
4060 0 : && abi_limb_prec > limb_prec
4061 0 : && (CEIL (prec, abi_limb_prec) * abi_limb_prec
4062 0 : > CEIL (prec, limb_prec) * limb_prec))
4063 : {
4064 0 : m_gsi = gsi_after_labels (edge_false->dest);
4065 0 : tree p2 = build_int_cst (sizetype,
4066 : CEIL (prec, abi_limb_prec)
4067 0 : * abi_limb_prec / limb_prec - 1);
4068 0 : tree v;
4069 0 : if (TYPE_UNSIGNED (TREE_TYPE (lhs)))
4070 0 : v = build_zero_cst (m_limb_type);
4071 : else
4072 : {
4073 0 : tree l = limb_access (TREE_TYPE (lhs), obj, p, true);
4074 0 : v = make_ssa_name (m_limb_type);
4075 0 : g = gimple_build_assign (v, l);
4076 0 : insert_before (g);
4077 0 : v = add_cast (signed_type_for (m_limb_type), v);
4078 0 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (v)),
4079 : RSHIFT_EXPR, v,
4080 : build_int_cst (unsigned_type_node,
4081 0 : limb_prec - 1));
4082 0 : insert_before (g);
4083 0 : v = add_cast (m_limb_type, gimple_assign_lhs (g));
4084 : }
4085 0 : tree l = limb_access (TREE_TYPE (lhs), obj, p2, true);
4086 0 : g = gimple_build_assign (l, v);
4087 0 : insert_before (g);
4088 : }
4089 : }
4090 584 : }
4091 :
4092 : /* Lower large/huge _BitInt multiplication or division. */
4093 :
4094 : void
4095 371 : bitint_large_huge::lower_muldiv_stmt (tree obj, gimple *stmt)
4096 : {
4097 371 : tree rhs1 = gimple_assign_rhs1 (stmt);
4098 371 : tree rhs2 = gimple_assign_rhs2 (stmt);
4099 371 : tree lhs = gimple_assign_lhs (stmt);
4100 371 : tree_code rhs_code = gimple_assign_rhs_code (stmt);
4101 371 : tree type = TREE_TYPE (rhs1);
4102 371 : gcc_assert (BITINT_TYPE_P (type)
4103 : && bitint_precision_kind (type) >= bitint_prec_large);
4104 371 : int prec = TYPE_PRECISION (type), prec1, prec2;
4105 371 : bool ext_ms_limb = false;
4106 371 : bool do_ext = false;
4107 371 : rhs1 = handle_operand_addr (rhs1, stmt, NULL, &prec1);
4108 371 : rhs2 = handle_operand_addr (rhs2, stmt, NULL, &prec2);
4109 371 : if (obj == NULL_TREE)
4110 : {
4111 166 : int part = var_to_partition (m_map, lhs);
4112 166 : gcc_assert (m_vars[part] != NULL_TREE);
4113 166 : obj = m_vars[part];
4114 166 : lhs = build_fold_addr_expr (obj);
4115 : }
4116 : else
4117 : {
4118 205 : lhs = build_fold_addr_expr (obj);
4119 205 : lhs = force_gimple_operand_gsi (&m_gsi, lhs, true,
4120 : NULL_TREE, true, GSI_SAME_STMT);
4121 : }
4122 371 : if (bitint_extended && TYPE_OVERFLOW_WRAPS (type))
4123 : {
4124 0 : if (rhs_code == MULT_EXPR)
4125 : do_ext = true;
4126 : /* For signed division with -fwrapv, minimum negative / -1 needs
4127 : is minimum negative and the padding bits above it should be all
4128 : set. */
4129 0 : else if (!TYPE_UNSIGNED (type)
4130 0 : && (rhs_code == TRUNC_DIV_EXPR || rhs_code == EXACT_DIV_EXPR))
4131 371 : do_ext = true;
4132 : }
4133 371 : if (bitint_extended == bitint_ext_full
4134 0 : && abi_limb_prec > limb_prec
4135 0 : && (CEIL (prec, abi_limb_prec) * abi_limb_prec
4136 0 : > CEIL (prec, limb_prec) * limb_prec))
4137 : {
4138 : /* unsigned multiplication needs to wrap around, so we can't
4139 : increase prec. Similarly for -fwrapv. */
4140 0 : if (do_ext)
4141 : ext_ms_limb = true;
4142 : else
4143 0 : prec = CEIL (prec, abi_limb_prec) * abi_limb_prec;
4144 : }
4145 371 : tree sitype = lang_hooks.types.type_for_mode (SImode, 0);
4146 371 : gimple *g;
4147 371 : switch (rhs_code)
4148 : {
4149 208 : case MULT_EXPR:
4150 208 : g = gimple_build_call_internal (IFN_MULBITINT, 6,
4151 208 : lhs, build_int_cst (sitype, prec),
4152 208 : rhs1, build_int_cst (sitype, prec1),
4153 208 : rhs2, build_int_cst (sitype, prec2));
4154 208 : insert_before (g);
4155 208 : break;
4156 103 : case TRUNC_DIV_EXPR:
4157 103 : case EXACT_DIV_EXPR:
4158 103 : g = gimple_build_call_internal (IFN_DIVMODBITINT, 8,
4159 103 : lhs, build_int_cst (sitype, prec),
4160 : null_pointer_node,
4161 : build_int_cst (sitype, 0),
4162 103 : rhs1, build_int_cst (sitype, prec1),
4163 103 : rhs2, build_int_cst (sitype, prec2));
4164 103 : if (!stmt_ends_bb_p (stmt))
4165 102 : gimple_call_set_nothrow (as_a <gcall *> (g), true);
4166 103 : insert_before (g);
4167 103 : break;
4168 60 : case TRUNC_MOD_EXPR:
4169 60 : g = gimple_build_call_internal (IFN_DIVMODBITINT, 8, null_pointer_node,
4170 : build_int_cst (sitype, 0),
4171 60 : lhs, build_int_cst (sitype, prec),
4172 60 : rhs1, build_int_cst (sitype, prec1),
4173 60 : rhs2, build_int_cst (sitype, prec2));
4174 60 : if (!stmt_ends_bb_p (stmt))
4175 57 : gimple_call_set_nothrow (as_a <gcall *> (g), true);
4176 60 : insert_before (g);
4177 60 : break;
4178 0 : default:
4179 0 : gcc_unreachable ();
4180 : }
4181 371 : if (stmt_ends_bb_p (stmt))
4182 : {
4183 4 : maybe_duplicate_eh_stmt (g, stmt);
4184 4 : edge e1;
4185 4 : edge_iterator ei;
4186 4 : basic_block bb = gimple_bb (stmt);
4187 :
4188 4 : FOR_EACH_EDGE (e1, ei, bb->succs)
4189 4 : if (e1->flags & EDGE_EH)
4190 : break;
4191 4 : if (e1)
4192 : {
4193 4 : edge e2 = split_block (gsi_bb (m_gsi), g);
4194 4 : m_gsi = gsi_after_labels (e2->dest);
4195 4 : add_eh_edge (e2->src, e1);
4196 : }
4197 : }
4198 371 : if (do_ext
4199 371 : && ((prec % limb_prec) != 0 || (ext_ms_limb && !TYPE_UNSIGNED (type))))
4200 : {
4201 : /* Unsigned multiplication wraps, but libgcc function will return the
4202 : bits beyond prec within the top limb as another limb of the full
4203 : multiplication. So, clear the padding bits here. */
4204 0 : tree idx = size_int (bitint_big_endian ? 0 : prec / limb_prec);
4205 0 : tree l = limb_access (type, obj, idx, true);
4206 0 : tree ctype = limb_access_type (type, idx);
4207 0 : tree v = make_ssa_name (m_limb_type);
4208 0 : g = gimple_build_assign (v, l);
4209 0 : insert_before (g);
4210 0 : tree v2 = v;
4211 0 : if ((prec % limb_prec) != 0)
4212 : {
4213 0 : v = add_cast (ctype, v);
4214 0 : l = limb_access (type, obj, idx, true);
4215 0 : v = add_cast (m_limb_type, v);
4216 0 : v2 = v;
4217 0 : g = gimple_build_assign (l, v);
4218 0 : insert_before (g);
4219 : }
4220 0 : if (ext_ms_limb && !TYPE_UNSIGNED (type))
4221 : {
4222 0 : v2 = add_cast (signed_type_for (m_limb_type), v2);
4223 0 : tree lpm1 = build_int_cst (unsigned_type_node, limb_prec - 1);
4224 0 : v = make_ssa_name (TREE_TYPE (v2));
4225 0 : g = gimple_build_assign (v, RSHIFT_EXPR, v2, lpm1);
4226 0 : insert_before (g);
4227 0 : unsigned int i
4228 0 : = CEIL (prec, abi_limb_prec) * abi_limb_prec / limb_prec;
4229 0 : v = add_cast (m_limb_type, v);
4230 0 : g = gimple_build_assign (limb_access (type, obj, size_int (i - 1),
4231 : true), v);
4232 0 : insert_before (g);
4233 0 : ext_ms_limb = false;
4234 : }
4235 : }
4236 371 : if (ext_ms_limb)
4237 : {
4238 0 : unsigned int i = CEIL (prec, abi_limb_prec) * abi_limb_prec / limb_prec;
4239 0 : g = gimple_build_assign (limb_access (type, obj, size_int (i - 1), true),
4240 : build_zero_cst (m_limb_type));
4241 0 : insert_before (g);
4242 : }
4243 371 : }
4244 :
4245 : /* Lower large/huge _BitInt conversion to/from floating point. */
4246 :
4247 : void
4248 327 : bitint_large_huge::lower_float_conv_stmt (tree obj, gimple *stmt)
4249 : {
4250 327 : tree rhs1 = gimple_assign_rhs1 (stmt);
4251 327 : tree lhs = gimple_assign_lhs (stmt);
4252 327 : tree_code rhs_code = gimple_assign_rhs_code (stmt);
4253 327 : tree sitype = lang_hooks.types.type_for_mode (SImode, 0);
4254 327 : gimple *g;
4255 327 : if (rhs_code == FIX_TRUNC_EXPR)
4256 : {
4257 179 : tree type = TREE_TYPE (lhs);
4258 179 : int prec = TYPE_PRECISION (type);
4259 179 : bool extend_ms_limb = false;
4260 179 : if (bitint_extended == bitint_ext_full
4261 0 : && abi_limb_prec > limb_prec
4262 0 : && (CEIL (prec, abi_limb_prec) * abi_limb_prec
4263 0 : > CEIL (prec, limb_prec) * limb_prec))
4264 179 : extend_ms_limb = true;
4265 179 : if (!TYPE_UNSIGNED (type))
4266 93 : prec = -prec;
4267 179 : if (obj == NULL_TREE)
4268 : {
4269 135 : int part = var_to_partition (m_map, lhs);
4270 135 : gcc_assert (m_vars[part] != NULL_TREE);
4271 135 : obj = m_vars[part];
4272 135 : lhs = build_fold_addr_expr (obj);
4273 : }
4274 : else
4275 : {
4276 44 : lhs = build_fold_addr_expr (obj);
4277 44 : lhs = force_gimple_operand_gsi (&m_gsi, lhs, true,
4278 : NULL_TREE, true, GSI_SAME_STMT);
4279 : }
4280 179 : scalar_mode from_mode
4281 179 : = as_a <scalar_mode> (TYPE_MODE (TREE_TYPE (rhs1)));
4282 : #ifdef HAVE_SFmode
4283 : /* IEEE single is a full superset of both IEEE half and
4284 : bfloat formats, convert to float first and then to _BitInt
4285 : to avoid the need of another 2 library routines. */
4286 179 : if ((REAL_MODE_FORMAT (from_mode) == &arm_bfloat_half_format
4287 179 : || REAL_MODE_FORMAT (from_mode) == &ieee_half_format)
4288 191 : && REAL_MODE_FORMAT (SFmode) == &ieee_single_format)
4289 : {
4290 12 : tree type = lang_hooks.types.type_for_mode (SFmode, 0);
4291 12 : if (type)
4292 12 : rhs1 = add_cast (type, rhs1);
4293 : }
4294 : #endif
4295 179 : g = gimple_build_call_internal (IFN_FLOATTOBITINT, 3,
4296 179 : lhs, build_int_cst (sitype, prec),
4297 : rhs1);
4298 179 : insert_before (g);
4299 179 : if (extend_ms_limb)
4300 : {
4301 0 : unsigned int i
4302 0 : = (CEIL (prec < 0 ? -prec : prec, abi_limb_prec)
4303 0 : * abi_limb_prec / limb_prec);
4304 0 : tree val;
4305 0 : if (prec < 0)
4306 : {
4307 0 : g = gimple_build_assign (make_ssa_name (m_limb_type),
4308 : limb_access (type, obj,
4309 0 : size_int (i - 2),
4310 : true));
4311 0 : insert_before (g);
4312 0 : val = add_cast (signed_type_for (m_limb_type),
4313 : gimple_assign_lhs (g));
4314 0 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (val)),
4315 : RSHIFT_EXPR, val,
4316 : build_int_cst (unsigned_type_node,
4317 0 : limb_prec - 1));
4318 0 : insert_before (g);
4319 0 : val = add_cast (m_limb_type, gimple_assign_lhs (g));
4320 : }
4321 : else
4322 0 : val = build_zero_cst (m_limb_type);
4323 0 : g = gimple_build_assign (limb_access (type, obj, size_int (i - 1),
4324 : true), val);
4325 0 : insert_before (g);
4326 : }
4327 : }
4328 : else
4329 : {
4330 148 : int prec;
4331 148 : rhs1 = handle_operand_addr (rhs1, stmt, NULL, &prec);
4332 148 : g = gimple_build_call_internal (IFN_BITINTTOFLOAT, 2,
4333 148 : rhs1, build_int_cst (sitype, prec));
4334 148 : gimple_call_set_lhs (g, lhs);
4335 148 : if (!stmt_ends_bb_p (stmt))
4336 147 : gimple_call_set_nothrow (as_a <gcall *> (g), true);
4337 148 : gsi_replace (&m_gsi, g, true);
4338 : }
4339 327 : }
4340 :
4341 : /* Helper method for lower_addsub_overflow and lower_mul_overflow.
4342 : If check_zero is true, caller wants to check if all bits in [start, end)
4343 : are zero, otherwise if bits in [start, end) are either all zero or
4344 : all ones. L is the limb with index LIMB, START and END are measured
4345 : in bits. */
4346 :
4347 : tree
4348 6453 : bitint_large_huge::arith_overflow_extract_bits (unsigned int start,
4349 : unsigned int end, tree l,
4350 : unsigned int limb,
4351 : bool check_zero)
4352 : {
4353 6453 : unsigned startlimb = start / limb_prec;
4354 6453 : unsigned endlimb = (end - 1) / limb_prec;
4355 6453 : gimple *g;
4356 :
4357 6453 : if ((start % limb_prec) == 0 && (end % limb_prec) == 0)
4358 : return l;
4359 6184 : if (startlimb == endlimb && limb == startlimb)
4360 : {
4361 2039 : if (check_zero)
4362 : {
4363 1486 : wide_int w = wi::shifted_mask (start % limb_prec,
4364 1486 : end - start, false, limb_prec);
4365 2972 : g = gimple_build_assign (make_ssa_name (m_limb_type),
4366 : BIT_AND_EXPR, l,
4367 1486 : wide_int_to_tree (m_limb_type, w));
4368 1486 : insert_before (g);
4369 1486 : return gimple_assign_lhs (g);
4370 1486 : }
4371 553 : unsigned int shift = start % limb_prec;
4372 553 : if ((end % limb_prec) != 0)
4373 : {
4374 336 : unsigned int lshift = (-end) % limb_prec;
4375 336 : shift += lshift;
4376 336 : g = gimple_build_assign (make_ssa_name (m_limb_type),
4377 : LSHIFT_EXPR, l,
4378 : build_int_cst (unsigned_type_node,
4379 336 : lshift));
4380 336 : insert_before (g);
4381 336 : l = gimple_assign_lhs (g);
4382 : }
4383 553 : l = add_cast (signed_type_for (m_limb_type), l);
4384 553 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (l)),
4385 : RSHIFT_EXPR, l,
4386 553 : build_int_cst (unsigned_type_node, shift));
4387 553 : insert_before (g);
4388 553 : return add_cast (m_limb_type, gimple_assign_lhs (g));
4389 : }
4390 4145 : else if (limb == startlimb)
4391 : {
4392 1990 : if ((start % limb_prec) == 0)
4393 : return l;
4394 1876 : if (!check_zero)
4395 950 : l = add_cast (signed_type_for (m_limb_type), l);
4396 1876 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (l)),
4397 : RSHIFT_EXPR, l,
4398 : build_int_cst (unsigned_type_node,
4399 1876 : start % limb_prec));
4400 1876 : insert_before (g);
4401 1876 : l = gimple_assign_lhs (g);
4402 1876 : if (!check_zero)
4403 950 : l = add_cast (m_limb_type, l);
4404 : return l;
4405 : }
4406 2155 : else if (limb == endlimb)
4407 : {
4408 1704 : if ((end % limb_prec) == 0)
4409 : return l;
4410 1687 : if (check_zero)
4411 : {
4412 880 : wide_int w = wi::mask (end % limb_prec, false, limb_prec);
4413 1760 : g = gimple_build_assign (make_ssa_name (m_limb_type),
4414 : BIT_AND_EXPR, l,
4415 880 : wide_int_to_tree (m_limb_type, w));
4416 880 : insert_before (g);
4417 880 : return gimple_assign_lhs (g);
4418 880 : }
4419 807 : unsigned int shift = (-end) % limb_prec;
4420 807 : g = gimple_build_assign (make_ssa_name (m_limb_type),
4421 : LSHIFT_EXPR, l,
4422 807 : build_int_cst (unsigned_type_node, shift));
4423 807 : insert_before (g);
4424 807 : l = add_cast (signed_type_for (m_limb_type), gimple_assign_lhs (g));
4425 807 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (l)),
4426 : RSHIFT_EXPR, l,
4427 807 : build_int_cst (unsigned_type_node, shift));
4428 807 : insert_before (g);
4429 807 : return add_cast (m_limb_type, gimple_assign_lhs (g));
4430 : }
4431 : return l;
4432 : }
4433 :
4434 : /* Helper method for lower_addsub_overflow and lower_mul_overflow. Store
4435 : result including overflow flag into the right locations. */
4436 :
4437 : void
4438 4212 : bitint_large_huge::finish_arith_overflow (tree var, tree obj, tree type,
4439 : tree ovf, tree lhs, tree orig_obj,
4440 : gimple *stmt, unsigned nelts,
4441 : tree_code code)
4442 : {
4443 4212 : gimple *g;
4444 :
4445 4212 : if (obj == NULL_TREE
4446 4212 : && (!BITINT_TYPE_P (type)
4447 231 : || bitint_precision_kind (type) < bitint_prec_large))
4448 : {
4449 : /* Add support for 3 or more limbs filled in from normal integral
4450 : type if this assert fails. If no target chooses limb mode smaller
4451 : than half of largest supported normal integral type, this will not
4452 : be needed. */
4453 250 : gcc_assert (TYPE_PRECISION (type) <= 2 * limb_prec);
4454 250 : tree lhs_type = type;
4455 110 : if (BITINT_TYPE_P (type)
4456 250 : && bitint_precision_kind (type) == bitint_prec_middle)
4457 46 : lhs_type = build_nonstandard_integer_type (TYPE_PRECISION (type),
4458 46 : TYPE_UNSIGNED (type));
4459 250 : tree r1 = limb_access (NULL_TREE, var,
4460 : bitint_big_endian
4461 0 : ? size_int (nelts - 1) : size_zero_node, true);
4462 250 : g = gimple_build_assign (make_ssa_name (m_limb_type), r1);
4463 250 : insert_before (g);
4464 250 : r1 = gimple_assign_lhs (g);
4465 250 : if (!useless_type_conversion_p (lhs_type, TREE_TYPE (r1)))
4466 250 : r1 = add_cast (lhs_type, r1);
4467 250 : if (TYPE_PRECISION (lhs_type) > limb_prec)
4468 : {
4469 90 : tree r2 = limb_access (NULL_TREE, var,
4470 : bitint_big_endian
4471 0 : ? size_int (nelts - 2) : size_one_node, true);
4472 90 : g = gimple_build_assign (make_ssa_name (m_limb_type), r2);
4473 90 : insert_before (g);
4474 90 : r2 = gimple_assign_lhs (g);
4475 90 : r2 = add_cast (lhs_type, r2);
4476 90 : g = gimple_build_assign (make_ssa_name (lhs_type), LSHIFT_EXPR, r2,
4477 : build_int_cst (unsigned_type_node,
4478 90 : limb_prec));
4479 90 : insert_before (g);
4480 90 : g = gimple_build_assign (make_ssa_name (lhs_type), BIT_IOR_EXPR, r1,
4481 : gimple_assign_lhs (g));
4482 90 : insert_before (g);
4483 90 : r1 = gimple_assign_lhs (g);
4484 : }
4485 250 : if (lhs_type != type)
4486 46 : r1 = add_cast (type, r1);
4487 250 : ovf = add_cast (lhs_type, ovf);
4488 250 : if (lhs_type != type)
4489 46 : ovf = add_cast (type, ovf);
4490 250 : g = gimple_build_assign (lhs, COMPLEX_EXPR, r1, ovf);
4491 250 : m_gsi = gsi_for_stmt (stmt);
4492 250 : gsi_replace (&m_gsi, g, true);
4493 : }
4494 : else
4495 : {
4496 3962 : unsigned HOST_WIDE_INT obj_nelts = 0;
4497 3962 : tree atype = NULL_TREE;
4498 3962 : if (obj)
4499 : {
4500 3871 : obj_nelts = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (obj))) / limb_prec;
4501 3871 : if (orig_obj == NULL_TREE)
4502 2277 : obj_nelts >>= 1;
4503 3871 : atype = build_array_type_nelts (m_limb_type, obj_nelts);
4504 : }
4505 3962 : if (bitint_extended && (var || obj))
4506 : {
4507 0 : unsigned prec = TYPE_PRECISION (type);
4508 0 : unsigned prec_limbs = CEIL (prec, limb_prec);
4509 0 : bool ext_ms_limb
4510 : = (bitint_extended == bitint_ext_full
4511 0 : && abi_limb_prec > limb_prec
4512 0 : && (CEIL (prec, abi_limb_prec) * abi_limb_prec
4513 0 : > CEIL (prec, limb_prec) * limb_prec));
4514 : /* For .{ADD,SUB}_OVERFLOW the partial limb if any is
4515 : already extended in lower_addsub_overflow. */
4516 0 : if ((code == MULT_EXPR && (prec % limb_prec) != 0)
4517 0 : || (ext_ms_limb && !TYPE_UNSIGNED (type)))
4518 : {
4519 0 : tree plm1idx = size_int (bitint_big_endian
4520 : ? nelts - obj_nelts : prec_limbs - 1);
4521 0 : tree plm1type
4522 0 : = limb_access_type (type, bitint_big_endian
4523 : ? size_zero_node : plm1idx);
4524 0 : tree l = limb_access (bitint_big_endian ? NULL_TREE : type,
4525 : var ? var : obj, plm1idx, true);
4526 0 : tree rhs = make_ssa_name (TREE_TYPE (l));
4527 0 : g = gimple_build_assign (rhs, l);
4528 0 : insert_before (g);
4529 0 : if (code == MULT_EXPR && (prec % limb_prec) != 0)
4530 : {
4531 0 : if (!useless_type_conversion_p (plm1type, TREE_TYPE (rhs)))
4532 0 : rhs = add_cast (plm1type, rhs);
4533 0 : if (!useless_type_conversion_p (TREE_TYPE (l),
4534 0 : TREE_TYPE (rhs)))
4535 0 : rhs = add_cast (TREE_TYPE (l), rhs);
4536 0 : l = limb_access (bitint_big_endian ? NULL_TREE : type,
4537 : var ? var : obj, plm1idx, true);
4538 0 : g = gimple_build_assign (l, rhs);
4539 0 : insert_before (g);
4540 : }
4541 0 : if (ext_ms_limb && !TYPE_UNSIGNED (type))
4542 : {
4543 0 : rhs = add_cast (signed_type_for (m_limb_type), rhs);
4544 0 : tree lpm1 = build_int_cst (unsigned_type_node,
4545 0 : limb_prec - 1);
4546 0 : tree v = make_ssa_name (TREE_TYPE (rhs));
4547 0 : g = gimple_build_assign (v, RSHIFT_EXPR, rhs, lpm1);
4548 0 : insert_before (g);
4549 0 : unsigned int i
4550 0 : = CEIL (prec, abi_limb_prec) * abi_limb_prec / limb_prec;
4551 0 : v = add_cast (m_limb_type, v);
4552 0 : g = gimple_build_assign (limb_access (type, var ? var : obj,
4553 0 : size_int (i - 1),
4554 : true), v);
4555 0 : insert_before (g);
4556 0 : ext_ms_limb = false;
4557 : }
4558 : }
4559 0 : if (ext_ms_limb)
4560 : {
4561 0 : unsigned int i
4562 0 : = CEIL (prec, abi_limb_prec) * abi_limb_prec / limb_prec;
4563 0 : g = gimple_build_assign (limb_access (type, var ? var : obj,
4564 0 : size_int (i - 1), true),
4565 : build_zero_cst (m_limb_type));
4566 0 : insert_before (g);
4567 : }
4568 : }
4569 3962 : if (var && obj)
4570 : {
4571 519 : tree v1, v2;
4572 519 : tree off;
4573 519 : if (orig_obj == NULL_TREE)
4574 : {
4575 32 : off = build_zero_cst (build_pointer_type (TREE_TYPE (obj)));
4576 32 : v1 = build2 (MEM_REF, atype,
4577 : build_fold_addr_expr (unshare_expr (obj)), off);
4578 : }
4579 487 : else if (!useless_type_conversion_p (atype, TREE_TYPE (obj)))
4580 9 : v1 = build1 (VIEW_CONVERT_EXPR, atype, unshare_expr (obj));
4581 : else
4582 478 : v1 = unshare_expr (obj);
4583 519 : off = build_int_cst (build_pointer_type (TREE_TYPE (var)),
4584 : bitint_big_endian
4585 519 : ? (nelts - obj_nelts) * m_limb_size : 0);
4586 519 : v2 = build2 (MEM_REF, atype, build_fold_addr_expr (var), off);
4587 519 : g = gimple_build_assign (v1, v2);
4588 519 : insert_before (g);
4589 : }
4590 3443 : else if (obj && bitint_big_endian && nelts != obj_nelts)
4591 : {
4592 0 : gcc_assert (nelts > obj_nelts);
4593 0 : tree fn = builtin_decl_implicit (BUILT_IN_MEMMOVE);
4594 0 : tree off = build_int_cst (build_pointer_type (TREE_TYPE (obj)),
4595 0 : (nelts - obj_nelts) * m_limb_size);
4596 0 : tree src = build2 (MEM_REF, atype,
4597 : build_fold_addr_expr (unshare_expr (obj)), off);
4598 0 : g = gimple_build_call (fn, 3,
4599 : build_fold_addr_expr (unshare_expr (obj)),
4600 : build_fold_addr_expr (src),
4601 : build_int_cst (size_type_node,
4602 0 : obj_nelts * m_limb_size));
4603 0 : insert_before (g);
4604 : }
4605 3962 : if (orig_obj == NULL_TREE && obj)
4606 : {
4607 2277 : ovf = add_cast (m_limb_type, ovf);
4608 2277 : tree l = limb_access (NULL_TREE, obj,
4609 2277 : size_int (bitint_big_endian
4610 : ? obj_nelts * 2 - 1 : obj_nelts),
4611 : true);
4612 2277 : g = gimple_build_assign (l, ovf);
4613 2277 : insert_before (g);
4614 2277 : if (obj_nelts > 1)
4615 : {
4616 2277 : atype = build_array_type_nelts (m_limb_type, obj_nelts - 1);
4617 2277 : tree off = build_int_cst (build_pointer_type (TREE_TYPE (obj)),
4618 2277 : (obj_nelts + !bitint_big_endian)
4619 2277 : * m_limb_size);
4620 2277 : tree v1 = build2 (MEM_REF, atype,
4621 : build_fold_addr_expr (unshare_expr (obj)),
4622 : off);
4623 2277 : g = gimple_build_assign (v1, build_zero_cst (atype));
4624 2277 : insert_before (g);
4625 : }
4626 : }
4627 1685 : else if (TREE_CODE (TREE_TYPE (lhs)) == COMPLEX_TYPE)
4628 : {
4629 1640 : imm_use_iterator ui;
4630 1640 : use_operand_p use_p;
4631 1640 : FOR_EACH_IMM_USE_FAST (use_p, ui, lhs)
4632 : {
4633 1640 : g = USE_STMT (use_p);
4634 1640 : if (!is_gimple_assign (g)
4635 1640 : || gimple_assign_rhs_code (g) != IMAGPART_EXPR)
4636 0 : continue;
4637 1640 : tree lhs2 = gimple_assign_lhs (g);
4638 1640 : gimple *use_stmt;
4639 1640 : single_imm_use (lhs2, &use_p, &use_stmt);
4640 1640 : lhs2 = gimple_assign_lhs (use_stmt);
4641 1640 : gimple_stmt_iterator gsi = gsi_for_stmt (use_stmt);
4642 1640 : if (useless_type_conversion_p (TREE_TYPE (lhs2), TREE_TYPE (ovf)))
4643 1619 : g = gimple_build_assign (lhs2, ovf);
4644 : else
4645 21 : g = gimple_build_assign (lhs2, NOP_EXPR, ovf);
4646 1640 : gsi_replace (&gsi, g, true);
4647 1640 : if (gsi_stmt (m_gsi) == use_stmt)
4648 91 : m_gsi = gsi_for_stmt (g);
4649 1640 : break;
4650 1640 : }
4651 : }
4652 45 : else if (ovf != boolean_false_node)
4653 : {
4654 45 : g = gimple_build_cond (NE_EXPR, ovf, boolean_false_node,
4655 : NULL_TREE, NULL_TREE);
4656 45 : edge edge_true, edge_false;
4657 45 : if_then (g, profile_probability::very_unlikely (),
4658 : edge_true, edge_false);
4659 45 : tree zero = build_zero_cst (TREE_TYPE (lhs));
4660 45 : tree fn = ubsan_build_overflow_builtin (code, m_loc,
4661 45 : TREE_TYPE (lhs),
4662 : zero, zero, NULL);
4663 45 : force_gimple_operand_gsi (&m_gsi, fn, true, NULL_TREE,
4664 : true, GSI_SAME_STMT);
4665 45 : m_gsi = gsi_after_labels (edge_true->dest);
4666 : }
4667 : }
4668 4212 : if (var)
4669 : {
4670 781 : tree clobber = build_clobber (TREE_TYPE (var), CLOBBER_STORAGE_END);
4671 781 : g = gimple_build_assign (var, clobber);
4672 781 : gsi_insert_after (&m_gsi, g, GSI_SAME_STMT);
4673 : }
4674 4212 : }
4675 :
4676 : /* Helper function for lower_addsub_overflow and lower_mul_overflow.
4677 : Given precisions of result TYPE (PREC), argument 0 precision PREC0,
4678 : argument 1 precision PREC1 and minimum precision for the result
4679 : PREC2, compute *START, *END, *CHECK_ZERO and return OVF. */
4680 :
4681 : static tree
4682 4212 : arith_overflow (tree_code code, tree type, int prec, int prec0, int prec1,
4683 : int prec2, unsigned *start, unsigned *end, bool *check_zero)
4684 : {
4685 4212 : *start = 0;
4686 4212 : *end = 0;
4687 4212 : *check_zero = true;
4688 : /* Ignore this special rule for subtraction, even if both
4689 : prec0 >= 0 and prec1 >= 0, their subtraction can be negative
4690 : in infinite precision. */
4691 4212 : if (code != MINUS_EXPR && prec0 >= 0 && prec1 >= 0)
4692 : {
4693 : /* Result in [0, prec2) is unsigned, if prec > prec2,
4694 : all bits above it will be zero. */
4695 681 : if ((prec - !TYPE_UNSIGNED (type)) >= prec2)
4696 0 : return boolean_false_node;
4697 : else
4698 : {
4699 : /* ovf if any of bits in [start, end) is non-zero. */
4700 681 : *start = prec - !TYPE_UNSIGNED (type);
4701 681 : *end = prec2;
4702 : }
4703 : }
4704 3531 : else if (TYPE_UNSIGNED (type))
4705 : {
4706 : /* If result in [0, prec2) is signed and if prec > prec2,
4707 : all bits above it will be sign bit copies. */
4708 1950 : if (prec >= prec2)
4709 : {
4710 : /* ovf if bit prec - 1 is non-zero. */
4711 184 : *start = prec - 1;
4712 184 : *end = prec;
4713 : }
4714 : else
4715 : {
4716 : /* ovf if any of bits in [start, end) is non-zero. */
4717 1766 : *start = prec;
4718 1766 : *end = prec2;
4719 : }
4720 : }
4721 1581 : else if (prec >= prec2)
4722 2 : return boolean_false_node;
4723 : else
4724 : {
4725 : /* ovf if [start, end) bits aren't all zeros or all ones. */
4726 1579 : *start = prec - 1;
4727 1579 : *end = prec2;
4728 1579 : *check_zero = false;
4729 : }
4730 : return NULL_TREE;
4731 : }
4732 :
4733 : /* Lower a .{ADD,SUB}_OVERFLOW call with at least one large/huge _BitInt
4734 : argument or return type _Complex large/huge _BitInt. */
4735 :
4736 : void
4737 2733 : bitint_large_huge::lower_addsub_overflow (tree obj, gimple *stmt)
4738 : {
4739 2733 : tree arg0 = gimple_call_arg (stmt, 0);
4740 2733 : tree arg1 = gimple_call_arg (stmt, 1);
4741 2733 : tree lhs = gimple_call_lhs (stmt);
4742 2733 : gimple *g;
4743 :
4744 2733 : if (!lhs)
4745 : {
4746 0 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
4747 0 : gsi_remove (&gsi, true);
4748 0 : return;
4749 : }
4750 2733 : gimple *final_stmt = gsi_stmt (m_gsi);
4751 2733 : tree type = TREE_TYPE (lhs);
4752 2733 : if (TREE_CODE (type) == COMPLEX_TYPE)
4753 2703 : type = TREE_TYPE (type);
4754 2733 : int prec = TYPE_PRECISION (type);
4755 2733 : int prec0 = range_to_prec (arg0, stmt);
4756 2733 : int prec1 = range_to_prec (arg1, stmt);
4757 : /* If PREC0 >= 0 && PREC1 >= 0 and CODE is not MINUS_EXPR, PREC2 is
4758 : the minimum unsigned precision of any possible operation's
4759 : result, otherwise it is minimum signed precision.
4760 : Some examples:
4761 : If PREC0 or PREC1 is 8, it means that argument is [0, 0xff],
4762 : if PREC0 or PREC1 is 10, it means that argument is [0, 0x3ff],
4763 : if PREC0 or PREC1 is -8, it means that argument is [-0x80, 0x7f],
4764 : if PREC0 or PREC1 is -10, it means that argument is [-0x200, 0x1ff].
4765 : PREC0 CODE PREC1 RESULT PREC2 SIGNED vs. UNSIGNED
4766 : 8 + 8 [0, 0x1fe] 9 UNSIGNED
4767 : 8 + 10 [0, 0x4fe] 11 UNSIGNED
4768 : -8 + -8 [-0x100, 0xfe] 9 SIGNED
4769 : -8 + -10 [-0x280, 0x27e] 11 SIGNED
4770 : 8 + -8 [-0x80, 0x17e] 10 SIGNED
4771 : 8 + -10 [-0x200, 0x2fe] 11 SIGNED
4772 : 10 + -8 [-0x80, 0x47e] 12 SIGNED
4773 : 8 - 8 [-0xff, 0xff] 9 SIGNED
4774 : 8 - 10 [-0x3ff, 0xff] 11 SIGNED
4775 : 10 - 8 [-0xff, 0x3ff] 11 SIGNED
4776 : -8 - -8 [-0xff, 0xff] 9 SIGNED
4777 : -8 - -10 [-0x27f, 0x27f] 11 SIGNED
4778 : -10 - -8 [-0x27f, 0x27f] 11 SIGNED
4779 : 8 - -8 [-0x7f, 0x17f] 10 SIGNED
4780 : 8 - -10 [-0x1ff, 0x2ff] 11 SIGNED
4781 : 10 - -8 [-0x7f, 0x47f] 12 SIGNED
4782 : -8 - 8 [-0x17f, 0x7f] 10 SIGNED
4783 : -8 - 10 [-0x47f, 0x7f] 12 SIGNED
4784 : -10 - 8 [-0x2ff, 0x1ff] 11 SIGNED */
4785 2733 : int prec2 = MAX (prec0 < 0 ? -prec0 : prec0,
4786 : prec1 < 0 ? -prec1 : prec1);
4787 : /* If operands are either both signed or both unsigned,
4788 : we need just one additional bit. */
4789 3777 : prec2 = (((prec0 < 0) == (prec1 < 0)
4790 : /* If one operand is signed and one unsigned and
4791 : the signed one has larger precision, we need
4792 : just one extra bit, otherwise two. */
4793 707 : || (prec0 < 0 ? (prec2 == -prec0 && prec2 != prec1)
4794 337 : : (prec2 == -prec1 && prec2 != prec0)))
4795 2733 : ? prec2 + 1 : prec2 + 2);
4796 2733 : int prec3 = MAX (prec0 < 0 ? -prec0 : prec0,
4797 : prec1 < 0 ? -prec1 : prec1);
4798 2733 : prec3 = MAX (prec3, prec);
4799 2733 : tree var = NULL_TREE;
4800 2733 : tree orig_obj = obj;
4801 2733 : if (obj == NULL_TREE
4802 1753 : && BITINT_TYPE_P (type)
4803 1652 : && bitint_precision_kind (type) >= bitint_prec_large
4804 1536 : && m_names
4805 4245 : && bitmap_bit_p (m_names, SSA_NAME_VERSION (lhs)))
4806 : {
4807 1457 : int part = var_to_partition (m_map, lhs);
4808 1457 : gcc_assert (m_vars[part] != NULL_TREE);
4809 1457 : obj = m_vars[part];
4810 1457 : if (TREE_TYPE (lhs) == type)
4811 14 : orig_obj = obj;
4812 : }
4813 101 : if (!BITINT_TYPE_P (type)
4814 2733 : || bitint_precision_kind (type) < bitint_prec_large)
4815 : {
4816 217 : unsigned HOST_WIDE_INT nelts = CEIL (prec, limb_prec);
4817 217 : tree atype = build_array_type_nelts (m_limb_type, nelts);
4818 217 : var = create_tmp_var (atype);
4819 : }
4820 :
4821 2733 : enum tree_code code;
4822 2733 : switch (gimple_call_internal_fn (stmt))
4823 : {
4824 : case IFN_ADD_OVERFLOW:
4825 : case IFN_UBSAN_CHECK_ADD:
4826 : code = PLUS_EXPR;
4827 : break;
4828 1399 : case IFN_SUB_OVERFLOW:
4829 1399 : case IFN_UBSAN_CHECK_SUB:
4830 1399 : code = MINUS_EXPR;
4831 1399 : break;
4832 0 : default:
4833 0 : gcc_unreachable ();
4834 : }
4835 2733 : unsigned start, end;
4836 2733 : bool check_zero;
4837 2733 : tree ovf = arith_overflow (code, type, prec, prec0, prec1, prec2,
4838 : &start, &end, &check_zero);
4839 :
4840 2733 : unsigned startlimb, endlimb;
4841 2733 : if (ovf)
4842 : {
4843 : startlimb = ~0U;
4844 : endlimb = ~0U;
4845 : }
4846 : else
4847 : {
4848 2733 : startlimb = start / limb_prec;
4849 2733 : endlimb = (end - 1) / limb_prec;
4850 : }
4851 :
4852 2733 : int prec4 = ovf != NULL_TREE ? prec : prec3;
4853 2733 : bitint_prec_kind kind = bitint_precision_kind (prec4);
4854 2733 : unsigned cnt, rem = 0, fin = 0, nelts;
4855 2733 : tree idx = NULL_TREE, idx_first = NULL_TREE, idx_next = NULL_TREE;
4856 5466 : bool last_ovf = (ovf == NULL_TREE
4857 2733 : && CEIL (prec2, limb_prec) > CEIL (prec3, limb_prec));
4858 2733 : if (kind != bitint_prec_huge)
4859 1539 : nelts = cnt = CEIL (prec4, limb_prec) + last_ovf;
4860 : else
4861 : {
4862 1194 : rem = prec4 % (2 * limb_prec);
4863 1194 : fin = (prec4 - rem) / limb_prec;
4864 1194 : cnt = 2 + CEIL (rem, limb_prec) + last_ovf;
4865 1194 : nelts = fin + cnt - 2;
4866 1194 : idx = idx_first = create_loop (bitint_big_endian
4867 1194 : ? size_int (nelts - 1) : size_zero_node,
4868 : &idx_next);
4869 : }
4870 :
4871 2733 : if (kind == bitint_prec_huge)
4872 1194 : m_upwards_2limb = fin;
4873 2733 : m_upwards = true;
4874 :
4875 2733 : tree type0 = TREE_TYPE (arg0);
4876 2733 : tree type1 = TREE_TYPE (arg1);
4877 2733 : int prec5 = prec3;
4878 2733 : if (bitint_precision_kind (prec5) < bitint_prec_large)
4879 10 : prec5 = MAX (TYPE_PRECISION (type0), TYPE_PRECISION (type1));
4880 2733 : if (TYPE_PRECISION (type0) < prec5)
4881 : {
4882 146 : type0 = build_bitint_type (prec5, TYPE_UNSIGNED (type0));
4883 146 : if (TREE_CODE (arg0) == INTEGER_CST)
4884 27 : arg0 = fold_convert (type0, arg0);
4885 : }
4886 2733 : if (TYPE_PRECISION (type1) < prec5)
4887 : {
4888 156 : type1 = build_bitint_type (prec5, TYPE_UNSIGNED (type1));
4889 156 : if (TREE_CODE (arg1) == INTEGER_CST)
4890 76 : arg1 = fold_convert (type1, arg1);
4891 : }
4892 2733 : unsigned int data_cnt = 0;
4893 2733 : tree last_rhs1 = NULL_TREE, last_rhs2 = NULL_TREE;
4894 2733 : tree cmp = build_zero_cst (m_limb_type);
4895 2733 : unsigned prec_limbs = CEIL ((unsigned) prec, limb_prec);
4896 2733 : tree ovf_out = NULL_TREE, cmp_out = NULL_TREE;
4897 11904 : for (unsigned i = 0; i < cnt; i++)
4898 : {
4899 9171 : m_data_cnt = 0;
4900 9171 : tree rhs1, rhs2;
4901 9171 : if (kind != bitint_prec_huge)
4902 5303 : idx = size_int (bitint_big_endian ? nelts - 1 - i : i);
4903 3868 : else if (i >= 2)
4904 1480 : idx = size_int (bitint_big_endian ? nelts + 1 - fin - i : fin + i - 2);
4905 9171 : if (!last_ovf || i < cnt - 1)
4906 : {
4907 8225 : tree idx0 = idx, idx1 = idx;
4908 8225 : if (bitint_big_endian
4909 8225 : && CEIL ((unsigned) TYPE_PRECISION (type0), limb_prec) != nelts)
4910 : {
4911 0 : HOST_WIDE_INT diff
4912 0 : = ((HOST_WIDE_INT) CEIL (TYPE_PRECISION (type0), limb_prec)
4913 0 : - (HOST_WIDE_INT) nelts);
4914 0 : if (tree_fits_uhwi_p (idx))
4915 0 : idx0 = size_int (tree_to_uhwi (idx) + diff);
4916 : else
4917 : {
4918 0 : idx0 = make_ssa_name (sizetype);
4919 0 : g = gimple_build_assign (idx0, PLUS_EXPR, idx,
4920 0 : size_int (diff));
4921 0 : insert_before (g);
4922 : }
4923 : }
4924 8225 : if (type0 != TREE_TYPE (arg0))
4925 334 : rhs1 = handle_cast (type0, arg0, idx0);
4926 : else
4927 7891 : rhs1 = handle_operand (arg0, idx0);
4928 8225 : if (bitint_big_endian
4929 8225 : && CEIL ((unsigned) TYPE_PRECISION (type1), limb_prec) != nelts)
4930 : {
4931 0 : HOST_WIDE_INT diff
4932 0 : = ((HOST_WIDE_INT) CEIL (TYPE_PRECISION (type1), limb_prec)
4933 0 : - (HOST_WIDE_INT) nelts);
4934 0 : if (tree_fits_uhwi_p (idx))
4935 0 : idx1 = size_int (tree_to_uhwi (idx) + diff);
4936 : else
4937 : {
4938 0 : idx1 = make_ssa_name (sizetype);
4939 0 : g = gimple_build_assign (idx1, PLUS_EXPR, idx,
4940 0 : size_int (diff));
4941 0 : insert_before (g);
4942 : }
4943 : }
4944 8225 : if (type1 != TREE_TYPE (arg1))
4945 250 : rhs2 = handle_cast (type1, arg1, idx1);
4946 : else
4947 7975 : rhs2 = handle_operand (arg1, idx1);
4948 8225 : if (i == 0)
4949 2733 : data_cnt = m_data_cnt;
4950 8225 : if (!useless_type_conversion_p (m_limb_type, TREE_TYPE (rhs1)))
4951 1983 : rhs1 = add_cast (m_limb_type, rhs1);
4952 8225 : if (!useless_type_conversion_p (m_limb_type, TREE_TYPE (rhs2)))
4953 1983 : rhs2 = add_cast (m_limb_type, rhs2);
4954 : last_rhs1 = rhs1;
4955 : last_rhs2 = rhs2;
4956 : }
4957 : else
4958 : {
4959 946 : m_data_cnt = data_cnt;
4960 946 : if (TYPE_UNSIGNED (type0) || prec0 >= 0)
4961 421 : rhs1 = build_zero_cst (m_limb_type);
4962 : else
4963 : {
4964 525 : rhs1 = add_cast (signed_type_for (m_limb_type), last_rhs1);
4965 525 : if (TREE_CODE (rhs1) == INTEGER_CST)
4966 52 : rhs1 = build_int_cst (m_limb_type,
4967 74 : tree_int_cst_sgn (rhs1) < 0 ? -1 : 0);
4968 : else
4969 : {
4970 946 : tree lpm1 = build_int_cst (unsigned_type_node,
4971 473 : limb_prec - 1);
4972 473 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (rhs1)),
4973 : RSHIFT_EXPR, rhs1, lpm1);
4974 473 : insert_before (g);
4975 473 : rhs1 = add_cast (m_limb_type, gimple_assign_lhs (g));
4976 : }
4977 : }
4978 946 : if (TYPE_UNSIGNED (type1) || prec1 >= 0)
4979 543 : rhs2 = build_zero_cst (m_limb_type);
4980 : else
4981 : {
4982 403 : rhs2 = add_cast (signed_type_for (m_limb_type), last_rhs2);
4983 403 : if (TREE_CODE (rhs2) == INTEGER_CST)
4984 114 : rhs2 = build_int_cst (m_limb_type,
4985 153 : tree_int_cst_sgn (rhs2) < 0 ? -1 : 0);
4986 : else
4987 : {
4988 578 : tree lpm1 = build_int_cst (unsigned_type_node,
4989 289 : limb_prec - 1);
4990 289 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (rhs2)),
4991 : RSHIFT_EXPR, rhs2, lpm1);
4992 289 : insert_before (g);
4993 289 : rhs2 = add_cast (m_limb_type, gimple_assign_lhs (g));
4994 : }
4995 : }
4996 : }
4997 9171 : tree rhs = handle_plus_minus (code, rhs1, rhs2, idx);
4998 9171 : if (ovf != boolean_false_node)
4999 : {
5000 9171 : if (tree_fits_uhwi_p (idx))
5001 : {
5002 6783 : unsigned limb = tree_to_uhwi (idx);
5003 6783 : if (bitint_big_endian)
5004 0 : limb = nelts - 1 - limb;
5005 6783 : if (limb >= startlimb && limb <= endlimb)
5006 : {
5007 3370 : tree l = arith_overflow_extract_bits (start, end, rhs,
5008 : limb, check_zero);
5009 3370 : tree this_ovf = make_ssa_name (boolean_type_node);
5010 3370 : if (ovf == NULL_TREE && !check_zero)
5011 : {
5012 907 : cmp = l;
5013 907 : g = gimple_build_assign (make_ssa_name (m_limb_type),
5014 : PLUS_EXPR, l,
5015 : build_int_cst (m_limb_type, 1));
5016 907 : insert_before (g);
5017 907 : g = gimple_build_assign (this_ovf, GT_EXPR,
5018 : gimple_assign_lhs (g),
5019 : build_int_cst (m_limb_type, 1));
5020 : }
5021 : else
5022 2463 : g = gimple_build_assign (this_ovf, NE_EXPR, l, cmp);
5023 3370 : insert_before (g);
5024 3370 : if (ovf == NULL_TREE)
5025 : ovf = this_ovf;
5026 : else
5027 : {
5028 1064 : tree b = make_ssa_name (boolean_type_node);
5029 1064 : g = gimple_build_assign (b, BIT_IOR_EXPR, ovf, this_ovf);
5030 1064 : insert_before (g);
5031 1064 : ovf = b;
5032 : }
5033 : }
5034 : }
5035 2388 : else if (startlimb < fin)
5036 : {
5037 854 : if (m_first && startlimb + 2 < fin)
5038 : {
5039 324 : tree data_out;
5040 324 : ovf = prepare_data_in_out (boolean_false_node, idx, &data_out);
5041 324 : ovf_out = m_data.pop ();
5042 324 : m_data.pop ();
5043 324 : if (!check_zero)
5044 : {
5045 169 : cmp = prepare_data_in_out (cmp, idx, &data_out);
5046 169 : cmp_out = m_data.pop ();
5047 169 : m_data.pop ();
5048 : }
5049 : }
5050 854 : if (i != 0 || startlimb != fin - 1)
5051 : {
5052 839 : tree_code cmp_code;
5053 839 : bool single_comparison
5054 839 : = (startlimb + 2 >= fin || (startlimb & 1) != (i & 1));
5055 : if (!single_comparison)
5056 : cmp_code = GE_EXPR;
5057 515 : else if ((startlimb & 1) == (i & 1))
5058 : cmp_code = EQ_EXPR;
5059 : else
5060 412 : cmp_code = GT_EXPR;
5061 839 : if (bitint_big_endian)
5062 0 : g = gimple_build_cond (swap_tree_comparison (cmp_code),
5063 0 : idx, size_int (nelts - 1
5064 : - startlimb),
5065 : NULL_TREE, NULL_TREE);
5066 : else
5067 839 : g = gimple_build_cond (cmp_code, idx, size_int (startlimb),
5068 : NULL_TREE, NULL_TREE);
5069 839 : edge edge_true_true, edge_true_false, edge_false;
5070 839 : gimple *g2 = NULL;
5071 839 : if (!single_comparison)
5072 324 : g2 = gimple_build_cond (NE_EXPR, idx,
5073 324 : size_int (bitint_big_endian
5074 : ? nelts - 1 - startlimb
5075 : : startlimb),
5076 : NULL_TREE, NULL_TREE);
5077 839 : if_then_if_then_else (g, g2, profile_probability::likely (),
5078 : profile_probability::likely (),
5079 : edge_true_true, edge_true_false,
5080 : edge_false);
5081 839 : unsigned tidx = startlimb + (cmp_code == GT_EXPR);
5082 839 : tree l = arith_overflow_extract_bits (start, end, rhs, tidx,
5083 : check_zero);
5084 839 : tree this_ovf = make_ssa_name (boolean_type_node);
5085 839 : if (cmp_code != GT_EXPR && !check_zero)
5086 : {
5087 173 : g = gimple_build_assign (make_ssa_name (m_limb_type),
5088 : PLUS_EXPR, l,
5089 : build_int_cst (m_limb_type, 1));
5090 173 : insert_before (g);
5091 173 : g = gimple_build_assign (this_ovf, GT_EXPR,
5092 : gimple_assign_lhs (g),
5093 : build_int_cst (m_limb_type, 1));
5094 : }
5095 : else
5096 666 : g = gimple_build_assign (this_ovf, NE_EXPR, l, cmp);
5097 839 : insert_before (g);
5098 839 : if (cmp_code == GT_EXPR)
5099 : {
5100 412 : tree t = make_ssa_name (boolean_type_node);
5101 412 : g = gimple_build_assign (t, BIT_IOR_EXPR, ovf, this_ovf);
5102 412 : insert_before (g);
5103 412 : this_ovf = t;
5104 : }
5105 839 : tree this_ovf2 = NULL_TREE;
5106 839 : if (!single_comparison)
5107 : {
5108 324 : m_gsi = gsi_after_labels (edge_true_true->src);
5109 324 : tree t = make_ssa_name (boolean_type_node);
5110 324 : g = gimple_build_assign (t, NE_EXPR, rhs, cmp);
5111 324 : insert_before (g);
5112 324 : this_ovf2 = make_ssa_name (boolean_type_node);
5113 324 : g = gimple_build_assign (this_ovf2, BIT_IOR_EXPR,
5114 : ovf, t);
5115 324 : insert_before (g);
5116 : }
5117 839 : m_gsi = gsi_after_labels (edge_true_false->dest);
5118 839 : tree t;
5119 839 : if (i == 1 && ovf_out)
5120 : t = ovf_out;
5121 : else
5122 515 : t = make_ssa_name (boolean_type_node);
5123 839 : gphi *phi = create_phi_node (t, edge_true_false->dest);
5124 839 : add_phi_arg (phi, this_ovf, edge_true_false,
5125 : UNKNOWN_LOCATION);
5126 839 : add_phi_arg (phi, ovf ? ovf
5127 : : boolean_false_node, edge_false,
5128 : UNKNOWN_LOCATION);
5129 839 : if (edge_true_true)
5130 324 : add_phi_arg (phi, this_ovf2, edge_true_true,
5131 : UNKNOWN_LOCATION);
5132 839 : ovf = t;
5133 839 : if (!check_zero && cmp_code != GT_EXPR)
5134 : {
5135 173 : t = cmp_out ? cmp_out : make_ssa_name (m_limb_type);
5136 173 : phi = create_phi_node (t, edge_true_false->dest);
5137 173 : add_phi_arg (phi, l, edge_true_false, UNKNOWN_LOCATION);
5138 173 : add_phi_arg (phi, cmp, edge_false, UNKNOWN_LOCATION);
5139 173 : if (edge_true_true)
5140 169 : add_phi_arg (phi, cmp, edge_true_true,
5141 : UNKNOWN_LOCATION);
5142 : cmp = t;
5143 : }
5144 : }
5145 : }
5146 : }
5147 :
5148 9171 : if (var || obj)
5149 : {
5150 8943 : if (tree_fits_uhwi_p (idx)
5151 6673 : && (bitint_big_endian
5152 6673 : ? nelts - 1 - tree_to_uhwi (idx)
5153 6673 : : tree_to_uhwi (idx)) >= prec_limbs)
5154 : ;
5155 7555 : else if (!tree_fits_uhwi_p (idx)
5156 2270 : && (unsigned) prec < (fin - (i == 0)) * limb_prec)
5157 : {
5158 1458 : bool single_comparison
5159 729 : = (((unsigned) prec % limb_prec) == 0
5160 571 : || prec_limbs + 1 >= fin
5161 1231 : || (prec_limbs & 1) == (i & 1));
5162 729 : if (bitint_big_endian)
5163 0 : g = gimple_build_cond (GE_EXPR, idx,
5164 0 : size_int (nelts - prec_limbs),
5165 : NULL_TREE, NULL_TREE);
5166 : else
5167 729 : g = gimple_build_cond (LE_EXPR, idx, size_int (prec_limbs - 1),
5168 : NULL_TREE, NULL_TREE);
5169 729 : gimple *g2 = NULL;
5170 729 : if (!single_comparison)
5171 251 : g2 = gimple_build_cond (EQ_EXPR, idx,
5172 251 : size_int (bitint_big_endian
5173 : ? nelts - prec_limbs
5174 : : prec_limbs - 1),
5175 : NULL_TREE, NULL_TREE);
5176 729 : edge edge_true_true, edge_true_false, edge_false;
5177 729 : if_then_if_then_else (g, g2, profile_probability::likely (),
5178 : profile_probability::unlikely (),
5179 : edge_true_true, edge_true_false,
5180 : edge_false);
5181 729 : tree idxl = idx;
5182 729 : if (bitint_big_endian && prec_limbs != nelts)
5183 : {
5184 0 : HOST_WIDE_INT diff = ((HOST_WIDE_INT) prec_limbs
5185 0 : - (HOST_WIDE_INT) nelts);
5186 0 : if (tree_fits_uhwi_p (idx))
5187 0 : idxl = size_int (tree_to_uhwi (idx) + diff);
5188 : else
5189 : {
5190 0 : idxl = make_ssa_name (sizetype);
5191 0 : g = gimple_build_assign (idxl, PLUS_EXPR, idx,
5192 0 : size_int (diff));
5193 0 : insert_before (g);
5194 : }
5195 : }
5196 1082 : tree l = limb_access (type, var ? var : obj, idxl, true);
5197 729 : g = gimple_build_assign (l, rhs);
5198 729 : insert_before (g);
5199 729 : if (!single_comparison)
5200 : {
5201 251 : m_gsi = gsi_after_labels (edge_true_true->src);
5202 251 : tree plm1idx = size_int (bitint_big_endian
5203 : ? 0 : prec_limbs - 1);
5204 251 : tree plm1type = limb_access_type (type, plm1idx);
5205 251 : l = limb_access (type, var ? var : obj, plm1idx, true);
5206 251 : if (!useless_type_conversion_p (plm1type, TREE_TYPE (rhs)))
5207 251 : rhs = add_cast (plm1type, rhs);
5208 251 : if (!useless_type_conversion_p (TREE_TYPE (l),
5209 251 : TREE_TYPE (rhs)))
5210 251 : rhs = add_cast (TREE_TYPE (l), rhs);
5211 251 : g = gimple_build_assign (l, rhs);
5212 251 : insert_before (g);
5213 : }
5214 729 : m_gsi = gsi_after_labels (edge_true_false->dest);
5215 729 : }
5216 : else
5217 : {
5218 6826 : tree idxl = idx;
5219 6826 : if (bitint_big_endian && prec_limbs != nelts)
5220 : {
5221 0 : HOST_WIDE_INT diff = ((HOST_WIDE_INT) prec_limbs
5222 0 : - (HOST_WIDE_INT) nelts);
5223 0 : if (tree_fits_uhwi_p (idx))
5224 0 : idxl = size_int (tree_to_uhwi (idx) + diff);
5225 : else
5226 : {
5227 0 : idxl = make_ssa_name (sizetype);
5228 0 : g = gimple_build_assign (idxl, PLUS_EXPR, idx,
5229 0 : size_int (diff));
5230 0 : insert_before (g);
5231 : }
5232 : }
5233 13623 : tree l = limb_access (type, var ? var : obj, idxl, true);
5234 6826 : if (bitint_extended && tree_fits_uhwi_p (idxl))
5235 : {
5236 0 : tree atype = limb_access_type (type, idxl);
5237 0 : if (!useless_type_conversion_p (atype, TREE_TYPE (rhs)))
5238 0 : rhs = add_cast (atype, rhs);
5239 : }
5240 6826 : if (!useless_type_conversion_p (TREE_TYPE (l), TREE_TYPE (rhs)))
5241 0 : rhs = add_cast (TREE_TYPE (l), rhs);
5242 6826 : g = gimple_build_assign (l, rhs);
5243 6826 : insert_before (g);
5244 : }
5245 : }
5246 9171 : m_first = false;
5247 9171 : if (kind == bitint_prec_huge && i <= 1)
5248 : {
5249 2388 : if (i == 0)
5250 : {
5251 1194 : idx = make_ssa_name (sizetype);
5252 1194 : g = gimple_build_assign (idx, PLUS_EXPR, idx_first,
5253 : bitint_big_endian
5254 0 : ? size_int (-1) : size_one_node);
5255 1194 : insert_before (g);
5256 : }
5257 : else
5258 : {
5259 1194 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx_first,
5260 2388 : size_int (bitint_big_endian ? -2 : 2));
5261 1194 : insert_before (g);
5262 1194 : if (bitint_big_endian)
5263 0 : g = gimple_build_cond (NE_EXPR, idx_first,
5264 0 : size_int (nelts + 1 - fin),
5265 : NULL_TREE, NULL_TREE);
5266 : else
5267 1194 : g = gimple_build_cond (NE_EXPR, idx_next, size_int (fin),
5268 : NULL_TREE, NULL_TREE);
5269 1194 : insert_before (g);
5270 1194 : m_gsi = gsi_for_stmt (final_stmt);
5271 1194 : m_bb = NULL;
5272 : }
5273 : }
5274 : }
5275 :
5276 2733 : finish_arith_overflow (var, obj, type, ovf, lhs, orig_obj, stmt,
5277 : prec_limbs, code);
5278 : }
5279 :
5280 : /* Lower a .MUL_OVERFLOW call with at least one large/huge _BitInt
5281 : argument or return type _Complex large/huge _BitInt. */
5282 :
5283 : void
5284 1479 : bitint_large_huge::lower_mul_overflow (tree obj, gimple *stmt)
5285 : {
5286 1479 : tree arg0 = gimple_call_arg (stmt, 0);
5287 1479 : tree arg1 = gimple_call_arg (stmt, 1);
5288 1479 : tree lhs = gimple_call_lhs (stmt);
5289 1479 : if (!lhs)
5290 : {
5291 0 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
5292 0 : gsi_remove (&gsi, true);
5293 0 : return;
5294 : }
5295 1479 : gimple *final_stmt = gsi_stmt (m_gsi);
5296 1479 : tree type = TREE_TYPE (lhs);
5297 1479 : if (TREE_CODE (type) == COMPLEX_TYPE)
5298 1464 : type = TREE_TYPE (type);
5299 1479 : int prec = TYPE_PRECISION (type), prec0, prec1;
5300 1479 : arg0 = handle_operand_addr (arg0, stmt, NULL, &prec0);
5301 1479 : arg1 = handle_operand_addr (arg1, stmt, NULL, &prec1);
5302 1479 : int prec2 = ((prec0 < 0 ? -prec0 : prec0)
5303 1479 : + (prec1 < 0 ? -prec1 : prec1));
5304 1479 : if (prec0 == 1 || prec1 == 1)
5305 35 : --prec2;
5306 1479 : tree var = NULL_TREE;
5307 1479 : tree orig_obj = obj;
5308 1479 : bool force_var = false;
5309 1479 : if (obj == NULL_TREE
5310 886 : && BITINT_TYPE_P (type)
5311 877 : && bitint_precision_kind (type) >= bitint_prec_large
5312 853 : && m_names
5313 2320 : && bitmap_bit_p (m_names, SSA_NAME_VERSION (lhs)))
5314 : {
5315 841 : int part = var_to_partition (m_map, lhs);
5316 841 : gcc_assert (m_vars[part] != NULL_TREE);
5317 841 : obj = m_vars[part];
5318 841 : if (TREE_TYPE (lhs) == type)
5319 7 : orig_obj = obj;
5320 : }
5321 638 : else if (obj != NULL_TREE && DECL_P (obj))
5322 : {
5323 1755 : for (int i = 0; i < 2; ++i)
5324 : {
5325 1170 : tree arg = i ? arg1 : arg0;
5326 1170 : if (TREE_CODE (arg) == ADDR_EXPR)
5327 1170 : arg = TREE_OPERAND (arg, 0);
5328 1170 : if (get_base_address (arg) == obj)
5329 : {
5330 : force_var = true;
5331 : break;
5332 : }
5333 : }
5334 : }
5335 1479 : if (obj == NULL_TREE
5336 1479 : || force_var
5337 1434 : || !BITINT_TYPE_P (type)
5338 1434 : || bitint_precision_kind (type) < bitint_prec_large
5339 3747 : || prec2 > (CEIL (prec, limb_prec) * limb_prec * (orig_obj ? 1 : 2)))
5340 : {
5341 564 : unsigned HOST_WIDE_INT nelts = CEIL (MAX (prec, prec2), limb_prec);
5342 564 : tree atype = build_array_type_nelts (m_limb_type, nelts);
5343 564 : var = create_tmp_var (atype);
5344 : }
5345 1479 : tree addr = build_fold_addr_expr (var ? var : obj);
5346 1479 : addr = force_gimple_operand_gsi (&m_gsi, addr, true,
5347 : NULL_TREE, true, GSI_SAME_STMT);
5348 1479 : tree sitype = lang_hooks.types.type_for_mode (SImode, 0);
5349 1479 : gimple *g
5350 1479 : = gimple_build_call_internal (IFN_MULBITINT, 6,
5351 : addr, build_int_cst (sitype,
5352 1479 : MAX (prec2, prec)),
5353 1479 : arg0, build_int_cst (sitype, prec0),
5354 1479 : arg1, build_int_cst (sitype, prec1));
5355 1479 : insert_before (g);
5356 :
5357 1479 : unsigned start, end;
5358 1479 : bool check_zero;
5359 1479 : tree ovf = arith_overflow (MULT_EXPR, type, prec, prec0, prec1, prec2,
5360 : &start, &end, &check_zero);
5361 1479 : if (ovf == NULL_TREE)
5362 : {
5363 1477 : unsigned startlimb = start / limb_prec;
5364 1477 : unsigned endlimb = (end - 1) / limb_prec;
5365 1477 : unsigned nelts = CEIL (MAX (prec, prec2), limb_prec);
5366 1477 : unsigned cnt;
5367 1477 : bool use_loop = false;
5368 1477 : if (startlimb == endlimb)
5369 : cnt = 1;
5370 1190 : else if (startlimb + 1 == endlimb)
5371 : cnt = 2;
5372 1013 : else if ((end % limb_prec) == 0)
5373 : {
5374 : cnt = 2;
5375 : use_loop = true;
5376 : }
5377 : else
5378 : {
5379 767 : cnt = 3;
5380 767 : use_loop = startlimb + 2 < endlimb;
5381 : }
5382 767 : if (cnt == 1)
5383 : {
5384 507 : tree l = limb_access (NULL_TREE, var ? var : obj,
5385 287 : size_int (bitint_big_endian
5386 : ? nelts - 1 - startlimb
5387 : : startlimb), true);
5388 287 : g = gimple_build_assign (make_ssa_name (m_limb_type), l);
5389 287 : insert_before (g);
5390 287 : l = arith_overflow_extract_bits (start, end, gimple_assign_lhs (g),
5391 : startlimb, check_zero);
5392 287 : ovf = make_ssa_name (boolean_type_node);
5393 287 : if (check_zero)
5394 247 : g = gimple_build_assign (ovf, NE_EXPR, l,
5395 : build_zero_cst (m_limb_type));
5396 : else
5397 : {
5398 40 : g = gimple_build_assign (make_ssa_name (m_limb_type),
5399 : PLUS_EXPR, l,
5400 : build_int_cst (m_limb_type, 1));
5401 40 : insert_before (g);
5402 40 : g = gimple_build_assign (ovf, GT_EXPR, gimple_assign_lhs (g),
5403 : build_int_cst (m_limb_type, 1));
5404 : }
5405 287 : insert_before (g);
5406 : }
5407 : else
5408 : {
5409 1190 : basic_block edge_bb = NULL;
5410 1190 : gimple_stmt_iterator gsi = m_gsi;
5411 1190 : gsi_prev (&gsi);
5412 1190 : edge e = split_block (gsi_bb (gsi), gsi_stmt (gsi));
5413 1190 : edge_bb = e->src;
5414 1190 : m_gsi = gsi_end_bb (edge_bb);
5415 :
5416 1190 : tree cmp = build_zero_cst (m_limb_type);
5417 5527 : for (unsigned i = 0; i < cnt; i++)
5418 : {
5419 3147 : tree idx, idx_next = NULL_TREE;
5420 3147 : if (i == 0)
5421 1190 : idx = size_int (bitint_big_endian
5422 : ? nelts - 1 - startlimb : startlimb);
5423 1957 : else if (i == 2)
5424 767 : idx = size_int (bitint_big_endian
5425 : ? nelts - 1 - endlimb : endlimb);
5426 1190 : else if (use_loop)
5427 665 : idx = create_loop (size_int (bitint_big_endian
5428 : ? nelts - startlimb - 2
5429 : : startlimb + 1), &idx_next);
5430 : else
5431 525 : idx = size_int (bitint_big_endian
5432 : ? nelts - startlimb - 2 : startlimb + 1);
5433 4988 : tree l = limb_access (NULL_TREE, var ? var : obj, idx, true);
5434 3147 : g = gimple_build_assign (make_ssa_name (m_limb_type), l);
5435 3147 : insert_before (g);
5436 3147 : l = gimple_assign_lhs (g);
5437 3147 : if (i == 0 || i == 2)
5438 2724 : l = arith_overflow_extract_bits (start, end, l,
5439 : i == 0 ? startlimb : endlimb,
5440 : check_zero);
5441 1957 : if (i == 0 && !check_zero)
5442 : {
5443 459 : cmp = l;
5444 459 : g = gimple_build_assign (make_ssa_name (m_limb_type),
5445 : PLUS_EXPR, l,
5446 : build_int_cst (m_limb_type, 1));
5447 459 : insert_before (g);
5448 459 : g = gimple_build_cond (GT_EXPR, gimple_assign_lhs (g),
5449 : build_int_cst (m_limb_type, 1),
5450 : NULL_TREE, NULL_TREE);
5451 : }
5452 : else
5453 2688 : g = gimple_build_cond (NE_EXPR, l, cmp, NULL_TREE, NULL_TREE);
5454 3147 : insert_before (g);
5455 3147 : edge e1 = split_block (gsi_bb (m_gsi), g);
5456 3147 : e1->flags = EDGE_FALSE_VALUE;
5457 3147 : edge e2 = make_edge (e1->src, gimple_bb (final_stmt),
5458 : EDGE_TRUE_VALUE);
5459 3147 : e1->probability = profile_probability::likely ();
5460 3147 : e2->probability = e1->probability.invert ();
5461 3147 : if (i == 0)
5462 1190 : set_immediate_dominator (CDI_DOMINATORS, e2->dest, e2->src);
5463 3147 : m_gsi = gsi_after_labels (e1->dest);
5464 3147 : if (i == 1 && use_loop)
5465 : {
5466 665 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
5467 : bitint_big_endian
5468 0 : ? size_int (-1) : size_one_node);
5469 665 : insert_before (g);
5470 665 : if (bitint_big_endian)
5471 0 : g = gimple_build_cond (NE_EXPR, idx,
5472 0 : size_int (nelts - endlimb
5473 : - (cnt == 2)),
5474 : NULL_TREE, NULL_TREE);
5475 : else
5476 665 : g = gimple_build_cond (NE_EXPR, idx_next,
5477 665 : size_int (endlimb + (cnt == 2)),
5478 : NULL_TREE, NULL_TREE);
5479 665 : insert_before (g);
5480 665 : edge true_edge, false_edge;
5481 665 : extract_true_false_edges_from_block (gsi_bb (m_gsi),
5482 : &true_edge,
5483 : &false_edge);
5484 665 : m_gsi = gsi_after_labels (false_edge->dest);
5485 665 : m_bb = NULL;
5486 : }
5487 : }
5488 :
5489 1190 : ovf = make_ssa_name (boolean_type_node);
5490 1190 : basic_block bb = gimple_bb (final_stmt);
5491 1190 : gphi *phi = create_phi_node (ovf, bb);
5492 1190 : edge e1 = find_edge (gsi_bb (m_gsi), bb);
5493 1190 : edge_iterator ei;
5494 5527 : FOR_EACH_EDGE (e, ei, bb->preds)
5495 : {
5496 4337 : tree val = e == e1 ? boolean_false_node : boolean_true_node;
5497 4337 : add_phi_arg (phi, val, e, UNKNOWN_LOCATION);
5498 : }
5499 1190 : m_gsi = gsi_for_stmt (final_stmt);
5500 : }
5501 : }
5502 :
5503 1479 : finish_arith_overflow (var, obj, type, ovf, lhs, orig_obj, stmt,
5504 1479 : CEIL (MAX (prec, prec2), limb_prec), MULT_EXPR);
5505 : }
5506 :
5507 : /* Lower REALPART_EXPR or IMAGPART_EXPR stmt extracting part of result from
5508 : .{ADD,SUB,MUL}_OVERFLOW call. */
5509 :
5510 : void
5511 6002 : bitint_large_huge::lower_cplxpart_stmt (tree obj, gimple *stmt)
5512 : {
5513 6002 : tree rhs1 = gimple_assign_rhs1 (stmt);
5514 6002 : rhs1 = TREE_OPERAND (rhs1, 0);
5515 6002 : if (obj == NULL_TREE)
5516 : {
5517 5929 : int part = var_to_partition (m_map, gimple_assign_lhs (stmt));
5518 5929 : gcc_assert (m_vars[part] != NULL_TREE);
5519 : obj = m_vars[part];
5520 : }
5521 6002 : if (TREE_CODE (rhs1) == SSA_NAME
5522 6002 : && (m_names == NULL
5523 6001 : || !bitmap_bit_p (m_names, SSA_NAME_VERSION (rhs1))))
5524 : {
5525 1549 : lower_call (obj, SSA_NAME_DEF_STMT (rhs1));
5526 1549 : return;
5527 : }
5528 4453 : int part = var_to_partition (m_map, rhs1);
5529 4453 : gcc_assert (m_vars[part] != NULL_TREE);
5530 4453 : tree var = m_vars[part];
5531 4453 : unsigned HOST_WIDE_INT nelts
5532 4453 : = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (obj))) / limb_prec;
5533 4453 : tree atype = build_array_type_nelts (m_limb_type, nelts);
5534 4453 : if (!useless_type_conversion_p (atype, TREE_TYPE (obj)))
5535 102 : obj = build1 (VIEW_CONVERT_EXPR, atype, obj);
5536 4453 : tree off = build_int_cst (build_pointer_type (TREE_TYPE (var)),
5537 4453 : gimple_assign_rhs_code (stmt) == REALPART_EXPR
5538 4453 : ? 0 : nelts * m_limb_size);
5539 4453 : tree v2 = build2 (MEM_REF, atype, build_fold_addr_expr (var), off);
5540 4453 : gimple *g = gimple_build_assign (obj, v2);
5541 4453 : insert_before (g);
5542 : }
5543 :
5544 : /* Lower COMPLEX_EXPR stmt. */
5545 :
5546 : void
5547 18 : bitint_large_huge::lower_complexexpr_stmt (gimple *stmt)
5548 : {
5549 18 : tree lhs = gimple_assign_lhs (stmt);
5550 18 : tree rhs1 = gimple_assign_rhs1 (stmt);
5551 18 : tree rhs2 = gimple_assign_rhs2 (stmt);
5552 18 : int part = var_to_partition (m_map, lhs);
5553 18 : gcc_assert (m_vars[part] != NULL_TREE);
5554 18 : lhs = m_vars[part];
5555 18 : unsigned HOST_WIDE_INT nelts
5556 18 : = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (rhs1))) / limb_prec;
5557 18 : tree atype = build_array_type_nelts (m_limb_type, nelts);
5558 18 : tree zero = build_zero_cst (build_pointer_type (TREE_TYPE (lhs)));
5559 18 : tree v1 = build2 (MEM_REF, atype, build_fold_addr_expr (lhs), zero);
5560 18 : tree v2;
5561 18 : if (TREE_CODE (rhs1) == SSA_NAME)
5562 : {
5563 18 : part = var_to_partition (m_map, rhs1);
5564 18 : gcc_assert (m_vars[part] != NULL_TREE);
5565 : v2 = m_vars[part];
5566 : }
5567 0 : else if (integer_zerop (rhs1))
5568 0 : v2 = build_zero_cst (atype);
5569 : else
5570 0 : v2 = tree_output_constant_def (rhs1);
5571 18 : if (!useless_type_conversion_p (atype, TREE_TYPE (v2)))
5572 18 : v2 = build1 (VIEW_CONVERT_EXPR, atype, v2);
5573 18 : gimple *g = gimple_build_assign (v1, v2);
5574 18 : insert_before (g);
5575 18 : tree off = fold_convert (build_pointer_type (TREE_TYPE (lhs)),
5576 : TYPE_SIZE_UNIT (atype));
5577 18 : v1 = build2 (MEM_REF, atype, build_fold_addr_expr (lhs), off);
5578 18 : if (TREE_CODE (rhs2) == SSA_NAME)
5579 : {
5580 0 : part = var_to_partition (m_map, rhs2);
5581 0 : gcc_assert (m_vars[part] != NULL_TREE);
5582 : v2 = m_vars[part];
5583 : }
5584 18 : else if (integer_zerop (rhs2))
5585 18 : v2 = build_zero_cst (atype);
5586 : else
5587 0 : v2 = tree_output_constant_def (rhs2);
5588 18 : if (!useless_type_conversion_p (atype, TREE_TYPE (v2)))
5589 0 : v2 = build1 (VIEW_CONVERT_EXPR, atype, v2);
5590 18 : g = gimple_build_assign (v1, v2);
5591 18 : insert_before (g);
5592 18 : }
5593 :
5594 : /* Lower a .{CLZ,CTZ,CLRSB,FFS,PARITY,POPCOUNT} call with one large/huge _BitInt
5595 : argument. */
5596 :
5597 : void
5598 101 : bitint_large_huge::lower_bit_query (gimple *stmt)
5599 : {
5600 101 : tree arg0 = gimple_call_arg (stmt, 0);
5601 101 : tree arg1 = (gimple_call_num_args (stmt) == 2
5602 101 : ? gimple_call_arg (stmt, 1) : NULL_TREE);
5603 101 : tree lhs = gimple_call_lhs (stmt);
5604 101 : gimple *g;
5605 :
5606 101 : if (!lhs)
5607 : {
5608 0 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
5609 0 : gsi_remove (&gsi, true);
5610 0 : return;
5611 : }
5612 101 : tree type = TREE_TYPE (arg0);
5613 101 : gcc_assert (BITINT_TYPE_P (type));
5614 101 : bitint_prec_kind kind = bitint_precision_kind (type);
5615 101 : gcc_assert (kind >= bitint_prec_large);
5616 101 : enum internal_fn ifn = gimple_call_internal_fn (stmt);
5617 101 : enum built_in_function fcode = END_BUILTINS;
5618 101 : gcc_assert (TYPE_PRECISION (unsigned_type_node) == limb_prec
5619 : || TYPE_PRECISION (long_unsigned_type_node) == limb_prec
5620 : || TYPE_PRECISION (long_long_unsigned_type_node) == limb_prec);
5621 101 : switch (ifn)
5622 : {
5623 26 : case IFN_CLZ:
5624 26 : if (TYPE_PRECISION (unsigned_type_node) == limb_prec)
5625 : fcode = BUILT_IN_CLZ;
5626 26 : else if (TYPE_PRECISION (long_unsigned_type_node) == limb_prec)
5627 : fcode = BUILT_IN_CLZL;
5628 : else
5629 0 : fcode = BUILT_IN_CLZLL;
5630 : break;
5631 11 : case IFN_FFS:
5632 : /* .FFS (X) is .CTZ (X, -1) + 1, though under the hood
5633 : we don't add the addend at the end. */
5634 11 : arg1 = integer_zero_node;
5635 : /* FALLTHRU */
5636 39 : case IFN_CTZ:
5637 39 : if (TYPE_PRECISION (unsigned_type_node) == limb_prec)
5638 : fcode = BUILT_IN_CTZ;
5639 39 : else if (TYPE_PRECISION (long_unsigned_type_node) == limb_prec)
5640 : fcode = BUILT_IN_CTZL;
5641 : else
5642 0 : fcode = BUILT_IN_CTZLL;
5643 39 : m_upwards = true;
5644 39 : break;
5645 9 : case IFN_CLRSB:
5646 9 : if (TYPE_PRECISION (unsigned_type_node) == limb_prec)
5647 : fcode = BUILT_IN_CLRSB;
5648 9 : else if (TYPE_PRECISION (long_unsigned_type_node) == limb_prec)
5649 : fcode = BUILT_IN_CLRSBL;
5650 : else
5651 0 : fcode = BUILT_IN_CLRSBLL;
5652 : break;
5653 16 : case IFN_PARITY:
5654 16 : if (TYPE_PRECISION (unsigned_type_node) == limb_prec)
5655 : fcode = BUILT_IN_PARITY;
5656 16 : else if (TYPE_PRECISION (long_unsigned_type_node) == limb_prec)
5657 : fcode = BUILT_IN_PARITYL;
5658 : else
5659 0 : fcode = BUILT_IN_PARITYLL;
5660 16 : m_upwards = true;
5661 16 : break;
5662 11 : case IFN_POPCOUNT:
5663 11 : if (TYPE_PRECISION (unsigned_type_node) == limb_prec)
5664 : fcode = BUILT_IN_POPCOUNT;
5665 11 : else if (TYPE_PRECISION (long_unsigned_type_node) == limb_prec)
5666 : fcode = BUILT_IN_POPCOUNTL;
5667 : else
5668 0 : fcode = BUILT_IN_POPCOUNTLL;
5669 11 : m_upwards = true;
5670 11 : break;
5671 0 : default:
5672 0 : gcc_unreachable ();
5673 : }
5674 101 : tree fndecl = builtin_decl_explicit (fcode), res = NULL_TREE;
5675 101 : unsigned cnt = 0, rem = 0, end = 0, prec = TYPE_PRECISION (type);
5676 101 : unsigned nelts = CEIL (prec, limb_prec);
5677 101 : struct bq_details { edge e; tree val, addend; } *bqp = NULL;
5678 101 : basic_block edge_bb = NULL;
5679 101 : if (m_upwards)
5680 : {
5681 66 : tree idx = NULL_TREE, idx_first = NULL_TREE, idx_next = NULL_TREE;
5682 66 : if (kind == bitint_prec_large)
5683 : cnt = nelts;
5684 : else
5685 : {
5686 39 : rem = (prec % (2 * limb_prec));
5687 39 : end = (prec - rem) / limb_prec;
5688 39 : cnt = 2 + CEIL (rem, limb_prec);
5689 39 : idx = idx_first = create_loop (bitint_big_endian
5690 39 : ? size_int (nelts - 1)
5691 : : size_zero_node, &idx_next);
5692 : }
5693 :
5694 66 : if (ifn == IFN_CTZ || ifn == IFN_FFS)
5695 : {
5696 39 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
5697 39 : gsi_prev (&gsi);
5698 39 : edge e = split_block (gsi_bb (gsi), gsi_stmt (gsi));
5699 39 : edge_bb = e->src;
5700 39 : if (kind == bitint_prec_large)
5701 32 : m_gsi = gsi_end_bb (edge_bb);
5702 39 : bqp = XALLOCAVEC (struct bq_details, cnt);
5703 : }
5704 : else
5705 27 : m_after_stmt = stmt;
5706 66 : if (kind != bitint_prec_large)
5707 39 : m_upwards_2limb = end;
5708 :
5709 244 : for (unsigned i = 0; i < cnt; i++)
5710 : {
5711 178 : m_data_cnt = 0;
5712 178 : if (kind == bitint_prec_large)
5713 83 : idx = size_int (bitint_big_endian ? nelts - 1 - i : i);
5714 95 : else if (i >= 2)
5715 17 : idx = size_int (bitint_big_endian
5716 : ? nelts - 1 - end - (i > 2) : end + (i > 2));
5717 :
5718 178 : tree rhs1 = handle_operand (arg0, idx);
5719 178 : if (!useless_type_conversion_p (m_limb_type, TREE_TYPE (rhs1)))
5720 : {
5721 32 : if (!TYPE_UNSIGNED (TREE_TYPE (rhs1)))
5722 5 : rhs1 = add_cast (unsigned_type_for (TREE_TYPE (rhs1)), rhs1);
5723 32 : rhs1 = add_cast (m_limb_type, rhs1);
5724 : }
5725 :
5726 178 : tree in, out, tem;
5727 178 : if (ifn == IFN_PARITY)
5728 43 : in = prepare_data_in_out (build_zero_cst (m_limb_type), idx, &out);
5729 135 : else if (ifn == IFN_FFS)
5730 29 : in = prepare_data_in_out (integer_one_node, idx, &out);
5731 : else
5732 106 : in = prepare_data_in_out (integer_zero_node, idx, &out);
5733 :
5734 178 : switch (ifn)
5735 : {
5736 104 : case IFN_CTZ:
5737 104 : case IFN_FFS:
5738 104 : g = gimple_build_cond (NE_EXPR, rhs1,
5739 : build_zero_cst (m_limb_type),
5740 : NULL_TREE, NULL_TREE);
5741 104 : insert_before (g);
5742 104 : edge e1, e2;
5743 104 : e1 = split_block (gsi_bb (m_gsi), g);
5744 104 : e1->flags = EDGE_FALSE_VALUE;
5745 104 : e2 = make_edge (e1->src, gimple_bb (stmt), EDGE_TRUE_VALUE);
5746 104 : e1->probability = profile_probability::unlikely ();
5747 104 : e2->probability = e1->probability.invert ();
5748 104 : if (i == 0)
5749 39 : set_immediate_dominator (CDI_DOMINATORS, e2->dest, e2->src);
5750 104 : m_gsi = gsi_after_labels (e1->dest);
5751 104 : bqp[i].e = e2;
5752 104 : bqp[i].val = rhs1;
5753 104 : if (tree_fits_uhwi_p (idx))
5754 58 : bqp[i].addend
5755 58 : = build_int_cst (integer_type_node,
5756 : (bitint_big_endian
5757 58 : ? nelts - 1 - tree_to_uhwi (idx)
5758 58 : : tree_to_uhwi (idx)) * limb_prec
5759 58 : + (ifn == IFN_FFS));
5760 : else
5761 : {
5762 46 : bqp[i].addend = in;
5763 46 : if (i == 1)
5764 23 : res = out;
5765 : else
5766 23 : res = make_ssa_name (integer_type_node);
5767 46 : g = gimple_build_assign (res, PLUS_EXPR, in,
5768 : build_int_cst (integer_type_node,
5769 46 : limb_prec));
5770 46 : insert_before (g);
5771 46 : m_data[m_data_cnt] = res;
5772 : }
5773 : break;
5774 43 : case IFN_PARITY:
5775 43 : if (!integer_zerop (in))
5776 : {
5777 37 : if (kind == bitint_prec_huge && i == 1)
5778 10 : res = out;
5779 : else
5780 27 : res = make_ssa_name (m_limb_type);
5781 37 : g = gimple_build_assign (res, BIT_XOR_EXPR, in, rhs1);
5782 37 : insert_before (g);
5783 : }
5784 : else
5785 : res = rhs1;
5786 43 : m_data[m_data_cnt] = res;
5787 43 : break;
5788 31 : case IFN_POPCOUNT:
5789 31 : g = gimple_build_call (fndecl, 1, rhs1);
5790 31 : tem = make_ssa_name (integer_type_node);
5791 31 : gimple_call_set_lhs (g, tem);
5792 31 : insert_before (g);
5793 31 : if (!integer_zerop (in))
5794 : {
5795 26 : if (kind == bitint_prec_huge && i == 1)
5796 6 : res = out;
5797 : else
5798 20 : res = make_ssa_name (integer_type_node);
5799 26 : g = gimple_build_assign (res, PLUS_EXPR, in, tem);
5800 26 : insert_before (g);
5801 : }
5802 : else
5803 : res = tem;
5804 31 : m_data[m_data_cnt] = res;
5805 31 : break;
5806 0 : default:
5807 0 : gcc_unreachable ();
5808 : }
5809 :
5810 178 : m_first = false;
5811 178 : if (kind == bitint_prec_huge && i <= 1)
5812 : {
5813 78 : if (i == 0)
5814 : {
5815 39 : idx = make_ssa_name (sizetype);
5816 39 : g = gimple_build_assign (idx, PLUS_EXPR, idx_first,
5817 : bitint_big_endian
5818 0 : ? size_int (-1) : size_one_node);
5819 39 : insert_before (g);
5820 : }
5821 : else
5822 : {
5823 39 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx_first,
5824 78 : size_int (bitint_big_endian
5825 : ? -2 : 2));
5826 39 : insert_before (g);
5827 39 : if (bitint_big_endian)
5828 0 : g = gimple_build_cond (NE_EXPR, idx_first,
5829 0 : size_int (cnt - 1),
5830 : NULL_TREE, NULL_TREE);
5831 : else
5832 39 : g = gimple_build_cond (NE_EXPR, idx_next, size_int (end),
5833 : NULL_TREE, NULL_TREE);
5834 39 : insert_before (g);
5835 39 : if (ifn == IFN_CTZ || ifn == IFN_FFS)
5836 23 : m_gsi = gsi_after_labels (edge_bb);
5837 : else
5838 16 : m_gsi = gsi_for_stmt (stmt);
5839 39 : m_bb = NULL;
5840 : }
5841 : }
5842 : }
5843 : }
5844 : else
5845 : {
5846 35 : tree idx = NULL_TREE, idx_next = NULL_TREE, first = NULL_TREE;
5847 35 : int sub_one = 0;
5848 35 : if (kind == bitint_prec_large)
5849 : cnt = nelts;
5850 : else
5851 : {
5852 18 : rem = prec % limb_prec;
5853 18 : if (rem == 0 && (!TYPE_UNSIGNED (type) || ifn == IFN_CLRSB))
5854 : rem = limb_prec;
5855 18 : end = (prec - rem) / limb_prec;
5856 18 : cnt = 1 + (rem != 0);
5857 18 : if (ifn == IFN_CLRSB)
5858 5 : sub_one = 1;
5859 : }
5860 :
5861 35 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
5862 35 : gsi_prev (&gsi);
5863 35 : edge e = split_block (gsi_bb (gsi), gsi_stmt (gsi));
5864 35 : edge_bb = e->src;
5865 35 : m_gsi = gsi_end_bb (edge_bb);
5866 :
5867 35 : if (ifn == IFN_CLZ)
5868 26 : bqp = XALLOCAVEC (struct bq_details, cnt);
5869 : else
5870 : {
5871 9 : gsi = gsi_for_stmt (stmt);
5872 9 : gsi_prev (&gsi);
5873 9 : e = split_block (gsi_bb (gsi), gsi_stmt (gsi));
5874 9 : edge_bb = e->src;
5875 9 : bqp = XALLOCAVEC (struct bq_details, 2 * cnt);
5876 : }
5877 :
5878 116 : for (unsigned i = 0; i < cnt; i++)
5879 : {
5880 81 : m_data_cnt = 0;
5881 81 : if (kind == bitint_prec_large)
5882 51 : idx = size_int (bitint_big_endian ? i : cnt - i - 1);
5883 30 : else if (i == cnt - 1)
5884 18 : idx = create_loop (size_int (bitint_big_endian ? i : end - 1),
5885 : &idx_next);
5886 : else
5887 12 : idx = bitint_big_endian ? size_zero_node : size_int (end);
5888 :
5889 81 : tree rhs1 = handle_operand (arg0, idx);
5890 81 : if (!useless_type_conversion_p (m_limb_type, TREE_TYPE (rhs1)))
5891 : {
5892 19 : if (ifn == IFN_CLZ && !TYPE_UNSIGNED (TREE_TYPE (rhs1)))
5893 0 : rhs1 = add_cast (unsigned_type_for (TREE_TYPE (rhs1)), rhs1);
5894 19 : else if (ifn == IFN_CLRSB && TYPE_UNSIGNED (TREE_TYPE (rhs1)))
5895 0 : rhs1 = add_cast (signed_type_for (TREE_TYPE (rhs1)), rhs1);
5896 19 : rhs1 = add_cast (m_limb_type, rhs1);
5897 : }
5898 :
5899 81 : if (ifn == IFN_CLZ)
5900 : {
5901 59 : g = gimple_build_cond (NE_EXPR, rhs1,
5902 : build_zero_cst (m_limb_type),
5903 : NULL_TREE, NULL_TREE);
5904 59 : insert_before (g);
5905 59 : edge e1 = split_block (gsi_bb (m_gsi), g);
5906 59 : e1->flags = EDGE_FALSE_VALUE;
5907 59 : edge e2 = make_edge (e1->src, gimple_bb (stmt), EDGE_TRUE_VALUE);
5908 59 : e1->probability = profile_probability::unlikely ();
5909 59 : e2->probability = e1->probability.invert ();
5910 59 : if (i == 0)
5911 26 : set_immediate_dominator (CDI_DOMINATORS, e2->dest, e2->src);
5912 59 : m_gsi = gsi_after_labels (e1->dest);
5913 59 : bqp[i].e = e2;
5914 59 : bqp[i].val = rhs1;
5915 : }
5916 : else
5917 : {
5918 22 : if (i == 0)
5919 : {
5920 9 : first = rhs1;
5921 9 : g = gimple_build_assign (make_ssa_name (m_limb_type),
5922 : PLUS_EXPR, rhs1,
5923 : build_int_cst (m_limb_type, 1));
5924 9 : insert_before (g);
5925 9 : g = gimple_build_cond (GT_EXPR, gimple_assign_lhs (g),
5926 : build_int_cst (m_limb_type, 1),
5927 : NULL_TREE, NULL_TREE);
5928 9 : insert_before (g);
5929 : }
5930 : else
5931 : {
5932 13 : g = gimple_build_assign (make_ssa_name (m_limb_type),
5933 : BIT_XOR_EXPR, rhs1, first);
5934 13 : insert_before (g);
5935 13 : tree stype = signed_type_for (m_limb_type);
5936 13 : g = gimple_build_cond (LT_EXPR,
5937 : add_cast (stype,
5938 : gimple_assign_lhs (g)),
5939 : build_zero_cst (stype),
5940 : NULL_TREE, NULL_TREE);
5941 13 : insert_before (g);
5942 13 : edge e1 = split_block (gsi_bb (m_gsi), g);
5943 13 : e1->flags = EDGE_FALSE_VALUE;
5944 13 : edge e2 = make_edge (e1->src, gimple_bb (stmt),
5945 : EDGE_TRUE_VALUE);
5946 13 : e1->probability = profile_probability::unlikely ();
5947 13 : e2->probability = e1->probability.invert ();
5948 13 : if (i == 1)
5949 9 : set_immediate_dominator (CDI_DOMINATORS, e2->dest,
5950 : e2->src);
5951 13 : m_gsi = gsi_after_labels (e1->dest);
5952 13 : bqp[2 * i].e = e2;
5953 13 : g = gimple_build_cond (NE_EXPR, rhs1, first,
5954 : NULL_TREE, NULL_TREE);
5955 13 : insert_before (g);
5956 : }
5957 22 : edge e1 = split_block (gsi_bb (m_gsi), g);
5958 22 : e1->flags = EDGE_FALSE_VALUE;
5959 22 : edge e2 = make_edge (e1->src, edge_bb, EDGE_TRUE_VALUE);
5960 22 : e1->probability = profile_probability::unlikely ();
5961 22 : e2->probability = e1->probability.invert ();
5962 22 : if (i == 0)
5963 9 : set_immediate_dominator (CDI_DOMINATORS, e2->dest, e2->src);
5964 22 : m_gsi = gsi_after_labels (e1->dest);
5965 22 : bqp[2 * i + 1].e = e2;
5966 22 : bqp[i].val = rhs1;
5967 : }
5968 81 : if (tree_fits_uhwi_p (idx))
5969 126 : bqp[i].addend
5970 63 : = build_int_cst (integer_type_node,
5971 63 : (int) prec
5972 63 : - (((int) (bitint_big_endian
5973 0 : ? nelts - 1 - tree_to_uhwi (idx)
5974 63 : : tree_to_uhwi (idx)) + 1)
5975 63 : * limb_prec) - sub_one);
5976 : else
5977 : {
5978 18 : tree in, out;
5979 18 : in = build_int_cst (integer_type_node, rem - sub_one);
5980 18 : m_first = true;
5981 18 : in = prepare_data_in_out (in, idx, &out);
5982 18 : out = m_data[m_data_cnt + 1];
5983 18 : bqp[i].addend = in;
5984 18 : g = gimple_build_assign (out, PLUS_EXPR, in,
5985 : build_int_cst (integer_type_node,
5986 18 : limb_prec));
5987 18 : insert_before (g);
5988 18 : m_data[m_data_cnt] = out;
5989 : }
5990 :
5991 81 : m_first = false;
5992 81 : if (kind == bitint_prec_huge && i == cnt - 1)
5993 : {
5994 36 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
5995 : bitint_big_endian
5996 18 : ? size_one_node : size_int (-1));
5997 18 : insert_before (g);
5998 18 : g = gimple_build_cond (NE_EXPR, idx,
5999 : bitint_big_endian
6000 0 : ? size_int (nelts - 1) : size_zero_node,
6001 : NULL_TREE, NULL_TREE);
6002 18 : insert_before (g);
6003 18 : edge true_edge, false_edge;
6004 18 : extract_true_false_edges_from_block (gsi_bb (m_gsi),
6005 : &true_edge, &false_edge);
6006 18 : m_gsi = gsi_after_labels (false_edge->dest);
6007 18 : m_bb = NULL;
6008 : }
6009 : }
6010 : }
6011 101 : switch (ifn)
6012 : {
6013 65 : case IFN_CLZ:
6014 65 : case IFN_CTZ:
6015 65 : case IFN_FFS:
6016 65 : gphi *phi1, *phi2, *phi3;
6017 65 : basic_block bb;
6018 65 : bb = gsi_bb (m_gsi);
6019 65 : remove_edge (find_edge (bb, gimple_bb (stmt)));
6020 65 : phi1 = create_phi_node (make_ssa_name (m_limb_type),
6021 : gimple_bb (stmt));
6022 65 : phi2 = create_phi_node (make_ssa_name (integer_type_node),
6023 : gimple_bb (stmt));
6024 293 : for (unsigned i = 0; i < cnt; i++)
6025 : {
6026 163 : add_phi_arg (phi1, bqp[i].val, bqp[i].e, UNKNOWN_LOCATION);
6027 163 : add_phi_arg (phi2, bqp[i].addend, bqp[i].e, UNKNOWN_LOCATION);
6028 : }
6029 65 : if (arg1 == NULL_TREE)
6030 : {
6031 35 : g = gimple_build_builtin_unreachable (m_loc);
6032 35 : insert_before (g);
6033 : }
6034 65 : m_gsi = gsi_for_stmt (stmt);
6035 65 : g = gimple_build_call (fndecl, 1, gimple_phi_result (phi1));
6036 65 : gimple_call_set_lhs (g, make_ssa_name (integer_type_node));
6037 65 : insert_before (g);
6038 65 : if (arg1 == NULL_TREE)
6039 35 : g = gimple_build_assign (lhs, PLUS_EXPR,
6040 : gimple_phi_result (phi2),
6041 : gimple_call_lhs (g));
6042 : else
6043 : {
6044 30 : g = gimple_build_assign (make_ssa_name (integer_type_node),
6045 : PLUS_EXPR, gimple_phi_result (phi2),
6046 : gimple_call_lhs (g));
6047 30 : insert_before (g);
6048 30 : edge e1 = split_block (gimple_bb (stmt), g);
6049 30 : edge e2 = make_edge (bb, e1->dest, EDGE_FALLTHRU);
6050 30 : e2->probability = profile_probability::always ();
6051 30 : set_immediate_dominator (CDI_DOMINATORS, e1->dest,
6052 : get_immediate_dominator (CDI_DOMINATORS,
6053 : e1->src));
6054 30 : phi3 = create_phi_node (make_ssa_name (integer_type_node), e1->dest);
6055 30 : add_phi_arg (phi3, gimple_assign_lhs (g), e1, UNKNOWN_LOCATION);
6056 30 : add_phi_arg (phi3, arg1, e2, UNKNOWN_LOCATION);
6057 30 : m_gsi = gsi_for_stmt (stmt);
6058 30 : g = gimple_build_assign (lhs, gimple_phi_result (phi3));
6059 : }
6060 65 : gsi_replace (&m_gsi, g, true);
6061 65 : break;
6062 9 : case IFN_CLRSB:
6063 9 : bb = gsi_bb (m_gsi);
6064 9 : remove_edge (find_edge (bb, edge_bb));
6065 9 : edge e;
6066 9 : e = make_edge (bb, gimple_bb (stmt), EDGE_FALLTHRU);
6067 9 : e->probability = profile_probability::always ();
6068 9 : set_immediate_dominator (CDI_DOMINATORS, gimple_bb (stmt),
6069 : get_immediate_dominator (CDI_DOMINATORS,
6070 : edge_bb));
6071 9 : phi1 = create_phi_node (make_ssa_name (m_limb_type),
6072 : edge_bb);
6073 9 : phi2 = create_phi_node (make_ssa_name (integer_type_node),
6074 : edge_bb);
6075 9 : phi3 = create_phi_node (make_ssa_name (integer_type_node),
6076 : gimple_bb (stmt));
6077 40 : for (unsigned i = 0; i < cnt; i++)
6078 : {
6079 22 : add_phi_arg (phi1, bqp[i].val, bqp[2 * i + 1].e, UNKNOWN_LOCATION);
6080 22 : add_phi_arg (phi2, bqp[i].addend, bqp[2 * i + 1].e,
6081 : UNKNOWN_LOCATION);
6082 22 : tree a = bqp[i].addend;
6083 22 : if (i && kind == bitint_prec_large)
6084 8 : a = int_const_binop (PLUS_EXPR, a, integer_minus_one_node);
6085 22 : if (i)
6086 13 : add_phi_arg (phi3, a, bqp[2 * i].e, UNKNOWN_LOCATION);
6087 : }
6088 9 : add_phi_arg (phi3, build_int_cst (integer_type_node, prec - 1), e,
6089 : UNKNOWN_LOCATION);
6090 9 : m_gsi = gsi_after_labels (edge_bb);
6091 9 : g = gimple_build_call (fndecl, 1,
6092 : add_cast (signed_type_for (m_limb_type),
6093 : gimple_phi_result (phi1)));
6094 9 : gimple_call_set_lhs (g, make_ssa_name (integer_type_node));
6095 9 : insert_before (g);
6096 9 : g = gimple_build_assign (make_ssa_name (integer_type_node),
6097 : PLUS_EXPR, gimple_call_lhs (g),
6098 : gimple_phi_result (phi2));
6099 9 : insert_before (g);
6100 9 : if (kind != bitint_prec_large)
6101 : {
6102 5 : g = gimple_build_assign (make_ssa_name (integer_type_node),
6103 : PLUS_EXPR, gimple_assign_lhs (g),
6104 : integer_one_node);
6105 5 : insert_before (g);
6106 : }
6107 9 : add_phi_arg (phi3, gimple_assign_lhs (g),
6108 : find_edge (edge_bb, gimple_bb (stmt)), UNKNOWN_LOCATION);
6109 9 : m_gsi = gsi_for_stmt (stmt);
6110 9 : g = gimple_build_assign (lhs, gimple_phi_result (phi3));
6111 9 : gsi_replace (&m_gsi, g, true);
6112 9 : break;
6113 16 : case IFN_PARITY:
6114 16 : g = gimple_build_call (fndecl, 1, res);
6115 16 : gimple_call_set_lhs (g, lhs);
6116 16 : gsi_replace (&m_gsi, g, true);
6117 16 : break;
6118 11 : case IFN_POPCOUNT:
6119 11 : g = gimple_build_assign (lhs, res);
6120 11 : gsi_replace (&m_gsi, g, true);
6121 11 : break;
6122 0 : default:
6123 0 : gcc_unreachable ();
6124 : }
6125 : }
6126 :
6127 : /* Lower a .{BSWAP,BITREVERSE} call with one large/huge _BitInt argument. */
6128 :
6129 : void
6130 15 : bitint_large_huge::lower_bswap_bitreverse (tree obj, gimple *stmt)
6131 : {
6132 15 : tree arg = gimple_call_arg (stmt, 0);
6133 15 : tree lhs = gimple_call_lhs (stmt);
6134 15 : gimple *g;
6135 :
6136 15 : if (!lhs)
6137 : {
6138 0 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
6139 0 : gsi_remove (&gsi, true);
6140 0 : return;
6141 : }
6142 15 : tree type = TREE_TYPE (arg);
6143 15 : gcc_assert (BITINT_TYPE_P (type));
6144 15 : bitint_prec_kind kind = bitint_precision_kind (type);
6145 15 : gcc_assert (kind >= bitint_prec_large);
6146 15 : if (!obj)
6147 : {
6148 2 : int part = var_to_partition (m_map, lhs);
6149 2 : gcc_assert (m_vars[part] != NULL_TREE);
6150 : obj = m_vars[part];
6151 : }
6152 15 : enum internal_fn ifn = gimple_call_internal_fn (stmt);
6153 15 : enum built_in_function bcode = END_BUILTINS;
6154 15 : switch (limb_prec)
6155 : {
6156 0 : case 16:
6157 0 : bcode = ifn == IFN_BSWAP ? BUILT_IN_BSWAP16 : BUILT_IN_BITREVERSE16;
6158 : break;
6159 0 : case 32:
6160 0 : bcode = ifn == IFN_BSWAP ? BUILT_IN_BSWAP32 : BUILT_IN_BITREVERSE32;
6161 : break;
6162 15 : case 64:
6163 15 : bcode = ifn == IFN_BSWAP ? BUILT_IN_BSWAP64 : BUILT_IN_BITREVERSE64;
6164 : break;
6165 0 : case 128:
6166 0 : bcode = ifn == IFN_BSWAP ? BUILT_IN_BSWAP128 : BUILT_IN_BITREVERSE128;
6167 : break;
6168 0 : default:
6169 0 : gcc_unreachable ();
6170 : }
6171 15 : tree fndecl = builtin_decl_explicit (bcode);
6172 15 : unsigned prec = TYPE_PRECISION (type);
6173 30 : tree p = build_int_cst (sizetype,
6174 15 : prec / limb_prec - (prec % limb_prec == 0));
6175 : /* For IFN .BSWAP or .BITREVERSE and
6176 : FN corresponding __builtin_bswapN or __builtin_bitreverseN where N
6177 : is limb_prec, lower
6178 : dst = IFN (src);
6179 : as
6180 : size_t p = prec / limb_prec - (prec % limb_prec == 0);
6181 : if constexpr ((prec % limb_prec) == 0)
6182 : {
6183 : for (idx = 0; idx <= p; ++idx)
6184 : dst[p - idx] = FN (src[idx]);
6185 : }
6186 : else
6187 : {
6188 : unsigned n1 = prec % limb_prec;
6189 : unsigned n2 = limb_prec - n1;
6190 : dst[p] = FN (src[0]) >> n2;
6191 : for (idx = p - 1; (ssize_t) idx >= 0; --idx)
6192 : dst[idx] = FN ((src[p - idx] << n2) | (src[p - idx - 1] >> n1));
6193 : } */
6194 15 : if (prec % limb_prec == 0)
6195 : {
6196 3 : tree idx_next;
6197 3 : tree idx = create_loop (size_zero_node, &idx_next);
6198 3 : tree pmidx = make_ssa_name (sizetype);
6199 3 : g = gimple_build_assign (pmidx, MINUS_EXPR, p, idx);
6200 3 : insert_before (g);
6201 3 : m_data_cnt = 0;
6202 3 : tree t = handle_operand (arg, idx);
6203 3 : m_first = false;
6204 3 : g = gimple_build_call (fndecl, 1, t);
6205 3 : t = make_ssa_name (m_limb_type);
6206 3 : gimple_call_set_lhs (g, t);
6207 3 : insert_before (g);
6208 3 : tree l = limb_access (TREE_TYPE (lhs), obj, pmidx, true);
6209 3 : g = gimple_build_assign (l, t);
6210 3 : insert_before (g);
6211 3 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx, size_one_node);
6212 3 : insert_before (g);
6213 3 : g = gimple_build_cond (LE_EXPR, idx_next, p, NULL_TREE, NULL_TREE);
6214 3 : insert_before (g);
6215 : }
6216 : else
6217 : {
6218 12 : tree n1 = build_int_cst (unsigned_type_node, prec % limb_prec);
6219 24 : tree n2 = build_int_cst (unsigned_type_node,
6220 12 : limb_prec - (prec % limb_prec));
6221 12 : m_data_cnt = 0;
6222 12 : tree t = handle_operand (arg, bitint_big_endian ? p : size_zero_node);
6223 12 : m_first = false;
6224 : /* Nothing is needed to bswap 8 bits or bitreverse 1 bit, so just
6225 : mask off higher bits in that case. */
6226 19 : if ((prec % limb_prec) != (ifn == IFN_BSWAP ? 8 : 1))
6227 : {
6228 8 : g = gimple_build_call (fndecl, 1, t);
6229 8 : t = make_ssa_name (m_limb_type);
6230 8 : gimple_call_set_lhs (g, t);
6231 8 : insert_before (g);
6232 8 : g = gimple_build_assign (make_ssa_name (m_limb_type), RSHIFT_EXPR,
6233 : t, n2);
6234 : }
6235 : else
6236 4 : g = gimple_build_assign (make_ssa_name (m_limb_type), BIT_AND_EXPR,
6237 : t, build_int_cst (m_limb_type,
6238 : ifn == IFN_BSWAP
6239 6 : ? 0xff : 1));
6240 12 : insert_before (g);
6241 12 : t = gimple_assign_lhs (g);
6242 12 : tree l = limb_access (TREE_TYPE (lhs), obj,
6243 : bitint_big_endian ? size_zero_node : p, true);
6244 12 : g = gimple_build_assign (l, t);
6245 12 : insert_before (g);
6246 12 : tree pm1 = build_int_cst (sizetype, prec / limb_prec - 1);
6247 12 : tree idx_next;
6248 12 : tree idx = create_loop (pm1, &idx_next);
6249 12 : tree pmidx = make_ssa_name (sizetype);
6250 12 : g = gimple_build_assign (pmidx, MINUS_EXPR, p, idx);
6251 12 : insert_before (g);
6252 12 : m_data_cnt = 0;
6253 24 : t = handle_operand (arg, bitint_big_endian ? idx : pmidx);
6254 12 : m_data_cnt = 0;
6255 12 : g = gimple_build_assign (make_ssa_name (m_limb_type), LSHIFT_EXPR,
6256 : t, n2);
6257 12 : insert_before (g);
6258 12 : t = gimple_assign_lhs (g);
6259 12 : tree t2 = make_ssa_name (sizetype);
6260 12 : if (bitint_big_endian)
6261 0 : g = gimple_build_assign (t2, PLUS_EXPR, idx, size_one_node);
6262 : else
6263 12 : g = gimple_build_assign (t2, PLUS_EXPR, pmidx, size_int (-1));
6264 12 : insert_before (g);
6265 12 : t2 = handle_operand (arg, t2);
6266 12 : g = gimple_build_assign (make_ssa_name (m_limb_type), RSHIFT_EXPR,
6267 : t2, n1);
6268 12 : insert_before (g);
6269 12 : t2 = gimple_assign_lhs (g);
6270 12 : g = gimple_build_assign (make_ssa_name (m_limb_type), BIT_IOR_EXPR,
6271 : t, t2);
6272 12 : insert_before (g);
6273 12 : t = gimple_assign_lhs (g);
6274 12 : g = gimple_build_call (fndecl, 1, t);
6275 12 : t = make_ssa_name (m_limb_type);
6276 12 : gimple_call_set_lhs (g, t);
6277 12 : insert_before (g);
6278 24 : l = limb_access (TREE_TYPE (lhs), obj,
6279 : bitint_big_endian ? pmidx : idx, true);
6280 12 : g = gimple_build_assign (l, t);
6281 12 : insert_before (g);
6282 12 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx, size_int (-1));
6283 12 : insert_before (g);
6284 12 : g = gimple_build_cond (NE_EXPR, idx, size_zero_node, NULL_TREE,
6285 : NULL_TREE);
6286 12 : insert_before (g);
6287 : }
6288 15 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
6289 15 : if (bitint_extended == bitint_ext_full
6290 0 : && abi_limb_prec > limb_prec
6291 0 : && (CEIL (prec, abi_limb_prec) * abi_limb_prec
6292 0 : > CEIL (prec, limb_prec) * limb_prec))
6293 : {
6294 0 : m_gsi = gsi;
6295 0 : tree p2 = build_int_cst (sizetype,
6296 : CEIL (prec, abi_limb_prec)
6297 0 : * abi_limb_prec / limb_prec - 1);
6298 0 : tree l = limb_access (TREE_TYPE (lhs), obj, p2, true);
6299 0 : g = gimple_build_assign (l, build_zero_cst (m_limb_type));
6300 0 : insert_before (g);
6301 : }
6302 : }
6303 :
6304 : /* Lower a call statement with one or more large/huge _BitInt
6305 : arguments or large/huge _BitInt return value. */
6306 :
6307 : void
6308 8857 : bitint_large_huge::lower_call (tree obj, gimple *stmt)
6309 : {
6310 8857 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
6311 8857 : unsigned int nargs = gimple_call_num_args (stmt);
6312 8857 : if (gimple_call_internal_p (stmt))
6313 4328 : switch (gimple_call_internal_fn (stmt))
6314 : {
6315 2733 : case IFN_ADD_OVERFLOW:
6316 2733 : case IFN_SUB_OVERFLOW:
6317 2733 : case IFN_UBSAN_CHECK_ADD:
6318 2733 : case IFN_UBSAN_CHECK_SUB:
6319 2733 : lower_addsub_overflow (obj, stmt);
6320 7061 : return;
6321 1479 : case IFN_MUL_OVERFLOW:
6322 1479 : case IFN_UBSAN_CHECK_MUL:
6323 1479 : lower_mul_overflow (obj, stmt);
6324 1479 : return;
6325 101 : case IFN_CLZ:
6326 101 : case IFN_CTZ:
6327 101 : case IFN_CLRSB:
6328 101 : case IFN_FFS:
6329 101 : case IFN_PARITY:
6330 101 : case IFN_POPCOUNT:
6331 101 : lower_bit_query (stmt);
6332 101 : return;
6333 15 : case IFN_BSWAP:
6334 15 : case IFN_BITREVERSE:
6335 15 : lower_bswap_bitreverse (obj, stmt);
6336 15 : return;
6337 : default:
6338 : break;
6339 : }
6340 4529 : bool returns_twice = (gimple_call_flags (stmt) & ECF_RETURNS_TWICE) != 0;
6341 10202 : for (unsigned int i = 0; i < nargs; ++i)
6342 : {
6343 5673 : tree arg = gimple_call_arg (stmt, i);
6344 9003 : if (TREE_CODE (arg) != SSA_NAME
6345 2440 : || !BITINT_TYPE_P (TREE_TYPE (arg))
6346 8051 : || bitint_precision_kind (TREE_TYPE (arg)) <= bitint_prec_middle)
6347 3330 : continue;
6348 2343 : if (SSA_NAME_IS_DEFAULT_DEF (arg)
6349 2343 : && (!SSA_NAME_VAR (arg) || VAR_P (SSA_NAME_VAR (arg))))
6350 : {
6351 1 : tree var = create_tmp_reg (TREE_TYPE (arg));
6352 1 : arg = get_or_create_ssa_default_def (cfun, var);
6353 : }
6354 : else
6355 : {
6356 2342 : int p = var_to_partition (m_map, arg);
6357 2342 : tree v = m_vars[p];
6358 2342 : gcc_assert (v != NULL_TREE);
6359 2342 : if (!types_compatible_p (TREE_TYPE (arg), TREE_TYPE (v)))
6360 2322 : v = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (arg), v);
6361 2342 : arg = make_ssa_name (TREE_TYPE (arg));
6362 2342 : gimple *g = gimple_build_assign (arg, v);
6363 2342 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
6364 2342 : if (returns_twice && bb_has_abnormal_pred (gimple_bb (stmt)))
6365 : {
6366 11 : m_returns_twice_calls.safe_push (stmt);
6367 11 : returns_twice = false;
6368 : }
6369 : }
6370 2343 : gimple_call_set_arg (stmt, i, arg);
6371 2343 : if (m_preserved == NULL)
6372 412 : m_preserved = BITMAP_ALLOC (NULL);
6373 2343 : bitmap_set_bit (m_preserved, SSA_NAME_VERSION (arg));
6374 : }
6375 4529 : tree lhs = gimple_call_lhs (stmt);
6376 4529 : if (lhs
6377 4398 : && TREE_CODE (lhs) == SSA_NAME
6378 4398 : && BITINT_TYPE_P (TREE_TYPE (lhs))
6379 8849 : && bitint_precision_kind (TREE_TYPE (lhs)) >= bitint_prec_large)
6380 : {
6381 4294 : int p = var_to_partition (m_map, lhs);
6382 4294 : tree v = m_vars[p];
6383 4294 : gcc_assert (v != NULL_TREE);
6384 4294 : if (!types_compatible_p (TREE_TYPE (lhs), TREE_TYPE (v)))
6385 4294 : v = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (lhs), v);
6386 4294 : gimple_call_set_lhs (stmt, v);
6387 4294 : SSA_NAME_DEF_STMT (lhs) = gimple_build_nop ();
6388 : }
6389 4529 : update_stmt (stmt);
6390 : }
6391 :
6392 : /* Lower __asm STMT which involves large/huge _BitInt values. */
6393 :
6394 : void
6395 3 : bitint_large_huge::lower_asm (gimple *stmt)
6396 : {
6397 3 : gasm *g = as_a <gasm *> (stmt);
6398 3 : unsigned noutputs = gimple_asm_noutputs (g);
6399 3 : unsigned ninputs = gimple_asm_ninputs (g);
6400 :
6401 5 : for (unsigned i = 0; i < noutputs; ++i)
6402 : {
6403 2 : tree t = gimple_asm_output_op (g, i);
6404 2 : tree s = TREE_VALUE (t);
6405 2 : if (TREE_CODE (s) == SSA_NAME
6406 1 : && BITINT_TYPE_P (TREE_TYPE (s))
6407 3 : && bitint_precision_kind (TREE_TYPE (s)) >= bitint_prec_large)
6408 : {
6409 1 : int part = var_to_partition (m_map, s);
6410 1 : gcc_assert (m_vars[part] != NULL_TREE);
6411 1 : TREE_VALUE (t) = m_vars[part];
6412 : }
6413 : }
6414 8 : for (unsigned i = 0; i < ninputs; ++i)
6415 : {
6416 5 : tree t = gimple_asm_input_op (g, i);
6417 5 : tree s = TREE_VALUE (t);
6418 5 : if (TREE_CODE (s) == SSA_NAME
6419 4 : && BITINT_TYPE_P (TREE_TYPE (s))
6420 9 : && bitint_precision_kind (TREE_TYPE (s)) >= bitint_prec_large)
6421 : {
6422 4 : if (SSA_NAME_IS_DEFAULT_DEF (s)
6423 4 : && (!SSA_NAME_VAR (s) || VAR_P (SSA_NAME_VAR (s))))
6424 : {
6425 1 : TREE_VALUE (t) = create_tmp_var (TREE_TYPE (s), "bitint");
6426 1 : mark_addressable (TREE_VALUE (t));
6427 : }
6428 : else
6429 : {
6430 3 : int part = var_to_partition (m_map, s);
6431 3 : gcc_assert (m_vars[part] != NULL_TREE);
6432 3 : TREE_VALUE (t) = m_vars[part];
6433 : }
6434 : }
6435 : }
6436 3 : update_stmt (stmt);
6437 3 : }
6438 :
6439 : /* Lower statement STMT which involves large/huge _BitInt values
6440 : into code accessing individual limbs. */
6441 :
6442 : void
6443 42903 : bitint_large_huge::lower_stmt (gimple *stmt)
6444 : {
6445 42903 : m_first = true;
6446 42903 : m_lhs = NULL_TREE;
6447 42903 : m_data.truncate (0);
6448 42903 : m_data_cnt = 0;
6449 42903 : m_gsi = gsi_for_stmt (stmt);
6450 42903 : m_after_stmt = NULL;
6451 42903 : m_bb = NULL;
6452 42903 : m_init_gsi = m_gsi;
6453 42903 : gsi_prev (&m_init_gsi);
6454 42903 : m_preheader_bb = NULL;
6455 42903 : m_upwards_2limb = 0;
6456 42903 : m_upwards = false;
6457 42903 : m_var_msb = false;
6458 42903 : m_cast_conditional = false;
6459 42903 : m_bitfld_load = 0;
6460 42903 : m_loc = gimple_location (stmt);
6461 42903 : if (is_gimple_call (stmt))
6462 : {
6463 7180 : lower_call (NULL_TREE, stmt);
6464 7180 : return;
6465 : }
6466 35723 : if (gimple_code (stmt) == GIMPLE_ASM)
6467 : {
6468 3 : lower_asm (stmt);
6469 3 : return;
6470 : }
6471 35720 : tree lhs = NULL_TREE, cmp_op1 = NULL_TREE, cmp_op2 = NULL_TREE;
6472 35720 : tree_code cmp_code = comparison_op (stmt, &cmp_op1, &cmp_op2);
6473 35720 : bool eq_p = (cmp_code == EQ_EXPR || cmp_code == NE_EXPR);
6474 35720 : bool mergeable_cast_p = false;
6475 35720 : bool final_cast_p = false;
6476 35720 : if (gimple_assign_cast_p (stmt))
6477 : {
6478 5449 : lhs = gimple_assign_lhs (stmt);
6479 5449 : tree rhs1 = gimple_assign_rhs1 (stmt);
6480 5449 : if (TREE_CODE (rhs1) == VIEW_CONVERT_EXPR)
6481 46 : rhs1 = TREE_OPERAND (rhs1, 0);
6482 9670 : if (BITINT_TYPE_P (TREE_TYPE (lhs))
6483 1228 : && bitint_precision_kind (TREE_TYPE (lhs)) >= bitint_prec_large
6484 6641 : && INTEGRAL_TYPE_P (TREE_TYPE (rhs1)))
6485 : mergeable_cast_p = true;
6486 4547 : else if (BITINT_TYPE_P (TREE_TYPE (rhs1))
6487 4257 : && bitint_precision_kind (TREE_TYPE (rhs1)) >= bitint_prec_large
6488 8659 : && (INTEGRAL_TYPE_P (TREE_TYPE (lhs))
6489 36 : || POINTER_TYPE_P (TREE_TYPE (lhs))
6490 35 : || gimple_assign_rhs_code (stmt) == VIEW_CONVERT_EXPR))
6491 : {
6492 4257 : final_cast_p = true;
6493 4257 : if (((TREE_CODE (TREE_TYPE (lhs)) == INTEGER_TYPE
6494 572 : && TYPE_PRECISION (TREE_TYPE (lhs)) > MAX_FIXED_MODE_SIZE)
6495 4257 : || (!INTEGRAL_TYPE_P (TREE_TYPE (lhs))
6496 36 : && !POINTER_TYPE_P (TREE_TYPE (lhs))))
6497 4292 : && gimple_assign_rhs_code (stmt) == VIEW_CONVERT_EXPR)
6498 : {
6499 : /* Handle VIEW_CONVERT_EXPRs to not generally supported
6500 : huge INTEGER_TYPEs like uint256_t or uint512_t. These
6501 : are usually emitted from memcpy folding and backends
6502 : support moves with them but that is usually it.
6503 : Similarly handle VCEs to vector/complex types etc. */
6504 35 : gcc_assert (TREE_CODE (rhs1) == SSA_NAME);
6505 35 : if (SSA_NAME_IS_DEFAULT_DEF (rhs1)
6506 35 : && (!SSA_NAME_VAR (rhs1) || VAR_P (SSA_NAME_VAR (rhs1))))
6507 : {
6508 0 : tree var = create_tmp_reg (TREE_TYPE (lhs));
6509 0 : rhs1 = get_or_create_ssa_default_def (cfun, var);
6510 0 : gimple_assign_set_rhs1 (stmt, rhs1);
6511 0 : gimple_assign_set_rhs_code (stmt, SSA_NAME);
6512 : }
6513 35 : else if (m_names == NULL
6514 35 : || !bitmap_bit_p (m_names, SSA_NAME_VERSION (rhs1)))
6515 : {
6516 0 : gimple *g = SSA_NAME_DEF_STMT (rhs1);
6517 0 : gcc_assert (gimple_assign_load_p (g));
6518 0 : tree mem = gimple_assign_rhs1 (g);
6519 0 : tree ltype = TREE_TYPE (lhs);
6520 0 : addr_space_t as = TYPE_ADDR_SPACE (TREE_TYPE (mem));
6521 0 : if (as != TYPE_ADDR_SPACE (ltype))
6522 0 : ltype
6523 0 : = build_qualified_type (ltype,
6524 0 : TYPE_QUALS (ltype)
6525 0 : | ENCODE_QUAL_ADDR_SPACE (as));
6526 0 : rhs1 = build1 (VIEW_CONVERT_EXPR, ltype, unshare_expr (mem));
6527 0 : gimple_assign_set_rhs1 (stmt, rhs1);
6528 : }
6529 : else
6530 : {
6531 35 : int part = var_to_partition (m_map, rhs1);
6532 35 : gcc_assert (m_vars[part] != NULL_TREE);
6533 35 : rhs1 = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (lhs),
6534 : m_vars[part]);
6535 35 : gimple_assign_set_rhs1 (stmt, rhs1);
6536 : }
6537 35 : update_stmt (stmt);
6538 35 : return;
6539 : }
6540 4222 : if (TREE_CODE (rhs1) == SSA_NAME
6541 4222 : && (m_names == NULL
6542 4185 : || !bitmap_bit_p (m_names, SSA_NAME_VERSION (rhs1))))
6543 : {
6544 1723 : gimple *g = SSA_NAME_DEF_STMT (rhs1);
6545 1723 : if (is_gimple_assign (g)
6546 1723 : && gimple_assign_rhs_code (g) == IMAGPART_EXPR)
6547 : {
6548 1640 : tree rhs2 = TREE_OPERAND (gimple_assign_rhs1 (g), 0);
6549 1640 : if (TREE_CODE (rhs2) == SSA_NAME
6550 1640 : && (m_names == NULL
6551 1603 : || !bitmap_bit_p (m_names, SSA_NAME_VERSION (rhs2))))
6552 : {
6553 1640 : g = SSA_NAME_DEF_STMT (rhs2);
6554 1640 : int ovf = optimizable_arith_overflow (g);
6555 1640 : if (ovf == 2)
6556 : /* If .{ADD,SUB,MUL}_OVERFLOW has both REALPART_EXPR
6557 : and IMAGPART_EXPR uses, where the latter is cast to
6558 : non-_BitInt, it will be optimized when handling
6559 : the REALPART_EXPR. */
6560 : return;
6561 91 : if (ovf == 1)
6562 : {
6563 91 : lower_call (NULL_TREE, g);
6564 91 : return;
6565 : }
6566 : }
6567 : }
6568 : }
6569 : }
6570 145 : else if (BITINT_TYPE_P (TREE_TYPE (lhs))
6571 145 : && bitint_precision_kind (TREE_TYPE (lhs)) >= bitint_prec_large
6572 145 : && !INTEGRAL_TYPE_P (TREE_TYPE (rhs1))
6573 145 : && !POINTER_TYPE_P (TREE_TYPE (rhs1))
6574 290 : && gimple_assign_rhs_code (stmt) == VIEW_CONVERT_EXPR)
6575 : {
6576 10 : int part = var_to_partition (m_map, lhs);
6577 10 : gcc_assert (m_vars[part] != NULL_TREE);
6578 10 : lhs = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (rhs1), m_vars[part]);
6579 10 : insert_before (gimple_build_assign (lhs, rhs1));
6580 10 : return;
6581 : }
6582 : }
6583 34035 : if (gimple_store_p (stmt))
6584 : {
6585 9009 : tree rhs1 = gimple_assign_rhs1 (stmt);
6586 9009 : if (TREE_CODE (rhs1) == SSA_NAME
6587 9009 : && (m_names == NULL
6588 7949 : || !bitmap_bit_p (m_names, SSA_NAME_VERSION (rhs1))))
6589 : {
6590 1641 : gimple *g = SSA_NAME_DEF_STMT (rhs1);
6591 1641 : m_loc = gimple_location (g);
6592 1641 : lhs = gimple_assign_lhs (stmt);
6593 1641 : if (is_gimple_assign (g) && !mergeable_op (g))
6594 645 : switch (gimple_assign_rhs_code (g))
6595 : {
6596 119 : case LSHIFT_EXPR:
6597 119 : case RSHIFT_EXPR:
6598 119 : lower_shift_stmt (lhs, g);
6599 478 : handled:
6600 478 : m_gsi = gsi_for_stmt (stmt);
6601 478 : unlink_stmt_vdef (stmt);
6602 956 : release_ssa_name (gimple_vdef (stmt));
6603 478 : gsi_remove (&m_gsi, true);
6604 478 : return;
6605 205 : case MULT_EXPR:
6606 205 : case TRUNC_DIV_EXPR:
6607 205 : case EXACT_DIV_EXPR:
6608 205 : case TRUNC_MOD_EXPR:
6609 205 : lower_muldiv_stmt (lhs, g);
6610 205 : goto handled;
6611 44 : case FIX_TRUNC_EXPR:
6612 44 : lower_float_conv_stmt (lhs, g);
6613 44 : goto handled;
6614 73 : case REALPART_EXPR:
6615 73 : case IMAGPART_EXPR:
6616 73 : lower_cplxpart_stmt (lhs, g);
6617 73 : goto handled;
6618 8 : case VIEW_CONVERT_EXPR:
6619 8 : {
6620 8 : tree rhs1 = gimple_assign_rhs1 (g);
6621 8 : rhs1 = TREE_OPERAND (rhs1, 0);
6622 8 : if (!INTEGRAL_TYPE_P (TREE_TYPE (rhs1))
6623 7 : && !POINTER_TYPE_P (TREE_TYPE (rhs1)))
6624 : {
6625 7 : tree ltype = TREE_TYPE (rhs1);
6626 7 : addr_space_t as = TYPE_ADDR_SPACE (TREE_TYPE (lhs));
6627 7 : ltype
6628 14 : = build_qualified_type (ltype,
6629 7 : TYPE_QUALS (TREE_TYPE (lhs))
6630 7 : | ENCODE_QUAL_ADDR_SPACE (as));
6631 7 : lhs = build1 (VIEW_CONVERT_EXPR, ltype, lhs);
6632 7 : gimple_assign_set_lhs (stmt, lhs);
6633 7 : gimple_assign_set_rhs1 (stmt, rhs1);
6634 7 : gimple_assign_set_rhs_code (stmt, TREE_CODE (rhs1));
6635 7 : update_stmt (stmt);
6636 7 : return;
6637 : }
6638 : }
6639 : break;
6640 : default:
6641 : break;
6642 : }
6643 996 : else if (optimizable_arith_overflow (g) == 3)
6644 : {
6645 24 : lower_call (lhs, g);
6646 24 : goto handled;
6647 : }
6648 972 : else if (is_gimple_call (g) && gimple_call_internal_p (g))
6649 13 : switch (gimple_call_internal_fn (g))
6650 : {
6651 13 : case IFN_BSWAP:
6652 13 : case IFN_BITREVERSE:
6653 13 : lower_call (lhs, g);
6654 13 : goto handled;
6655 : default:
6656 : break;
6657 : }
6658 :
6659 1156 : m_loc = gimple_location (stmt);
6660 : }
6661 : }
6662 33550 : if (mergeable_op (stmt)
6663 22114 : || gimple_store_p (stmt)
6664 22114 : || gimple_assign_load_p (stmt)
6665 : || eq_p
6666 17300 : || mergeable_cast_p
6667 43717 : || (is_gimple_assign (stmt)
6668 9779 : && gimple_assign_rhs_code (stmt) == PAREN_EXPR))
6669 : {
6670 23385 : lhs = lower_mergeable_stmt (stmt, cmp_code, cmp_op1, cmp_op2);
6671 23385 : if (!eq_p)
6672 : return;
6673 : }
6674 10165 : else if (cmp_code != ERROR_MARK)
6675 722 : lhs = lower_comparison_stmt (stmt, cmp_code, cmp_op1, cmp_op2);
6676 16717 : if (cmp_code != ERROR_MARK)
6677 : {
6678 7274 : if (gimple_code (stmt) == GIMPLE_COND)
6679 : {
6680 6500 : gcond *cstmt = as_a <gcond *> (stmt);
6681 6500 : gimple_cond_set_lhs (cstmt, lhs);
6682 6500 : gimple_cond_set_rhs (cstmt, boolean_false_node);
6683 6500 : gimple_cond_set_code (cstmt, cmp_code);
6684 6500 : update_stmt (stmt);
6685 6500 : return;
6686 : }
6687 774 : if (gimple_assign_rhs_code (stmt) == COND_EXPR)
6688 : {
6689 0 : tree cond = build2 (cmp_code, boolean_type_node, lhs,
6690 : boolean_false_node);
6691 0 : gimple_assign_set_rhs1 (stmt, cond);
6692 0 : lhs = gimple_assign_lhs (stmt);
6693 0 : gcc_assert (!BITINT_TYPE_P (TREE_TYPE (lhs))
6694 : || (bitint_precision_kind (TREE_TYPE (lhs))
6695 : <= bitint_prec_middle));
6696 0 : update_stmt (stmt);
6697 0 : return;
6698 : }
6699 774 : gimple_assign_set_rhs1 (stmt, lhs);
6700 774 : gimple_assign_set_rhs2 (stmt, boolean_false_node);
6701 774 : gimple_assign_set_rhs_code (stmt, cmp_code);
6702 774 : update_stmt (stmt);
6703 774 : return;
6704 : }
6705 9443 : if (final_cast_p)
6706 : {
6707 2582 : tree lhs_type = TREE_TYPE (lhs);
6708 : /* Add support for 3 or more limbs filled in from normal integral
6709 : type if this assert fails. If no target chooses limb mode smaller
6710 : than half of largest supported normal integral type, this will not
6711 : be needed. */
6712 2582 : gcc_assert (TYPE_PRECISION (lhs_type) <= 2 * limb_prec);
6713 2582 : gimple *g;
6714 2546 : if ((BITINT_TYPE_P (lhs_type)
6715 36 : && bitint_precision_kind (lhs_type) == bitint_prec_middle)
6716 5149 : || POINTER_TYPE_P (lhs_type))
6717 16 : lhs_type = build_nonstandard_integer_type (TYPE_PRECISION (lhs_type),
6718 16 : TYPE_UNSIGNED (lhs_type));
6719 2582 : m_data_cnt = 0;
6720 2582 : tree rhs1 = gimple_assign_rhs1 (stmt);
6721 2582 : unsigned int prec = TYPE_PRECISION (TREE_TYPE (rhs1));
6722 2582 : unsigned int cnt = CEIL (prec, limb_prec);
6723 2582 : tree r1 = handle_operand (rhs1, size_int (bitint_big_endian
6724 : ? cnt - 1 : 0));
6725 2582 : if (!useless_type_conversion_p (lhs_type, TREE_TYPE (r1)))
6726 2473 : r1 = add_cast (lhs_type, r1);
6727 2582 : if (TYPE_PRECISION (lhs_type) > limb_prec)
6728 : {
6729 70 : m_data_cnt = 0;
6730 70 : m_first = false;
6731 70 : tree r2 = handle_operand (rhs1, size_int (bitint_big_endian
6732 : ? cnt - 2 : 1));
6733 70 : r2 = add_cast (lhs_type, r2);
6734 70 : g = gimple_build_assign (make_ssa_name (lhs_type), LSHIFT_EXPR, r2,
6735 : build_int_cst (unsigned_type_node,
6736 70 : limb_prec));
6737 70 : insert_before (g);
6738 70 : g = gimple_build_assign (make_ssa_name (lhs_type), BIT_IOR_EXPR, r1,
6739 : gimple_assign_lhs (g));
6740 70 : insert_before (g);
6741 70 : r1 = gimple_assign_lhs (g);
6742 : }
6743 2582 : if (lhs_type != TREE_TYPE (lhs))
6744 16 : g = gimple_build_assign (lhs, NOP_EXPR, r1);
6745 : else
6746 2566 : g = gimple_build_assign (lhs, r1);
6747 2582 : gsi_replace (&m_gsi, g, true);
6748 2582 : return;
6749 : }
6750 6861 : if (is_gimple_assign (stmt))
6751 6861 : switch (gimple_assign_rhs_code (stmt))
6752 : {
6753 465 : case LSHIFT_EXPR:
6754 465 : case RSHIFT_EXPR:
6755 465 : lower_shift_stmt (NULL_TREE, stmt);
6756 465 : return;
6757 166 : case MULT_EXPR:
6758 166 : case TRUNC_DIV_EXPR:
6759 166 : case EXACT_DIV_EXPR:
6760 166 : case TRUNC_MOD_EXPR:
6761 166 : lower_muldiv_stmt (NULL_TREE, stmt);
6762 166 : return;
6763 283 : case FIX_TRUNC_EXPR:
6764 283 : case FLOAT_EXPR:
6765 283 : lower_float_conv_stmt (NULL_TREE, stmt);
6766 283 : return;
6767 5929 : case REALPART_EXPR:
6768 5929 : case IMAGPART_EXPR:
6769 5929 : lower_cplxpart_stmt (NULL_TREE, stmt);
6770 5929 : return;
6771 18 : case COMPLEX_EXPR:
6772 18 : lower_complexexpr_stmt (stmt);
6773 18 : return;
6774 : default:
6775 : break;
6776 : }
6777 0 : gcc_unreachable ();
6778 : }
6779 :
6780 : /* Helper for walk_non_aliased_vuses. Determine if we arrived at
6781 : the desired memory state. */
6782 :
6783 : void *
6784 2190 : vuse_eq (ao_ref *, tree vuse1, void *data)
6785 : {
6786 2190 : tree vuse2 = (tree) data;
6787 2190 : if (vuse1 == vuse2)
6788 820 : return data;
6789 :
6790 : return NULL;
6791 : }
6792 :
6793 : /* Return true if STMT uses a library function and needs to take
6794 : address of its inputs. We need to avoid bit-fields in those
6795 : cases. Similarly, we need to avoid overlap between destination
6796 : and source limb arrays. */
6797 :
6798 : bool
6799 15105 : stmt_needs_operand_addr (gimple *stmt)
6800 : {
6801 15105 : if (is_gimple_assign (stmt))
6802 10298 : switch (gimple_assign_rhs_code (stmt))
6803 : {
6804 585 : case MULT_EXPR:
6805 585 : case TRUNC_DIV_EXPR:
6806 585 : case EXACT_DIV_EXPR:
6807 585 : case TRUNC_MOD_EXPR:
6808 585 : case FLOAT_EXPR:
6809 585 : return true;
6810 : default:
6811 : break;
6812 : }
6813 4807 : else if (is_gimple_call (stmt) && gimple_call_internal_p (stmt))
6814 46 : switch (gimple_call_internal_fn (stmt))
6815 : {
6816 17 : case IFN_MUL_OVERFLOW:
6817 17 : case IFN_UBSAN_CHECK_MUL:
6818 17 : return true;
6819 : /* These two actually don't take address, but reshuffle
6820 : all bytes or bits, so need similar treatment. */
6821 4 : case IFN_BSWAP:
6822 4 : case IFN_BITREVERSE:
6823 4 : return true;
6824 : default:
6825 : break;
6826 : }
6827 : return false;
6828 : }
6829 :
6830 : /* Dominator walker used to discover which large/huge _BitInt
6831 : loads could be sunk into all their uses. */
6832 :
6833 630 : class bitint_dom_walker : public dom_walker
6834 : {
6835 : public:
6836 315 : bitint_dom_walker (bitmap names, bitmap loads)
6837 630 : : dom_walker (CDI_DOMINATORS), m_names (names), m_loads (loads) {}
6838 :
6839 : edge before_dom_children (basic_block) final override;
6840 :
6841 : private:
6842 : bitmap m_names, m_loads;
6843 : };
6844 :
6845 : edge
6846 4518 : bitint_dom_walker::before_dom_children (basic_block bb)
6847 : {
6848 4518 : gphi *phi = get_virtual_phi (bb);
6849 4518 : tree vop;
6850 4518 : if (phi)
6851 794 : vop = gimple_phi_result (phi);
6852 3724 : else if (bb == ENTRY_BLOCK_PTR_FOR_FN (cfun))
6853 : vop = NULL_TREE;
6854 : else
6855 3409 : vop = (tree) get_immediate_dominator (CDI_DOMINATORS, bb)->aux;
6856 :
6857 4518 : auto_vec<tree, 16> worklist;
6858 9036 : for (gimple_stmt_iterator gsi = gsi_start_bb (bb);
6859 20012 : !gsi_end_p (gsi); gsi_next (&gsi))
6860 : {
6861 15494 : gimple *stmt = gsi_stmt (gsi);
6862 15494 : if (is_gimple_debug (stmt))
6863 2769 : continue;
6864 :
6865 15542 : if (!vop && gimple_vuse (stmt))
6866 : vop = gimple_vuse (stmt);
6867 :
6868 15105 : tree cvop = vop;
6869 28350 : if (gimple_vdef (stmt))
6870 : vop = gimple_vdef (stmt);
6871 :
6872 15105 : tree lhs = gimple_get_lhs (stmt);
6873 17485 : if (lhs
6874 10971 : && TREE_CODE (lhs) == SSA_NAME
6875 8748 : && BITINT_TYPE_P (TREE_TYPE (lhs))
6876 5871 : && bitint_precision_kind (TREE_TYPE (lhs)) >= bitint_prec_large
6877 20828 : && !bitmap_bit_p (m_names, SSA_NAME_VERSION (lhs)))
6878 : /* If lhs of stmt is large/huge _BitInt SSA_NAME not in m_names,
6879 : it means it will be handled in a loop or straight line code
6880 : at the location of its (ultimate) immediate use, so for
6881 : vop checking purposes check these only at the ultimate
6882 : immediate use. */
6883 2380 : continue;
6884 :
6885 12725 : ssa_op_iter oi;
6886 12725 : use_operand_p use_p;
6887 21573 : FOR_EACH_SSA_USE_OPERAND (use_p, stmt, oi, SSA_OP_USE)
6888 : {
6889 8848 : tree s = USE_FROM_PTR (use_p);
6890 14272 : if (BITINT_TYPE_P (TREE_TYPE (s))
6891 8848 : && bitint_precision_kind (TREE_TYPE (s)) >= bitint_prec_large)
6892 3269 : worklist.safe_push (s);
6893 : }
6894 :
6895 12725 : bool needs_operand_addr = stmt_needs_operand_addr (stmt);
6896 31559 : while (worklist.length () > 0)
6897 : {
6898 6109 : tree s = worklist.pop ();
6899 :
6900 6109 : if (!bitmap_bit_p (m_names, SSA_NAME_VERSION (s)))
6901 : {
6902 2380 : gimple *g = SSA_NAME_DEF_STMT (s);
6903 2380 : needs_operand_addr |= stmt_needs_operand_addr (g);
6904 5459 : FOR_EACH_SSA_USE_OPERAND (use_p, g, oi, SSA_OP_USE)
6905 : {
6906 3079 : tree s2 = USE_FROM_PTR (use_p);
6907 3269 : if (BITINT_TYPE_P (TREE_TYPE (s2))
6908 3079 : && (bitint_precision_kind (TREE_TYPE (s2))
6909 : >= bitint_prec_large))
6910 2840 : worklist.safe_push (s2);
6911 : }
6912 3100 : continue;
6913 2380 : }
6914 3729 : if (!SSA_NAME_OCCURS_IN_ABNORMAL_PHI (s)
6915 3729 : && gimple_assign_cast_p (SSA_NAME_DEF_STMT (s)))
6916 : {
6917 226 : tree rhs = gimple_assign_rhs1 (SSA_NAME_DEF_STMT (s));
6918 396 : if (TREE_CODE (rhs) == SSA_NAME
6919 226 : && bitmap_bit_p (m_loads, SSA_NAME_VERSION (rhs)))
6920 : s = rhs;
6921 : else
6922 170 : continue;
6923 : }
6924 3503 : else if (!bitmap_bit_p (m_loads, SSA_NAME_VERSION (s)))
6925 548 : continue;
6926 :
6927 3011 : gimple *g = SSA_NAME_DEF_STMT (s);
6928 3011 : tree rhs1 = gimple_assign_rhs1 (g);
6929 3011 : if (needs_operand_addr
6930 218 : && TREE_CODE (rhs1) == COMPONENT_REF
6931 3028 : && DECL_BIT_FIELD_TYPE (TREE_OPERAND (rhs1, 1)))
6932 : {
6933 4 : tree fld = TREE_OPERAND (rhs1, 1);
6934 : /* For little-endian, we can allow as inputs bit-fields
6935 : which start at a limb boundary. */
6936 6 : if (!bitint_big_endian
6937 4 : && DECL_OFFSET_ALIGN (fld) >= TYPE_ALIGN (TREE_TYPE (rhs1))
6938 4 : && tree_fits_uhwi_p (DECL_FIELD_BIT_OFFSET (fld))
6939 4 : && (tree_to_uhwi (DECL_FIELD_BIT_OFFSET (fld))
6940 4 : % limb_prec) == 0)
6941 : ;
6942 : else
6943 : {
6944 2 : bitmap_clear_bit (m_loads, SSA_NAME_VERSION (s));
6945 2 : continue;
6946 : }
6947 : }
6948 :
6949 3009 : ao_ref ref;
6950 3009 : ao_ref_init (&ref, rhs1);
6951 3009 : tree lvop = gimple_vuse (g);
6952 3009 : unsigned limit = 64;
6953 3009 : tree vuse = cvop;
6954 3009 : if (vop != cvop
6955 1338 : && is_gimple_assign (stmt)
6956 1336 : && gimple_store_p (stmt)
6957 4345 : && (needs_operand_addr
6958 1143 : || !operand_equal_p (lhs, gimple_assign_rhs1 (g), 0)))
6959 : vuse = vop;
6960 3009 : if (vuse != lvop
6961 3009 : && walk_non_aliased_vuses (&ref, vuse, false, vuse_eq,
6962 : NULL, NULL, NULL, limit, lvop) == NULL)
6963 532 : bitmap_clear_bit (m_loads, SSA_NAME_VERSION (s));
6964 : }
6965 : }
6966 :
6967 4518 : bb->aux = (void *) vop;
6968 4518 : return NULL;
6969 4518 : }
6970 :
6971 : }
6972 :
6973 : /* Replacement for normal processing of STMT in tree-ssa-coalesce.cc
6974 : build_ssa_conflict_graph.
6975 : The differences are:
6976 : 1) don't process assignments with large/huge _BitInt lhs not in NAMES
6977 : 2) for large/huge _BitInt multiplication/division/modulo process def
6978 : only after processing uses rather than before to make uses conflict
6979 : with the definition
6980 : 3) for large/huge _BitInt uses not in NAMES mark the uses of their
6981 : SSA_NAME_DEF_STMT (recursively), because those uses will be sunk into
6982 : the final statement. */
6983 :
6984 : void
6985 84261 : build_bitint_stmt_ssa_conflicts (gimple *stmt, live_track *live,
6986 : ssa_conflicts *graph, bitmap names,
6987 : void (*def) (live_track *, tree,
6988 : ssa_conflicts *),
6989 : void (*use) (live_track *, tree),
6990 : void (*clear) (live_track *, tree))
6991 : {
6992 84261 : bool muldiv_p = false;
6993 84261 : tree lhs = NULL_TREE;
6994 84261 : if (is_gimple_assign (stmt))
6995 : {
6996 46265 : lhs = gimple_assign_lhs (stmt);
6997 46265 : if (TREE_CODE (lhs) == SSA_NAME)
6998 : {
6999 33443 : tree type = TREE_TYPE (lhs);
7000 33443 : if (TREE_CODE (type) == COMPLEX_TYPE)
7001 63 : type = TREE_TYPE (type);
7002 12866 : if (BITINT_TYPE_P (type)
7003 33448 : && bitint_precision_kind (type) >= bitint_prec_large)
7004 : {
7005 19964 : if (!bitmap_bit_p (names, SSA_NAME_VERSION (lhs)))
7006 4899 : return;
7007 :
7008 : /* A copy between 2 partitions does not introduce an interference
7009 : by itself. If they did, you would never be able to coalesce
7010 : two things which are copied. If the two variables really do
7011 : conflict, they will conflict elsewhere in the program.
7012 :
7013 : This is handled by simply removing the SRC of the copy from
7014 : the live list, and processing the stmt normally.
7015 :
7016 : Don't do this if lhs is not in names though, in such cases
7017 : it is actually used at some point later in the basic
7018 : block. */
7019 15065 : if (gimple_assign_copy_p (stmt))
7020 : {
7021 1686 : tree rhs1 = gimple_assign_rhs1 (stmt);
7022 1686 : if (TREE_CODE (rhs1) == SSA_NAME)
7023 45 : clear (live, rhs1);
7024 : }
7025 :
7026 15065 : switch (gimple_assign_rhs_code (stmt))
7027 : {
7028 166 : case MULT_EXPR:
7029 166 : case TRUNC_DIV_EXPR:
7030 166 : case EXACT_DIV_EXPR:
7031 166 : case TRUNC_MOD_EXPR:
7032 166 : muldiv_p = true;
7033 : default:
7034 : break;
7035 : }
7036 : }
7037 : }
7038 : }
7039 37996 : else if (is_gimple_call (stmt) && gimple_call_internal_p (stmt))
7040 4301 : switch (gimple_call_internal_fn (stmt))
7041 : {
7042 2699 : case IFN_ADD_OVERFLOW:
7043 2699 : case IFN_SUB_OVERFLOW:
7044 2699 : case IFN_UBSAN_CHECK_ADD:
7045 2699 : case IFN_UBSAN_CHECK_SUB:
7046 2699 : if (bitint_big_endian)
7047 : {
7048 0 : lhs = gimple_call_lhs (stmt);
7049 0 : if (lhs)
7050 79362 : muldiv_p = true;
7051 : }
7052 : break;
7053 1478 : case IFN_MUL_OVERFLOW:
7054 1478 : case IFN_UBSAN_CHECK_MUL:
7055 1478 : case IFN_BSWAP:
7056 1478 : case IFN_BITREVERSE:
7057 1478 : lhs = gimple_call_lhs (stmt);
7058 1478 : if (lhs)
7059 79362 : muldiv_p = true;
7060 : break;
7061 : default:
7062 : break;
7063 : }
7064 :
7065 158724 : auto_vec<tree, 16> worklist;
7066 79362 : ssa_op_iter iter;
7067 79362 : tree var;
7068 : /* On little-endian, mergeable ops process limbs from 0 up so except
7069 : for multiplication/division/modulo there is no risk in using the
7070 : same underlying variable for lhs and some operand, even when casts
7071 : are involved, the lhs limb is stored only after processing the source
7072 : limbs with the same index.
7073 : For multiplication/division/modulo, the libgcc library function requires
7074 : no aliasing between result and sources.
7075 : On big-endian, even mergeable ops limb processing can be problematic
7076 : though, because it can apply various index corrections e.g. when there
7077 : is a cast from operand with different number of limbs. So, make the
7078 : lhs conflict with all the operands which are (for now virtually) used on
7079 : the current stmt if there is any mismatch in the number of limbs between
7080 : operands and the lhs. */
7081 79362 : if (bitint_big_endian && lhs && !muldiv_p)
7082 : {
7083 0 : tree ltype = TREE_TYPE (lhs);
7084 0 : if (TREE_CODE (ltype) == COMPLEX_TYPE)
7085 : muldiv_p = true;
7086 0 : else if (TREE_CODE (lhs) == SSA_NAME
7087 0 : && BITINT_TYPE_P (ltype)
7088 0 : && bitint_precision_kind (ltype) >= bitint_prec_large)
7089 : {
7090 0 : unsigned lnelts = CEIL (TYPE_PRECISION (ltype), limb_prec);
7091 0 : FOR_EACH_SSA_TREE_OPERAND (var, stmt, iter, SSA_OP_USE)
7092 : {
7093 0 : tree type = TREE_TYPE (var);
7094 0 : if (TREE_CODE (type) == COMPLEX_TYPE)
7095 0 : type = TREE_TYPE (type);
7096 0 : if (BITINT_TYPE_P (type)
7097 0 : && bitint_precision_kind (type) >= bitint_prec_large)
7098 : {
7099 0 : if (bitmap_bit_p (names, SSA_NAME_VERSION (var)))
7100 : {
7101 0 : unsigned nelts = CEIL (TYPE_PRECISION (type), limb_prec);
7102 0 : if (TREE_CODE (TREE_TYPE (var)) == COMPLEX_TYPE
7103 0 : || lnelts != nelts)
7104 : {
7105 0 : muldiv_p = true;
7106 : break;
7107 : }
7108 : }
7109 : else
7110 0 : worklist.safe_push (var);
7111 : }
7112 : }
7113 :
7114 0 : while (!muldiv_p && worklist.length () > 0)
7115 : {
7116 0 : tree s = worklist.pop ();
7117 0 : FOR_EACH_SSA_TREE_OPERAND (var, SSA_NAME_DEF_STMT (s), iter,
7118 : SSA_OP_USE)
7119 : {
7120 0 : tree type = TREE_TYPE (var);
7121 0 : if (TREE_CODE (type) == COMPLEX_TYPE)
7122 0 : type = TREE_TYPE (type);
7123 0 : if (BITINT_TYPE_P (type)
7124 0 : && bitint_precision_kind (type) >= bitint_prec_large)
7125 : {
7126 0 : if (bitmap_bit_p (names, SSA_NAME_VERSION (var)))
7127 : {
7128 0 : unsigned nelts = CEIL (TYPE_PRECISION (type),
7129 : limb_prec);
7130 0 : if (TREE_CODE (TREE_TYPE (var)) == COMPLEX_TYPE
7131 0 : || lnelts != nelts)
7132 : {
7133 : muldiv_p = true;
7134 : break;
7135 : }
7136 : }
7137 : else
7138 0 : worklist.safe_push (var);
7139 : }
7140 : }
7141 : }
7142 0 : worklist.truncate (0);
7143 : }
7144 : }
7145 :
7146 79362 : if (!muldiv_p)
7147 : {
7148 : /* For stmts with more than one SSA_NAME definition pretend all the
7149 : SSA_NAME outputs but the first one are live at this point, so
7150 : that conflicts are added in between all those even when they are
7151 : actually not really live after the asm, because expansion might
7152 : copy those into pseudos after the asm and if multiple outputs
7153 : share the same partition, it might overwrite those that should
7154 : be live. E.g.
7155 : asm volatile (".." : "=r" (a) : "=r" (b) : "0" (a), "1" (a));
7156 : return a;
7157 : See PR70593. */
7158 77718 : bool first = true;
7159 113747 : FOR_EACH_SSA_TREE_OPERAND (var, stmt, iter, SSA_OP_DEF)
7160 36029 : if (first)
7161 : first = false;
7162 : else
7163 0 : use (live, var);
7164 :
7165 113747 : FOR_EACH_SSA_TREE_OPERAND (var, stmt, iter, SSA_OP_DEF)
7166 36029 : def (live, var, graph);
7167 : }
7168 :
7169 135299 : FOR_EACH_SSA_TREE_OPERAND (var, stmt, iter, SSA_OP_USE)
7170 : {
7171 55937 : tree type = TREE_TYPE (var);
7172 55937 : if (TREE_CODE (type) == COMPLEX_TYPE)
7173 6313 : type = TREE_TYPE (type);
7174 18881 : if (BITINT_TYPE_P (type)
7175 55954 : && bitint_precision_kind (type) >= bitint_prec_large)
7176 : {
7177 36074 : if (bitmap_bit_p (names, SSA_NAME_VERSION (var)))
7178 30974 : use (live, var);
7179 : else
7180 5100 : worklist.safe_push (var);
7181 : }
7182 : }
7183 :
7184 87462 : while (worklist.length () > 0)
7185 : {
7186 8100 : tree s = worklist.pop ();
7187 16940 : FOR_EACH_SSA_TREE_OPERAND (var, SSA_NAME_DEF_STMT (s), iter, SSA_OP_USE)
7188 : {
7189 8840 : tree type = TREE_TYPE (var);
7190 8840 : if (TREE_CODE (type) == COMPLEX_TYPE)
7191 1675 : type = TREE_TYPE (type);
7192 536 : if (BITINT_TYPE_P (type)
7193 8845 : && bitint_precision_kind (type) >= bitint_prec_large)
7194 : {
7195 8174 : if (bitmap_bit_p (names, SSA_NAME_VERSION (var)))
7196 5174 : use (live, var);
7197 : else
7198 3000 : worklist.safe_push (var);
7199 : }
7200 : }
7201 : }
7202 :
7203 79362 : if (muldiv_p)
7204 1644 : def (live, lhs, graph);
7205 : }
7206 :
7207 : /* If STMT is .{ADD,SUB,MUL}_OVERFLOW with INTEGER_CST arguments,
7208 : return the largest bitint_prec_kind of them, otherwise return
7209 : bitint_prec_small. */
7210 :
7211 : static bitint_prec_kind
7212 209983 : arith_overflow_arg_kind (gimple *stmt)
7213 : {
7214 209983 : bitint_prec_kind ret = bitint_prec_small;
7215 209983 : if (is_gimple_call (stmt) && gimple_call_internal_p (stmt))
7216 88685 : switch (gimple_call_internal_fn (stmt))
7217 : {
7218 : case IFN_ADD_OVERFLOW:
7219 : case IFN_SUB_OVERFLOW:
7220 : case IFN_MUL_OVERFLOW:
7221 225087 : for (int i = 0; i < 2; ++i)
7222 : {
7223 150058 : tree a = gimple_call_arg (stmt, i);
7224 150058 : if (TREE_CODE (a) == INTEGER_CST
7225 150058 : && BITINT_TYPE_P (TREE_TYPE (a)))
7226 : {
7227 5930 : bitint_prec_kind kind = bitint_precision_kind (TREE_TYPE (a));
7228 150058 : ret = MAX (ret, kind);
7229 : }
7230 : }
7231 : break;
7232 : default:
7233 : break;
7234 : }
7235 209983 : return ret;
7236 : }
7237 :
7238 : /* Entry point for _BitInt(N) operation lowering during optimization. */
7239 :
7240 : static unsigned int
7241 1512279 : gimple_lower_bitint (void)
7242 : {
7243 1512279 : small_max_prec = mid_min_prec = large_min_prec = huge_min_prec = 0;
7244 1512279 : limb_prec = abi_limb_prec = 0;
7245 1512279 : bitint_big_endian = false;
7246 :
7247 1512279 : unsigned int i;
7248 64946096 : for (i = 0; i < num_ssa_names; ++i)
7249 : {
7250 63441183 : tree s = ssa_name (i);
7251 63441183 : if (s == NULL)
7252 13616584 : continue;
7253 49824599 : tree type = TREE_TYPE (s);
7254 49824599 : if (TREE_CODE (type) == COMPLEX_TYPE)
7255 : {
7256 199303 : if (arith_overflow_arg_kind (SSA_NAME_DEF_STMT (s))
7257 : != bitint_prec_small)
7258 : break;
7259 199190 : type = TREE_TYPE (type);
7260 : }
7261 49807601 : if (BITINT_TYPE_P (type)
7262 49824493 : && bitint_precision_kind (type) != bitint_prec_small)
7263 : break;
7264 : /* We need to also rewrite stores of large/huge _BitInt INTEGER_CSTs
7265 : into memory. Such functions could have no large/huge SSA_NAMEs. */
7266 49817301 : if (SSA_NAME_IS_VIRTUAL_OPERAND (s))
7267 : {
7268 21793558 : gimple *g = SSA_NAME_DEF_STMT (s);
7269 21793558 : if (is_gimple_assign (g) && gimple_store_p (g))
7270 : {
7271 11065231 : tree t = gimple_assign_rhs1 (g);
7272 22127966 : if (BITINT_TYPE_P (TREE_TYPE (t))
7273 11065231 : && (bitint_precision_kind (TREE_TYPE (t))
7274 : >= bitint_prec_large))
7275 : break;
7276 : }
7277 : }
7278 : /* Similarly, e.g. with -frounding-math casts from _BitInt INTEGER_CSTs
7279 : to floating point types need to be rewritten. */
7280 28023743 : else if (SCALAR_FLOAT_TYPE_P (type))
7281 : {
7282 2327287 : gimple *g = SSA_NAME_DEF_STMT (s);
7283 2327287 : if (is_gimple_assign (g) && gimple_assign_rhs_code (g) == FLOAT_EXPR)
7284 : {
7285 128466 : tree t = gimple_assign_rhs1 (g);
7286 128466 : if (TREE_CODE (t) == INTEGER_CST
7287 110 : && BITINT_TYPE_P (TREE_TYPE (t))
7288 128467 : && (bitint_precision_kind (TREE_TYPE (t))
7289 : != bitint_prec_small))
7290 : break;
7291 : }
7292 : }
7293 : }
7294 3024558 : if (i == num_ssa_names)
7295 : return 0;
7296 :
7297 7366 : basic_block bb;
7298 7366 : auto_vec<gimple *, 4> switch_statements;
7299 46241 : FOR_EACH_BB_FN (bb, cfun)
7300 : {
7301 115858 : if (gswitch *swtch = safe_dyn_cast <gswitch *> (*gsi_last_bb (bb)))
7302 : {
7303 24 : tree idx = gimple_switch_index (swtch);
7304 32 : if (!BITINT_TYPE_P (TREE_TYPE (idx))
7305 24 : || bitint_precision_kind (TREE_TYPE (idx)) < bitint_prec_large)
7306 13 : continue;
7307 :
7308 11 : if (optimize)
7309 6 : group_case_labels_stmt (swtch);
7310 11 : if (gimple_switch_num_labels (swtch) == 1)
7311 : {
7312 0 : single_succ_edge (bb)->flags |= EDGE_FALLTHRU;
7313 0 : gimple_stmt_iterator gsi = gsi_for_stmt (swtch);
7314 0 : gsi_remove (&gsi, true);
7315 : }
7316 : else
7317 11 : switch_statements.safe_push (swtch);
7318 : }
7319 : }
7320 :
7321 7366 : if (!switch_statements.is_empty ())
7322 : {
7323 11 : bool expanded = false;
7324 11 : gimple *stmt;
7325 11 : unsigned int j;
7326 11 : i = 0;
7327 22 : FOR_EACH_VEC_ELT (switch_statements, j, stmt)
7328 : {
7329 11 : gswitch *swtch = as_a<gswitch *> (stmt);
7330 11 : tree_switch_conversion::switch_decision_tree dt (swtch);
7331 11 : expanded |= dt.analyze_switch_statement ();
7332 11 : }
7333 :
7334 11 : if (expanded)
7335 : {
7336 11 : free_dominance_info (CDI_DOMINATORS);
7337 11 : free_dominance_info (CDI_POST_DOMINATORS);
7338 11 : mark_virtual_operands_for_renaming (cfun);
7339 11 : cleanup_tree_cfg (TODO_update_ssa);
7340 : }
7341 : }
7342 :
7343 7366 : if (flag_sanitize & (SANITIZE_ADDRESS | SANITIZE_HWADDRESS))
7344 : {
7345 14 : hash_map<tree, tree> shadow_vars_mapping;
7346 14 : bool need_commit_edge_insert = false;
7347 14 : bool any_asan_poison = false;
7348 129 : for (unsigned j = 0; j < num_ssa_names; ++j)
7349 : {
7350 115 : tree s = ssa_name (j);
7351 115 : if (s == NULL)
7352 31 : continue;
7353 84 : tree type = TREE_TYPE (s);
7354 37 : if (BITINT_TYPE_P (type)
7355 47 : && gimple_call_internal_p (SSA_NAME_DEF_STMT (s),
7356 : IFN_ASAN_POISON)
7357 112 : && bitint_precision_kind (type) >= bitint_prec_large)
7358 : {
7359 28 : any_asan_poison = true;
7360 28 : gimple_stmt_iterator gsi = gsi_for_stmt (SSA_NAME_DEF_STMT (s));
7361 28 : asan_expand_poison_ifn (&gsi, &need_commit_edge_insert,
7362 : shadow_vars_mapping);
7363 : }
7364 : }
7365 14 : if (any_asan_poison)
7366 : {
7367 14 : if (need_commit_edge_insert)
7368 0 : gsi_commit_edge_inserts ();
7369 14 : i = 0;
7370 14 : free_dominance_info (CDI_DOMINATORS);
7371 14 : free_dominance_info (CDI_POST_DOMINATORS);
7372 14 : mark_virtual_operands_for_renaming (cfun);
7373 14 : cleanup_tree_cfg (TODO_update_ssa);
7374 : }
7375 14 : }
7376 :
7377 7366 : struct bitint_large_huge large_huge;
7378 7366 : bool has_large_huge_parm_result = false;
7379 7366 : bool has_large_huge = false;
7380 7366 : unsigned int ret = 0, first_large_huge = ~0U;
7381 7366 : bool edge_insertions = false;
7382 126841 : for (; i < num_ssa_names; ++i)
7383 : {
7384 119475 : tree s = ssa_name (i);
7385 119475 : if (s == NULL)
7386 2811 : continue;
7387 116664 : tree type = TREE_TYPE (s);
7388 116664 : if (TREE_CODE (type) == COMPLEX_TYPE)
7389 : {
7390 5355 : if (arith_overflow_arg_kind (SSA_NAME_DEF_STMT (s))
7391 : >= bitint_prec_large)
7392 1958 : has_large_huge = true;
7393 5355 : type = TREE_TYPE (type);
7394 : }
7395 70409 : if (BITINT_TYPE_P (type)
7396 116690 : && bitint_precision_kind (type) >= bitint_prec_large)
7397 : {
7398 36169 : if (first_large_huge == ~0U)
7399 5814 : first_large_huge = i;
7400 36169 : gimple *stmt = SSA_NAME_DEF_STMT (s), *g;
7401 36169 : gimple_stmt_iterator gsi;
7402 36169 : tree_code rhs_code;
7403 : /* Unoptimize certain constructs to simpler alternatives to
7404 : avoid having to lower all of them. */
7405 36169 : if (is_gimple_assign (stmt) && gimple_bb (stmt))
7406 22279 : switch (rhs_code = gimple_assign_rhs_code (stmt))
7407 : {
7408 : default:
7409 : break;
7410 371 : case MULT_EXPR:
7411 371 : case TRUNC_DIV_EXPR:
7412 371 : case EXACT_DIV_EXPR:
7413 371 : case TRUNC_MOD_EXPR:
7414 371 : if (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (s))
7415 : {
7416 2 : location_t loc = gimple_location (stmt);
7417 2 : gsi = gsi_for_stmt (stmt);
7418 2 : tree rhs1 = gimple_assign_rhs1 (stmt);
7419 2 : tree rhs2 = gimple_assign_rhs2 (stmt);
7420 : /* For multiplication and division with (ab)
7421 : lhs and one or both operands force the operands
7422 : into new SSA_NAMEs to avoid coalescing failures. */
7423 2 : if (TREE_CODE (rhs1) == SSA_NAME
7424 2 : && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (rhs1))
7425 : {
7426 2 : first_large_huge = 0;
7427 2 : tree t = make_ssa_name (TREE_TYPE (rhs1));
7428 2 : g = gimple_build_assign (t, SSA_NAME, rhs1);
7429 2 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7430 2 : gimple_set_location (g, loc);
7431 2 : gimple_assign_set_rhs1 (stmt, t);
7432 2 : if (rhs1 == rhs2)
7433 : {
7434 0 : gimple_assign_set_rhs2 (stmt, t);
7435 0 : rhs2 = t;
7436 : }
7437 2 : update_stmt (stmt);
7438 : }
7439 2 : if (TREE_CODE (rhs2) == SSA_NAME
7440 2 : && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (rhs2))
7441 : {
7442 0 : first_large_huge = 0;
7443 0 : tree t = make_ssa_name (TREE_TYPE (rhs2));
7444 0 : g = gimple_build_assign (t, SSA_NAME, rhs2);
7445 0 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7446 0 : gimple_set_location (g, loc);
7447 0 : gimple_assign_set_rhs2 (stmt, t);
7448 0 : update_stmt (stmt);
7449 : }
7450 : }
7451 : break;
7452 3 : case LROTATE_EXPR:
7453 3 : case RROTATE_EXPR:
7454 3 : {
7455 3 : first_large_huge = 0;
7456 3 : location_t loc = gimple_location (stmt);
7457 3 : gsi = gsi_for_stmt (stmt);
7458 3 : tree rhs1 = gimple_assign_rhs1 (stmt);
7459 3 : tree type = TREE_TYPE (rhs1);
7460 3 : tree n = gimple_assign_rhs2 (stmt), m;
7461 3 : tree p = build_int_cst (TREE_TYPE (n),
7462 3 : TYPE_PRECISION (type));
7463 3 : if (TREE_CODE (n) == INTEGER_CST)
7464 : {
7465 0 : if (integer_zerop (n))
7466 : m = n;
7467 : else
7468 0 : m = fold_build2 (MINUS_EXPR, TREE_TYPE (n), p, n);
7469 : }
7470 : else
7471 : {
7472 3 : tree tem = make_ssa_name (TREE_TYPE (n));
7473 3 : g = gimple_build_assign (tem, MINUS_EXPR, p, n);
7474 3 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7475 3 : gimple_set_location (g, loc);
7476 3 : m = make_ssa_name (TREE_TYPE (n));
7477 3 : g = gimple_build_assign (m, TRUNC_MOD_EXPR, tem, p);
7478 3 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7479 3 : gimple_set_location (g, loc);
7480 : }
7481 3 : if (!TYPE_UNSIGNED (type))
7482 : {
7483 0 : tree utype = build_bitint_type (TYPE_PRECISION (type),
7484 : 1);
7485 0 : if (TREE_CODE (rhs1) == INTEGER_CST)
7486 0 : rhs1 = fold_convert (utype, rhs1);
7487 : else
7488 : {
7489 0 : tree t = make_ssa_name (type);
7490 0 : g = gimple_build_assign (t, NOP_EXPR, rhs1);
7491 0 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7492 0 : gimple_set_location (g, loc);
7493 : }
7494 : }
7495 4 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (rhs1)),
7496 : rhs_code == LROTATE_EXPR
7497 : ? LSHIFT_EXPR : RSHIFT_EXPR,
7498 : rhs1, n);
7499 3 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7500 3 : gimple_set_location (g, loc);
7501 3 : tree op1 = gimple_assign_lhs (g);
7502 4 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (rhs1)),
7503 : rhs_code == LROTATE_EXPR
7504 : ? RSHIFT_EXPR : LSHIFT_EXPR,
7505 : rhs1, m);
7506 3 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7507 3 : gimple_set_location (g, loc);
7508 3 : tree op2 = gimple_assign_lhs (g);
7509 3 : tree lhs = gimple_assign_lhs (stmt);
7510 3 : if (!TYPE_UNSIGNED (type))
7511 : {
7512 0 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (op1)),
7513 : BIT_IOR_EXPR, op1, op2);
7514 0 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7515 0 : gimple_set_location (g, loc);
7516 0 : g = gimple_build_assign (lhs, NOP_EXPR,
7517 : gimple_assign_lhs (g));
7518 : }
7519 : else
7520 3 : g = gimple_build_assign (lhs, BIT_IOR_EXPR, op1, op2);
7521 3 : gsi_replace (&gsi, g, true);
7522 3 : gimple_set_location (g, loc);
7523 : }
7524 3 : break;
7525 17 : case ABS_EXPR:
7526 17 : case ABSU_EXPR:
7527 17 : case MIN_EXPR:
7528 17 : case MAX_EXPR:
7529 17 : case COND_EXPR:
7530 17 : first_large_huge = 0;
7531 17 : gsi = gsi_for_stmt (stmt);
7532 17 : tree lhs = gimple_assign_lhs (stmt);
7533 17 : tree rhs1 = gimple_assign_rhs1 (stmt), rhs2 = NULL_TREE;
7534 17 : location_t loc = gimple_location (stmt);
7535 17 : if (rhs_code == ABS_EXPR)
7536 4 : g = gimple_build_cond (LT_EXPR, rhs1,
7537 4 : build_zero_cst (TREE_TYPE (rhs1)),
7538 : NULL_TREE, NULL_TREE);
7539 13 : else if (rhs_code == ABSU_EXPR)
7540 : {
7541 4 : rhs2 = make_ssa_name (TREE_TYPE (lhs));
7542 4 : g = gimple_build_assign (rhs2, NOP_EXPR, rhs1);
7543 4 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7544 4 : gimple_set_location (g, loc);
7545 4 : g = gimple_build_cond (LT_EXPR, rhs1,
7546 4 : build_zero_cst (TREE_TYPE (rhs1)),
7547 : NULL_TREE, NULL_TREE);
7548 4 : rhs1 = rhs2;
7549 : }
7550 9 : else if (rhs_code == MIN_EXPR || rhs_code == MAX_EXPR)
7551 : {
7552 9 : rhs2 = gimple_assign_rhs2 (stmt);
7553 9 : if (TREE_CODE (rhs1) == INTEGER_CST)
7554 0 : std::swap (rhs1, rhs2);
7555 9 : g = gimple_build_cond (LT_EXPR, rhs1, rhs2,
7556 : NULL_TREE, NULL_TREE);
7557 9 : if (rhs_code == MAX_EXPR)
7558 5 : std::swap (rhs1, rhs2);
7559 : }
7560 : else
7561 : {
7562 0 : g = gimple_build_cond (NE_EXPR, rhs1,
7563 0 : build_zero_cst (TREE_TYPE (rhs1)),
7564 : NULL_TREE, NULL_TREE);
7565 0 : rhs1 = gimple_assign_rhs2 (stmt);
7566 0 : rhs2 = gimple_assign_rhs3 (stmt);
7567 : }
7568 17 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7569 17 : gimple_set_location (g, loc);
7570 17 : edge e1 = split_block (gsi_bb (gsi), g);
7571 17 : edge e2 = split_block (e1->dest, (gimple *) NULL);
7572 17 : edge e3 = make_edge (e1->src, e2->dest, EDGE_FALSE_VALUE);
7573 17 : e3->probability = profile_probability::even ();
7574 17 : e1->flags = EDGE_TRUE_VALUE;
7575 17 : e1->probability = e3->probability.invert ();
7576 17 : if (dom_info_available_p (CDI_DOMINATORS))
7577 9 : set_immediate_dominator (CDI_DOMINATORS, e2->dest, e1->src);
7578 17 : if (rhs_code == ABS_EXPR || rhs_code == ABSU_EXPR)
7579 : {
7580 8 : gsi = gsi_after_labels (e1->dest);
7581 8 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (rhs1)),
7582 : NEGATE_EXPR, rhs1);
7583 8 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7584 8 : gimple_set_location (g, loc);
7585 8 : rhs2 = gimple_assign_lhs (g);
7586 8 : std::swap (rhs1, rhs2);
7587 : }
7588 17 : gsi = gsi_for_stmt (stmt);
7589 17 : gsi_remove (&gsi, true);
7590 17 : gphi *phi = create_phi_node (lhs, e2->dest);
7591 17 : add_phi_arg (phi, rhs1, e2, UNKNOWN_LOCATION);
7592 17 : add_phi_arg (phi, rhs2, e3, UNKNOWN_LOCATION);
7593 17 : break;
7594 : }
7595 13890 : else if (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (s)
7596 76 : && is_gimple_call (stmt)
7597 13910 : && gimple_call_internal_p (stmt))
7598 20 : switch (gimple_call_internal_fn (stmt))
7599 : {
7600 12 : case IFN_UBSAN_CHECK_ADD:
7601 12 : case IFN_UBSAN_CHECK_SUB:
7602 12 : if (!bitint_big_endian)
7603 : break;
7604 : /* FALLTHRU */
7605 : case IFN_UBSAN_CHECK_MUL:
7606 : case IFN_BSWAP:
7607 : case IFN_BITREVERSE:
7608 22 : for (unsigned i = 0; i < gimple_call_num_args (stmt); ++i)
7609 : {
7610 14 : location_t loc = gimple_location (stmt);
7611 14 : gsi = gsi_for_stmt (stmt);
7612 : /* Similar case to multiplication/division with (ab)
7613 : above for internal functions which set muldiv_p. */
7614 14 : tree arg = gimple_call_arg (stmt, i);
7615 14 : if (TREE_CODE (arg) == SSA_NAME
7616 14 : && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (arg))
7617 : {
7618 8 : first_large_huge = 0;
7619 8 : tree t = make_ssa_name (TREE_TYPE (arg));
7620 8 : g = gimple_build_assign (t, SSA_NAME, arg);
7621 8 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7622 8 : gimple_set_location (g, loc);
7623 8 : gimple_call_set_arg (stmt, i, t);
7624 8 : if (i == 0
7625 8 : && gimple_call_num_args (stmt) >= 2
7626 14 : && gimple_call_arg (stmt, 1) == arg)
7627 0 : gimple_call_set_arg (stmt, 1, t);
7628 8 : update_stmt (stmt);
7629 : }
7630 : }
7631 : break;
7632 : default:
7633 : break;
7634 : }
7635 : }
7636 : /* We need to also rewrite stores of large/huge _BitInt INTEGER_CSTs
7637 : into memory. Such functions could have no large/huge SSA_NAMEs. */
7638 80495 : else if (SSA_NAME_IS_VIRTUAL_OPERAND (s))
7639 : {
7640 51244 : gimple *g = SSA_NAME_DEF_STMT (s);
7641 51244 : if (is_gimple_assign (g) && gimple_store_p (g))
7642 : {
7643 16396 : tree t = gimple_assign_rhs1 (g);
7644 18579 : if (BITINT_TYPE_P (TREE_TYPE (t))
7645 16396 : && (bitint_precision_kind (TREE_TYPE (t))
7646 : >= bitint_prec_large))
7647 : has_large_huge = true;
7648 : }
7649 : }
7650 : /* Similarly, e.g. with -frounding-math casts from _BitInt INTEGER_CSTs
7651 : to floating point types need to be rewritten. */
7652 29251 : else if (SCALAR_FLOAT_TYPE_P (type))
7653 : {
7654 694 : gimple *g = SSA_NAME_DEF_STMT (s);
7655 694 : if (is_gimple_assign (g) && gimple_assign_rhs_code (g) == FLOAT_EXPR)
7656 : {
7657 187 : tree t = gimple_assign_rhs1 (g);
7658 187 : if (TREE_CODE (t) == INTEGER_CST
7659 1 : && BITINT_TYPE_P (TREE_TYPE (t))
7660 188 : && (bitint_precision_kind (TREE_TYPE (t))
7661 : >= bitint_prec_large))
7662 : has_large_huge = true;
7663 : }
7664 : }
7665 : }
7666 103229 : for (i = first_large_huge; i < num_ssa_names; ++i)
7667 : {
7668 95863 : tree s = ssa_name (i);
7669 95863 : if (s == NULL)
7670 2465 : continue;
7671 93398 : tree type = TREE_TYPE (s);
7672 93398 : if (TREE_CODE (type) == COMPLEX_TYPE)
7673 4124 : type = TREE_TYPE (type);
7674 56591 : if (BITINT_TYPE_P (type)
7675 93422 : && bitint_precision_kind (type) >= bitint_prec_large)
7676 : {
7677 36169 : use_operand_p use_p;
7678 36169 : gimple *use_stmt;
7679 36169 : has_large_huge = true;
7680 37833 : if (optimize
7681 53220 : && optimizable_arith_overflow (SSA_NAME_DEF_STMT (s)))
7682 6577 : continue;
7683 : /* Ignore large/huge _BitInt SSA_NAMEs which have single use in
7684 : the same bb and could be handled in the same loop with the
7685 : immediate use. */
7686 34505 : if (optimize
7687 15387 : && !SSA_NAME_OCCURS_IN_ABNORMAL_PHI (s)
7688 15320 : && single_imm_use (s, &use_p, &use_stmt)
7689 49212 : && gimple_bb (SSA_NAME_DEF_STMT (s)) == gimple_bb (use_stmt))
7690 : {
7691 10345 : if (mergeable_op (SSA_NAME_DEF_STMT (s)))
7692 : {
7693 2222 : if (mergeable_op (use_stmt))
7694 1911 : continue;
7695 311 : tree_code cmp_code = comparison_op (use_stmt, NULL, NULL);
7696 311 : if (cmp_code == EQ_EXPR || cmp_code == NE_EXPR)
7697 26 : continue;
7698 285 : if (gimple_assign_cast_p (use_stmt))
7699 : {
7700 130 : tree lhs = gimple_assign_lhs (use_stmt);
7701 260 : if (INTEGRAL_TYPE_P (TREE_TYPE (lhs))
7702 : /* Don't merge with VIEW_CONVERT_EXPRs to
7703 : huge INTEGER_TYPEs used sometimes in memcpy
7704 : expansion. */
7705 248 : && (TREE_CODE (TREE_TYPE (lhs)) != INTEGER_TYPE
7706 6 : || (TYPE_PRECISION (TREE_TYPE (lhs))
7707 12 : <= MAX_FIXED_MODE_SIZE)))
7708 118 : continue;
7709 : }
7710 155 : else if (gimple_store_p (use_stmt)
7711 0 : && is_gimple_assign (use_stmt)
7712 0 : && !gimple_has_volatile_ops (use_stmt)
7713 155 : && !stmt_ends_bb_p (use_stmt))
7714 0 : continue;
7715 : }
7716 8290 : if (gimple_assign_cast_p (SSA_NAME_DEF_STMT (s)))
7717 : {
7718 864 : tree rhs1 = gimple_assign_rhs1 (SSA_NAME_DEF_STMT (s));
7719 864 : if (TREE_CODE (rhs1) == VIEW_CONVERT_EXPR)
7720 : {
7721 20 : rhs1 = TREE_OPERAND (rhs1, 0);
7722 20 : if (!INTEGRAL_TYPE_P (TREE_TYPE (rhs1))
7723 16 : && !POINTER_TYPE_P (TREE_TYPE (rhs1))
7724 16 : && gimple_store_p (use_stmt))
7725 7 : continue;
7726 : }
7727 1714 : if (INTEGRAL_TYPE_P (TREE_TYPE (rhs1))
7728 766 : && ((is_gimple_assign (use_stmt)
7729 719 : && (gimple_assign_rhs_code (use_stmt)
7730 : != COMPLEX_EXPR))
7731 47 : || gimple_code (use_stmt) == GIMPLE_COND)
7732 756 : && (!gimple_store_p (use_stmt)
7733 132 : || (is_gimple_assign (use_stmt)
7734 132 : && !gimple_has_volatile_ops (use_stmt)
7735 132 : && !stmt_ends_bb_p (use_stmt)))
7736 1613 : && (TREE_CODE (rhs1) != SSA_NAME
7737 756 : || !SSA_NAME_OCCURS_IN_ABNORMAL_PHI (rhs1)))
7738 : {
7739 756 : if (is_gimple_assign (use_stmt))
7740 719 : switch (gimple_assign_rhs_code (use_stmt))
7741 : {
7742 54 : case TRUNC_DIV_EXPR:
7743 54 : case EXACT_DIV_EXPR:
7744 54 : case TRUNC_MOD_EXPR:
7745 54 : case FLOAT_EXPR:
7746 : /* For division, modulo and casts to floating
7747 : point, avoid representing unsigned operands
7748 : using negative prec if they were sign-extended
7749 : from narrower precision. */
7750 54 : if (TYPE_UNSIGNED (TREE_TYPE (s))
7751 28 : && !TYPE_UNSIGNED (TREE_TYPE (rhs1))
7752 63 : && (TYPE_PRECISION (TREE_TYPE (s))
7753 9 : > TYPE_PRECISION (TREE_TYPE (rhs1))))
7754 8 : goto force_name;
7755 : /* FALLTHRU */
7756 113 : case MULT_EXPR:
7757 128 : if (!BITINT_TYPE_P (TREE_TYPE (rhs1))
7758 113 : || (bitint_precision_kind (TREE_TYPE (rhs1))
7759 : < bitint_prec_large))
7760 48 : continue;
7761 : /* Uses which use handle_operand_addr can't
7762 : deal with nested casts. */
7763 65 : if (TREE_CODE (rhs1) == SSA_NAME
7764 65 : && gimple_assign_cast_p
7765 65 : (SSA_NAME_DEF_STMT (rhs1))
7766 43 : && has_single_use (rhs1)
7767 108 : && (gimple_bb (SSA_NAME_DEF_STMT (rhs1))
7768 43 : == gimple_bb (SSA_NAME_DEF_STMT (s))))
7769 43 : goto force_name;
7770 : break;
7771 0 : case VIEW_CONVERT_EXPR:
7772 0 : {
7773 0 : tree lhs = gimple_assign_lhs (use_stmt);
7774 : /* Don't merge with VIEW_CONVERT_EXPRs to
7775 : non-integral types. */
7776 0 : if (!INTEGRAL_TYPE_P (TREE_TYPE (lhs)))
7777 0 : goto force_name;
7778 : /* Don't merge with VIEW_CONVERT_EXPRs to
7779 : huge INTEGER_TYPEs used sometimes in memcpy
7780 : expansion. */
7781 0 : if (TREE_CODE (TREE_TYPE (lhs)) == INTEGER_TYPE
7782 0 : && (TYPE_PRECISION (TREE_TYPE (lhs))
7783 0 : > MAX_FIXED_MODE_SIZE))
7784 0 : goto force_name;
7785 : }
7786 : break;
7787 : default:
7788 : break;
7789 : }
7790 834 : if (!BITINT_TYPE_P (TREE_TYPE (rhs1))
7791 657 : || (bitint_precision_kind (TREE_TYPE (rhs1))
7792 : < bitint_prec_large))
7793 239 : continue;
7794 418 : if ((TYPE_PRECISION (TREE_TYPE (rhs1))
7795 418 : >= TYPE_PRECISION (TREE_TYPE (s)))
7796 418 : && mergeable_op (use_stmt))
7797 60 : continue;
7798 : /* Prevent merging a widening non-mergeable cast
7799 : on result of some narrower mergeable op
7800 : together with later mergeable operations. E.g.
7801 : result of _BitInt(223) addition shouldn't be
7802 : sign-extended to _BitInt(513) and have another
7803 : _BitInt(513) added to it, as handle_plus_minus
7804 : with its PHI node handling inside of handle_cast
7805 : will not work correctly. An exception is if
7806 : use_stmt is a store, this is handled directly
7807 : in lower_mergeable_stmt. */
7808 709 : if (TREE_CODE (rhs1) != SSA_NAME
7809 358 : || !has_single_use (rhs1)
7810 272 : || (gimple_bb (SSA_NAME_DEF_STMT (rhs1))
7811 272 : != gimple_bb (SSA_NAME_DEF_STMT (s)))
7812 216 : || !mergeable_op (SSA_NAME_DEF_STMT (rhs1))
7813 445 : || gimple_store_p (use_stmt))
7814 351 : continue;
7815 7 : if ((TYPE_PRECISION (TREE_TYPE (rhs1))
7816 7 : < TYPE_PRECISION (TREE_TYPE (s)))
7817 9 : && gimple_assign_cast_p (SSA_NAME_DEF_STMT (rhs1)))
7818 : {
7819 : /* Another exception is if the widening cast is
7820 : from mergeable same precision cast from something
7821 : not mergeable. */
7822 0 : tree rhs2
7823 0 : = gimple_assign_rhs1 (SSA_NAME_DEF_STMT (rhs1));
7824 0 : if (BITINT_TYPE_P (TREE_TYPE (rhs2))
7825 0 : && (TYPE_PRECISION (TREE_TYPE (rhs1))
7826 0 : == TYPE_PRECISION (TREE_TYPE (rhs2))))
7827 : {
7828 0 : if (TREE_CODE (rhs2) != SSA_NAME
7829 0 : || !has_single_use (rhs2)
7830 0 : || (gimple_bb (SSA_NAME_DEF_STMT (rhs2))
7831 0 : != gimple_bb (SSA_NAME_DEF_STMT (s)))
7832 0 : || !mergeable_op (SSA_NAME_DEF_STMT (rhs2)))
7833 0 : continue;
7834 : }
7835 : }
7836 : }
7837 : }
7838 7534 : if (is_gimple_assign (SSA_NAME_DEF_STMT (s)))
7839 5424 : switch (gimple_assign_rhs_code (SSA_NAME_DEF_STMT (s)))
7840 : {
7841 1794 : case REALPART_EXPR:
7842 1794 : case IMAGPART_EXPR:
7843 1794 : {
7844 1794 : gimple *ds = SSA_NAME_DEF_STMT (s);
7845 1794 : tree rhs1 = gimple_assign_rhs1 (ds);
7846 1794 : rhs1 = TREE_OPERAND (rhs1, 0);
7847 1794 : if (TREE_CODE (rhs1) == SSA_NAME)
7848 : {
7849 1794 : gimple *g = SSA_NAME_DEF_STMT (rhs1);
7850 1794 : if (optimizable_arith_overflow (g))
7851 : {
7852 1650 : if (gimple_assign_rhs_code (ds) == IMAGPART_EXPR)
7853 1640 : continue;
7854 10 : if (gimple_store_p (use_stmt))
7855 : {
7856 : /* Punt if the cast use of IMAGPART_EXPR stmt
7857 : appears before the store use_stmt, because
7858 : optimizable arith overflow can't be
7859 : lowered at the store location in that case.
7860 : See PR121828. */
7861 10 : gimple_stmt_iterator gsi
7862 10 : = gsi_for_stmt (use_stmt);
7863 10 : unsigned int cnt = 0;
7864 12 : do
7865 : {
7866 12 : gsi_prev_nondebug (&gsi);
7867 12 : if (gsi_end_p (gsi))
7868 : break;
7869 12 : gimple *g2 = gsi_stmt (gsi);
7870 12 : if (g2 == ds)
7871 : break;
7872 3 : if (++cnt == 64)
7873 : break;
7874 3 : if (!gimple_assign_cast_p (g2))
7875 2 : continue;
7876 1 : tree rhs2 = gimple_assign_rhs1 (g2);
7877 1 : if (TREE_CODE (rhs2) != SSA_NAME)
7878 0 : continue;
7879 1 : gimple *g3 = SSA_NAME_DEF_STMT (rhs2);
7880 1 : if (!is_gimple_assign (g3))
7881 0 : continue;
7882 1 : if (gimple_assign_rhs_code (g3)
7883 : != IMAGPART_EXPR)
7884 0 : continue;
7885 1 : rhs2 = gimple_assign_rhs1 (g3);
7886 1 : rhs2 = TREE_OPERAND (rhs2, 0);
7887 1 : if (rhs2 != rhs1)
7888 0 : continue;
7889 : cnt = 64;
7890 : break;
7891 : }
7892 : while (1);
7893 10 : if (cnt == 64)
7894 : break;
7895 : }
7896 : }
7897 : }
7898 : }
7899 : /* FALLTHRU */
7900 844 : case LSHIFT_EXPR:
7901 844 : case RSHIFT_EXPR:
7902 844 : case MULT_EXPR:
7903 844 : case TRUNC_DIV_EXPR:
7904 844 : case EXACT_DIV_EXPR:
7905 844 : case TRUNC_MOD_EXPR:
7906 844 : case FIX_TRUNC_EXPR:
7907 844 : if (gimple_store_p (use_stmt)
7908 444 : && is_gimple_assign (use_stmt)
7909 444 : && !gimple_has_volatile_ops (use_stmt)
7910 1288 : && !stmt_ends_bb_p (use_stmt))
7911 : {
7912 444 : tree lhs = gimple_assign_lhs (use_stmt);
7913 : /* As multiply/division passes address of the lhs
7914 : to library function and that assumes it can extend
7915 : it to whole number of limbs, avoid merging those
7916 : with bit-field stores. Don't allow it for
7917 : shifts etc. either, so that the bit-field store
7918 : handling doesn't have to be done everywhere. */
7919 444 : if (TREE_CODE (lhs) == COMPONENT_REF
7920 444 : && DECL_BIT_FIELD_TYPE (TREE_OPERAND (lhs, 1)))
7921 : break;
7922 441 : continue;
7923 441 : }
7924 : break;
7925 : default:
7926 : break;
7927 : }
7928 2110 : else if (is_gimple_call (SSA_NAME_DEF_STMT (s))
7929 2110 : && gimple_call_internal_p (SSA_NAME_DEF_STMT (s)))
7930 15 : switch (gimple_call_internal_fn (SSA_NAME_DEF_STMT (s)))
7931 : {
7932 13 : case IFN_BSWAP:
7933 13 : case IFN_BITREVERSE:
7934 13 : if (gimple_store_p (use_stmt)
7935 13 : && is_gimple_assign (use_stmt)
7936 13 : && !gimple_has_volatile_ops (use_stmt)
7937 26 : && !stmt_ends_bb_p (use_stmt))
7938 : {
7939 13 : tree lhs = gimple_assign_lhs (use_stmt);
7940 13 : if (TREE_CODE (lhs) == COMPONENT_REF
7941 13 : && DECL_BIT_FIELD_TYPE (TREE_OPERAND (lhs, 1)))
7942 : break;
7943 13 : continue;
7944 13 : }
7945 : break;
7946 : default:
7947 : break;
7948 : }
7949 : }
7950 :
7951 : /* Also ignore uninitialized uses. */
7952 29600 : if (SSA_NAME_IS_DEFAULT_DEF (s)
7953 29600 : && (!SSA_NAME_VAR (s) || VAR_P (SSA_NAME_VAR (s))))
7954 59 : continue;
7955 :
7956 29592 : force_name:
7957 29592 : if (!large_huge.m_names)
7958 5691 : large_huge.m_names = BITMAP_ALLOC (NULL);
7959 29592 : bitmap_set_bit (large_huge.m_names, SSA_NAME_VERSION (s));
7960 29592 : if (has_single_use (s))
7961 : {
7962 : tree s2 = s;
7963 : /* The coalescing hook special cases SSA_NAME copies.
7964 : Make sure not to mark in m_single_use_names single
7965 : use SSA_NAMEs copied from non-single use SSA_NAMEs. */
7966 25889 : while (gimple_assign_copy_p (SSA_NAME_DEF_STMT (s2)))
7967 : {
7968 938 : s2 = gimple_assign_rhs1 (SSA_NAME_DEF_STMT (s2));
7969 938 : if (TREE_CODE (s2) != SSA_NAME)
7970 : break;
7971 41 : if (!has_single_use (s2))
7972 : {
7973 : s2 = NULL_TREE;
7974 : break;
7975 : }
7976 : }
7977 25876 : if (s2)
7978 : {
7979 25848 : if (!large_huge.m_single_use_names)
7980 5577 : large_huge.m_single_use_names = BITMAP_ALLOC (NULL);
7981 25848 : bitmap_set_bit (large_huge.m_single_use_names,
7982 25848 : SSA_NAME_VERSION (s));
7983 : }
7984 : }
7985 29592 : if (SSA_NAME_VAR (s)
7986 7370 : && ((TREE_CODE (SSA_NAME_VAR (s)) == PARM_DECL
7987 5420 : && SSA_NAME_IS_DEFAULT_DEF (s))
7988 1987 : || TREE_CODE (SSA_NAME_VAR (s)) == RESULT_DECL))
7989 : has_large_huge_parm_result = true;
7990 29592 : if (optimize
7991 10474 : && !SSA_NAME_OCCURS_IN_ABNORMAL_PHI (s)
7992 10407 : && gimple_assign_load_p (SSA_NAME_DEF_STMT (s))
7993 6074 : && !gimple_has_volatile_ops (SSA_NAME_DEF_STMT (s))
7994 32580 : && !stmt_ends_bb_p (SSA_NAME_DEF_STMT (s)))
7995 : {
7996 2988 : use_operand_p use_p;
7997 2988 : imm_use_iterator iter;
7998 2988 : bool optimizable_load = true;
7999 6041 : FOR_EACH_IMM_USE_FAST (use_p, iter, s)
8000 : {
8001 3151 : gimple *use_stmt = USE_STMT (use_p);
8002 3151 : if (is_gimple_debug (use_stmt))
8003 0 : continue;
8004 3151 : if (is_gimple_call (use_stmt)
8005 3151 : && gimple_call_internal_p (use_stmt))
8006 66 : switch (gimple_call_internal_fn (use_stmt))
8007 : {
8008 10 : case IFN_CLZ:
8009 10 : case IFN_CTZ:
8010 10 : case IFN_CLRSB:
8011 10 : case IFN_FFS:
8012 10 : case IFN_PARITY:
8013 10 : case IFN_POPCOUNT:
8014 10 : case IFN_BSWAP:
8015 10 : case IFN_BITREVERSE:
8016 10 : continue;
8017 : default:
8018 : break;
8019 : }
8020 3141 : if (gimple_code (use_stmt) == GIMPLE_PHI
8021 : || is_gimple_call (use_stmt)
8022 : || gimple_code (use_stmt) == GIMPLE_ASM
8023 : || (is_gimple_assign (use_stmt)
8024 1907 : && (gimple_assign_rhs_code (use_stmt)
8025 : == COMPLEX_EXPR)))
8026 : {
8027 : optimizable_load = false;
8028 : break;
8029 : }
8030 2988 : }
8031 :
8032 2988 : ssa_op_iter oi;
8033 4037 : FOR_EACH_SSA_USE_OPERAND (use_p, SSA_NAME_DEF_STMT (s),
8034 : oi, SSA_OP_USE)
8035 : {
8036 1049 : tree s2 = USE_FROM_PTR (use_p);
8037 1049 : if (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (s2))
8038 : {
8039 : optimizable_load = false;
8040 : break;
8041 : }
8042 : }
8043 :
8044 2988 : if (optimizable_load && !stmt_ends_bb_p (SSA_NAME_DEF_STMT (s)))
8045 : {
8046 2890 : if (!large_huge.m_loads)
8047 315 : large_huge.m_loads = BITMAP_ALLOC (NULL);
8048 2890 : bitmap_set_bit (large_huge.m_loads, SSA_NAME_VERSION (s));
8049 : }
8050 : }
8051 : }
8052 : /* We need to also rewrite stores of large/huge _BitInt INTEGER_CSTs
8053 : into memory. Such functions could have no large/huge SSA_NAMEs. */
8054 57229 : else if (SSA_NAME_IS_VIRTUAL_OPERAND (s))
8055 : {
8056 40419 : gimple *g = SSA_NAME_DEF_STMT (s);
8057 40419 : if (is_gimple_assign (g) && gimple_store_p (g))
8058 : {
8059 13741 : tree t = gimple_assign_rhs1 (g);
8060 15620 : if (BITINT_TYPE_P (TREE_TYPE (t))
8061 13741 : && bitint_precision_kind (TREE_TYPE (t)) >= bitint_prec_large)
8062 : has_large_huge = true;
8063 : }
8064 : }
8065 : }
8066 :
8067 7366 : if (large_huge.m_names || has_large_huge)
8068 : {
8069 5889 : ret = TODO_update_ssa_only_virtuals | TODO_cleanup_cfg;
8070 5889 : calculate_dominance_info (CDI_DOMINATORS);
8071 5889 : if (optimize)
8072 3105 : enable_ranger (cfun);
8073 5889 : if (large_huge.m_loads)
8074 : {
8075 315 : basic_block entry = ENTRY_BLOCK_PTR_FOR_FN (cfun);
8076 315 : entry->aux = NULL;
8077 630 : bitint_dom_walker (large_huge.m_names,
8078 315 : large_huge.m_loads).walk (entry);
8079 315 : bitmap_and_compl_into (large_huge.m_names, large_huge.m_loads);
8080 315 : clear_aux_for_blocks ();
8081 315 : BITMAP_FREE (large_huge.m_loads);
8082 : }
8083 5889 : large_huge.m_limb_type = build_nonstandard_integer_type (limb_prec, 1);
8084 5889 : large_huge.m_limb_size
8085 5889 : = tree_to_uhwi (TYPE_SIZE_UNIT (large_huge.m_limb_type));
8086 : }
8087 7366 : if (large_huge.m_names)
8088 : {
8089 5691 : large_huge.m_map
8090 11382 : = init_var_map (num_ssa_names, NULL, large_huge.m_names);
8091 5691 : coalesce_ssa_name (large_huge.m_map);
8092 5691 : partition_view_normal (large_huge.m_map);
8093 5691 : if (dump_file && (dump_flags & TDF_DETAILS))
8094 : {
8095 0 : fprintf (dump_file, "After Coalescing:\n");
8096 0 : dump_var_map (dump_file, large_huge.m_map);
8097 : }
8098 5691 : large_huge.m_vars
8099 5691 : = XCNEWVEC (tree, num_var_partitions (large_huge.m_map));
8100 5691 : bitmap_iterator bi;
8101 5691 : if (has_large_huge_parm_result)
8102 19742 : EXECUTE_IF_SET_IN_BITMAP (large_huge.m_names, 0, i, bi)
8103 : {
8104 15530 : tree s = ssa_name (i);
8105 15530 : if (SSA_NAME_VAR (s)
8106 6224 : && ((TREE_CODE (SSA_NAME_VAR (s)) == PARM_DECL
8107 5420 : && SSA_NAME_IS_DEFAULT_DEF (s))
8108 841 : || TREE_CODE (SSA_NAME_VAR (s)) == RESULT_DECL))
8109 : {
8110 5383 : int p = var_to_partition (large_huge.m_map, s);
8111 5383 : if (large_huge.m_vars[p] == NULL_TREE)
8112 : {
8113 5383 : large_huge.m_vars[p] = SSA_NAME_VAR (s);
8114 5383 : mark_addressable (SSA_NAME_VAR (s));
8115 : }
8116 : }
8117 : }
8118 5691 : tree atype = NULL_TREE;
8119 5691 : if (dump_file && (dump_flags & TDF_DETAILS))
8120 0 : fprintf (dump_file, "Mapping SSA_NAMEs to decls:\n");
8121 32927 : EXECUTE_IF_SET_IN_BITMAP (large_huge.m_names, 0, i, bi)
8122 : {
8123 27236 : tree s = ssa_name (i);
8124 27236 : int p = var_to_partition (large_huge.m_map, s);
8125 27236 : if (large_huge.m_vars[p] == NULL_TREE)
8126 : {
8127 18927 : if (atype == NULL_TREE
8128 33060 : || !tree_int_cst_equal (TYPE_SIZE (atype),
8129 14133 : TYPE_SIZE (TREE_TYPE (s))))
8130 : {
8131 7771 : unsigned HOST_WIDE_INT nelts
8132 7771 : = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (s))) / limb_prec;
8133 7771 : atype = build_array_type_nelts (large_huge.m_limb_type,
8134 7771 : nelts);
8135 : }
8136 18927 : large_huge.m_vars[p] = create_tmp_var (atype, "bitint");
8137 18927 : mark_addressable (large_huge.m_vars[p]);
8138 : }
8139 27236 : if (dump_file && (dump_flags & TDF_DETAILS))
8140 : {
8141 0 : print_generic_expr (dump_file, s, TDF_SLIM);
8142 0 : fprintf (dump_file, " -> ");
8143 0 : print_generic_expr (dump_file, large_huge.m_vars[p], TDF_SLIM);
8144 0 : fprintf (dump_file, "\n");
8145 : }
8146 : }
8147 : }
8148 :
8149 46528 : FOR_EACH_BB_REVERSE_FN (bb, cfun)
8150 : {
8151 39162 : gimple_stmt_iterator prev;
8152 198150 : for (gimple_stmt_iterator gsi = gsi_last_bb (bb); !gsi_end_p (gsi);
8153 119826 : gsi = prev)
8154 : {
8155 119826 : prev = gsi;
8156 119826 : gsi_prev (&prev);
8157 119826 : ssa_op_iter iter;
8158 119826 : gimple *stmt = gsi_stmt (gsi);
8159 119826 : if (is_gimple_debug (stmt))
8160 76923 : continue;
8161 114843 : bitint_prec_kind kind = bitint_prec_small;
8162 114843 : tree t;
8163 345555 : FOR_EACH_SSA_TREE_OPERAND (t, stmt, iter, SSA_OP_ALL_OPERANDS)
8164 230712 : if (BITINT_TYPE_P (TREE_TYPE (t)))
8165 : {
8166 79432 : bitint_prec_kind this_kind
8167 79432 : = bitint_precision_kind (TREE_TYPE (t));
8168 230930 : kind = MAX (kind, this_kind);
8169 : }
8170 114843 : if (is_gimple_assign (stmt) && gimple_store_p (stmt))
8171 : {
8172 16418 : t = gimple_assign_rhs1 (stmt);
8173 16418 : if (BITINT_TYPE_P (TREE_TYPE (t)))
8174 : {
8175 14217 : bitint_prec_kind this_kind
8176 14217 : = bitint_precision_kind (TREE_TYPE (t));
8177 14217 : kind = MAX (kind, this_kind);
8178 : }
8179 : }
8180 114843 : if (is_gimple_assign (stmt)
8181 114843 : && gimple_assign_rhs_code (stmt) == FLOAT_EXPR)
8182 : {
8183 189 : t = gimple_assign_rhs1 (stmt);
8184 218 : if (BITINT_TYPE_P (TREE_TYPE (t))
8185 191 : && TREE_CODE (t) == INTEGER_CST)
8186 : {
8187 1 : bitint_prec_kind this_kind
8188 1 : = bitint_precision_kind (TREE_TYPE (t));
8189 1 : kind = MAX (kind, this_kind);
8190 : }
8191 : }
8192 114843 : if (is_gimple_call (stmt))
8193 : {
8194 25873 : t = gimple_call_lhs (stmt);
8195 25873 : if (t && TREE_CODE (TREE_TYPE (t)) == COMPLEX_TYPE)
8196 : {
8197 5325 : bitint_prec_kind this_kind = arith_overflow_arg_kind (stmt);
8198 5325 : kind = MAX (kind, this_kind);
8199 5325 : if (BITINT_TYPE_P (TREE_TYPE (TREE_TYPE (t))))
8200 : {
8201 5200 : this_kind
8202 5200 : = bitint_precision_kind (TREE_TYPE (TREE_TYPE (t)));
8203 5200 : kind = MAX (kind, this_kind);
8204 : }
8205 : }
8206 : }
8207 114521 : if (kind == bitint_prec_small)
8208 45199 : continue;
8209 69644 : switch (gimple_code (stmt))
8210 : {
8211 11139 : case GIMPLE_CALL:
8212 : /* For now. We'll need to handle some internal functions and
8213 : perhaps some builtins. */
8214 11139 : if (kind == bitint_prec_middle)
8215 2282 : continue;
8216 : break;
8217 4 : case GIMPLE_ASM:
8218 4 : if (kind == bitint_prec_middle)
8219 1 : continue;
8220 : break;
8221 1125 : case GIMPLE_RETURN:
8222 1125 : continue;
8223 48746 : case GIMPLE_ASSIGN:
8224 48746 : if (gimple_clobber_p (stmt))
8225 3511 : continue;
8226 45235 : if (kind >= bitint_prec_large)
8227 : break;
8228 8818 : if (gimple_assign_single_p (stmt))
8229 : /* No need to lower copies, loads or stores. */
8230 5809 : continue;
8231 3009 : if (gimple_assign_cast_p (stmt))
8232 : {
8233 2432 : tree lhs = gimple_assign_lhs (stmt);
8234 2432 : tree rhs = gimple_assign_rhs1 (stmt);
8235 4864 : if (INTEGRAL_TYPE_P (TREE_TYPE (lhs))
8236 2432 : && INTEGRAL_TYPE_P (TREE_TYPE (rhs))
8237 4858 : && (TYPE_PRECISION (TREE_TYPE (lhs))
8238 2426 : == TYPE_PRECISION (TREE_TYPE (rhs))))
8239 : /* No need to lower casts to same precision. */
8240 35 : continue;
8241 : }
8242 : break;
8243 : default:
8244 : break;
8245 1125 : }
8246 :
8247 11604 : if (kind == bitint_prec_middle)
8248 : {
8249 5104 : tree type = NULL_TREE;
8250 : /* Middle _BitInt(N) is rewritten to casts to INTEGER_TYPEs
8251 : with the same precision and back. */
8252 5104 : unsigned int nops = gimple_num_ops (stmt);
8253 17082 : for (unsigned int i = is_gimple_assign (stmt) ? 1 : 0;
8254 17082 : i < nops; ++i)
8255 11978 : if (tree op = gimple_op (stmt, i))
8256 : {
8257 7728 : tree nop = maybe_cast_middle_bitint (&gsi, op, type);
8258 7728 : if (nop != op)
8259 6829 : gimple_set_op (stmt, i, nop);
8260 899 : else if (COMPARISON_CLASS_P (op))
8261 : {
8262 0 : TREE_OPERAND (op, 0)
8263 0 : = maybe_cast_middle_bitint (&gsi,
8264 0 : TREE_OPERAND (op, 0),
8265 : type);
8266 0 : TREE_OPERAND (op, 1)
8267 0 : = maybe_cast_middle_bitint (&gsi,
8268 0 : TREE_OPERAND (op, 1),
8269 : type);
8270 : }
8271 899 : else if (TREE_CODE (op) == CASE_LABEL_EXPR)
8272 : {
8273 28 : CASE_LOW (op)
8274 28 : = maybe_cast_middle_bitint (&gsi, CASE_LOW (op),
8275 : type);
8276 56 : CASE_HIGH (op)
8277 56 : = maybe_cast_middle_bitint (&gsi, CASE_HIGH (op),
8278 : type);
8279 : }
8280 : }
8281 5104 : if (tree lhs = gimple_get_lhs (stmt))
8282 4538 : if (BITINT_TYPE_P (TREE_TYPE (lhs))
8283 2974 : && (bitint_precision_kind (TREE_TYPE (lhs))
8284 : == bitint_prec_middle))
8285 : {
8286 1399 : int prec = TYPE_PRECISION (TREE_TYPE (lhs));
8287 1399 : int uns = TYPE_UNSIGNED (TREE_TYPE (lhs));
8288 1399 : type = build_nonstandard_integer_type (prec, uns);
8289 1399 : tree lhs2 = make_ssa_name (type);
8290 1399 : gimple_set_lhs (stmt, lhs2);
8291 1399 : gimple *g = gimple_build_assign (lhs, NOP_EXPR, lhs2);
8292 1399 : if (stmt_ends_bb_p (stmt))
8293 : {
8294 4 : edge e = find_fallthru_edge (gsi_bb (gsi)->succs);
8295 4 : gsi_insert_on_edge (e, g);
8296 4 : edge_insertions = true;
8297 : }
8298 : else
8299 1395 : gsi_insert_after (&gsi, g, GSI_SAME_STMT);
8300 : }
8301 5104 : update_stmt (stmt);
8302 5104 : continue;
8303 5104 : }
8304 :
8305 51777 : if (tree lhs = gimple_get_lhs (stmt))
8306 45143 : if (TREE_CODE (lhs) == SSA_NAME)
8307 : {
8308 36134 : tree type = TREE_TYPE (lhs);
8309 36134 : if (TREE_CODE (type) == COMPLEX_TYPE)
8310 4197 : type = TREE_TYPE (type);
8311 5444 : if (BITINT_TYPE_P (type)
8312 30702 : && bitint_precision_kind (type) >= bitint_prec_large
8313 66634 : && (large_huge.m_names == NULL
8314 30295 : || !bitmap_bit_p (large_huge.m_names,
8315 30295 : SSA_NAME_VERSION (lhs))))
8316 8874 : continue;
8317 : }
8318 :
8319 42903 : large_huge.lower_stmt (stmt);
8320 : }
8321 :
8322 39162 : tree atype = NULL_TREE;
8323 47817 : for (gphi_iterator gsi = gsi_start_phis (bb); !gsi_end_p (gsi);
8324 8655 : gsi_next (&gsi))
8325 : {
8326 8655 : gphi *phi = gsi.phi ();
8327 8655 : tree lhs = gimple_phi_result (phi);
8328 17070 : if (!BITINT_TYPE_P (TREE_TYPE (lhs))
8329 8655 : || bitint_precision_kind (TREE_TYPE (lhs)) < bitint_prec_large)
8330 8462 : continue;
8331 193 : int p1 = var_to_partition (large_huge.m_map, lhs);
8332 193 : gcc_assert (large_huge.m_vars[p1] != NULL_TREE);
8333 : tree v1 = large_huge.m_vars[p1];
8334 694 : for (unsigned i = 0; i < gimple_phi_num_args (phi); ++i)
8335 : {
8336 501 : tree arg = gimple_phi_arg_def (phi, i);
8337 501 : edge e = gimple_phi_arg_edge (phi, i);
8338 501 : gimple *g;
8339 501 : switch (TREE_CODE (arg))
8340 : {
8341 99 : case INTEGER_CST:
8342 99 : if (integer_zerop (arg) && VAR_P (v1))
8343 : {
8344 45 : tree zero = build_zero_cst (TREE_TYPE (v1));
8345 45 : g = gimple_build_assign (v1, zero);
8346 45 : gsi_insert_on_edge (e, g);
8347 45 : edge_insertions = true;
8348 162 : break;
8349 : }
8350 54 : int ext;
8351 54 : unsigned int min_prec, prec, rem;
8352 54 : tree c;
8353 54 : prec = TYPE_PRECISION (TREE_TYPE (arg));
8354 54 : rem = prec % (2 * limb_prec);
8355 54 : min_prec = bitint_min_cst_precision (arg, ext);
8356 54 : if (min_prec > prec - rem - 2 * limb_prec
8357 12 : && min_prec > (unsigned) limb_prec)
8358 : /* Constant which has enough significant bits that it
8359 : isn't worth trying to save .rodata space by extending
8360 : from smaller number. */
8361 : min_prec = prec;
8362 : else
8363 : {
8364 45 : min_prec = CEIL (min_prec, limb_prec) * limb_prec;
8365 45 : if (min_prec > (unsigned) limb_prec
8366 3 : && abi_limb_prec > limb_prec)
8367 : {
8368 : /* For targets with ABI limb precision higher than
8369 : limb precision round to ABI limb precision,
8370 : otherwise c can contain padding bits. */
8371 0 : min_prec
8372 0 : = CEIL (min_prec, abi_limb_prec) * abi_limb_prec;
8373 0 : if (min_prec > prec - rem - 2 * limb_prec)
8374 9 : min_prec = prec;
8375 : }
8376 : }
8377 54 : if (min_prec == 0)
8378 : c = NULL_TREE;
8379 51 : else if (min_prec == prec)
8380 9 : c = tree_output_constant_def (arg);
8381 42 : else if (min_prec == (unsigned) limb_prec)
8382 39 : c = fold_convert (large_huge.m_limb_type, arg);
8383 : else
8384 : {
8385 3 : tree ctype = build_bitint_type (min_prec, 1);
8386 3 : c = tree_output_constant_def (fold_convert (ctype, arg));
8387 : }
8388 51 : if (c)
8389 : {
8390 51 : if (VAR_P (v1) && min_prec == prec)
8391 : {
8392 8 : tree v2 = build1 (VIEW_CONVERT_EXPR,
8393 8 : TREE_TYPE (v1), c);
8394 8 : g = gimple_build_assign (v1, v2);
8395 8 : gsi_insert_on_edge (e, g);
8396 8 : edge_insertions = true;
8397 8 : break;
8398 : }
8399 43 : if (TREE_CODE (TREE_TYPE (c)) == INTEGER_TYPE)
8400 : {
8401 39 : if (bitint_big_endian)
8402 : {
8403 0 : tree ptype = build_pointer_type (TREE_TYPE (v1));
8404 0 : tree sz1 = TYPE_SIZE_UNIT (TREE_TYPE (v1));
8405 0 : tree sz2 = TYPE_SIZE_UNIT (TREE_TYPE (c));
8406 0 : tree off = build_int_cst (ptype,
8407 0 : tree_to_uhwi (sz1)
8408 0 : - tree_to_uhwi (sz2));
8409 0 : tree vd = build2 (MEM_REF, TREE_TYPE (c),
8410 : build_fold_addr_expr (v1),
8411 : off);
8412 0 : g = gimple_build_assign (vd, c);
8413 : }
8414 : else
8415 39 : g = gimple_build_assign (build1 (VIEW_CONVERT_EXPR,
8416 39 : TREE_TYPE (c),
8417 : v1), c);
8418 : }
8419 : else
8420 : {
8421 4 : unsigned HOST_WIDE_INT nelts
8422 4 : = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (c)))
8423 4 : / limb_prec;
8424 4 : tree vtype
8425 8 : = build_array_type_nelts (large_huge.m_limb_type,
8426 4 : nelts);
8427 4 : tree vd;
8428 4 : if (bitint_big_endian)
8429 : {
8430 0 : tree ptype = build_pointer_type (TREE_TYPE (v1));
8431 0 : tree sz1 = TYPE_SIZE_UNIT (TREE_TYPE (v1));
8432 0 : tree sz2 = TYPE_SIZE_UNIT (vtype);
8433 0 : tree off = build_int_cst (ptype,
8434 0 : tree_to_uhwi (sz1)
8435 0 : - tree_to_uhwi (sz2));
8436 0 : vd = build2 (MEM_REF, vtype,
8437 : build_fold_addr_expr (v1), off);
8438 : }
8439 : else
8440 4 : vd = build1 (VIEW_CONVERT_EXPR, vtype, v1);
8441 4 : g = gimple_build_assign (vd,
8442 : build1 (VIEW_CONVERT_EXPR,
8443 : vtype, c));
8444 : }
8445 43 : gsi_insert_on_edge (e, g);
8446 43 : if (min_prec == prec)
8447 : {
8448 : edge_insertions = true;
8449 : break;
8450 : }
8451 : }
8452 45 : if (ext == 0)
8453 : {
8454 39 : unsigned HOST_WIDE_INT nelts
8455 39 : = (tree_to_uhwi (TYPE_SIZE (TREE_TYPE (v1)))
8456 39 : - min_prec) / limb_prec;
8457 39 : tree vtype
8458 78 : = build_array_type_nelts (large_huge.m_limb_type,
8459 39 : nelts);
8460 39 : tree ptype = build_pointer_type (TREE_TYPE (v1));
8461 39 : tree off;
8462 39 : if (c && !bitint_big_endian)
8463 38 : off = fold_convert (ptype,
8464 : TYPE_SIZE_UNIT (TREE_TYPE (c)));
8465 : else
8466 1 : off = build_zero_cst (ptype);
8467 39 : tree vd = build2 (MEM_REF, vtype,
8468 : build_fold_addr_expr (v1), off);
8469 39 : g = gimple_build_assign (vd, build_zero_cst (vtype));
8470 : }
8471 : else
8472 : {
8473 6 : tree vd = v1;
8474 6 : if (c && !bitint_big_endian)
8475 : {
8476 4 : tree ptype = build_pointer_type (TREE_TYPE (v1));
8477 4 : tree off
8478 4 : = fold_convert (ptype,
8479 : TYPE_SIZE_UNIT (TREE_TYPE (c)));
8480 4 : vd = build2 (MEM_REF, large_huge.m_limb_type,
8481 : build_fold_addr_expr (v1), off);
8482 : }
8483 6 : vd = build_fold_addr_expr (vd);
8484 6 : unsigned HOST_WIDE_INT nbytes
8485 6 : = tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (v1)));
8486 6 : if (c)
8487 4 : nbytes
8488 4 : -= tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (c)));
8489 6 : tree fn = builtin_decl_implicit (BUILT_IN_MEMSET);
8490 6 : g = gimple_build_call (fn, 3, vd,
8491 : integer_minus_one_node,
8492 : build_int_cst (sizetype,
8493 6 : nbytes));
8494 : }
8495 45 : gsi_insert_on_edge (e, g);
8496 45 : edge_insertions = true;
8497 45 : break;
8498 0 : default:
8499 0 : gcc_unreachable ();
8500 402 : case SSA_NAME:
8501 402 : if (gimple_code (SSA_NAME_DEF_STMT (arg)) == GIMPLE_NOP)
8502 : {
8503 9 : if (large_huge.m_names == NULL
8504 18 : || !bitmap_bit_p (large_huge.m_names,
8505 9 : SSA_NAME_VERSION (arg)))
8506 384 : continue;
8507 : }
8508 402 : int p2 = var_to_partition (large_huge.m_map, arg);
8509 402 : if (p1 == p2)
8510 384 : continue;
8511 18 : gcc_assert (large_huge.m_vars[p2] != NULL_TREE);
8512 18 : tree v2 = large_huge.m_vars[p2];
8513 18 : if (VAR_P (v1) && VAR_P (v2))
8514 18 : g = gimple_build_assign (v1, v2);
8515 0 : else if (VAR_P (v1))
8516 0 : g = gimple_build_assign (v1, build1 (VIEW_CONVERT_EXPR,
8517 0 : TREE_TYPE (v1), v2));
8518 0 : else if (VAR_P (v2))
8519 0 : g = gimple_build_assign (build1 (VIEW_CONVERT_EXPR,
8520 0 : TREE_TYPE (v2), v1), v2);
8521 : else
8522 : {
8523 0 : if (atype == NULL_TREE
8524 0 : || !tree_int_cst_equal (TYPE_SIZE (atype),
8525 0 : TYPE_SIZE (TREE_TYPE (lhs))))
8526 : {
8527 0 : unsigned HOST_WIDE_INT nelts
8528 0 : = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (lhs)))
8529 0 : / limb_prec;
8530 0 : atype
8531 0 : = build_array_type_nelts (large_huge.m_limb_type,
8532 0 : nelts);
8533 : }
8534 0 : g = gimple_build_assign (build1 (VIEW_CONVERT_EXPR,
8535 : atype, v1),
8536 : build1 (VIEW_CONVERT_EXPR,
8537 : atype, v2));
8538 : }
8539 18 : gsi_insert_on_edge (e, g);
8540 18 : edge_insertions = true;
8541 18 : break;
8542 : }
8543 : }
8544 : }
8545 : }
8546 :
8547 7366 : if (large_huge.m_names || has_large_huge)
8548 : {
8549 5889 : gimple *nop = NULL;
8550 387033 : for (i = 0; i < num_ssa_names; ++i)
8551 : {
8552 381144 : tree s = ssa_name (i);
8553 381144 : if (s == NULL_TREE)
8554 15539 : continue;
8555 365605 : tree type = TREE_TYPE (s);
8556 365605 : if (TREE_CODE (type) == COMPLEX_TYPE)
8557 17612 : type = TREE_TYPE (type);
8558 325695 : if (BITINT_TYPE_P (type)
8559 365632 : && bitint_precision_kind (type) >= bitint_prec_large)
8560 : {
8561 40855 : if (large_huge.m_preserved
8562 45353 : && bitmap_bit_p (large_huge.m_preserved,
8563 6841 : SSA_NAME_VERSION (s)))
8564 2343 : continue;
8565 36169 : gimple *g = SSA_NAME_DEF_STMT (s);
8566 36169 : if (gimple_code (g) == GIMPLE_NOP)
8567 : {
8568 9736 : if (SSA_NAME_VAR (s))
8569 5452 : set_ssa_default_def (cfun, SSA_NAME_VAR (s), NULL_TREE);
8570 9736 : release_ssa_name (s);
8571 9736 : continue;
8572 : }
8573 26433 : if (gimple_bb (g) == NULL)
8574 : {
8575 0 : release_ssa_name (s);
8576 0 : continue;
8577 : }
8578 26433 : if (gimple_code (g) != GIMPLE_ASM)
8579 : {
8580 26432 : gimple_stmt_iterator gsi = gsi_for_stmt (g);
8581 26432 : bool save_vta = flag_var_tracking_assignments;
8582 26432 : flag_var_tracking_assignments = false;
8583 26432 : gsi_remove (&gsi, true);
8584 26432 : flag_var_tracking_assignments = save_vta;
8585 : }
8586 26433 : if (nop == NULL)
8587 4920 : nop = gimple_build_nop ();
8588 26433 : SSA_NAME_DEF_STMT (s) = nop;
8589 26433 : release_ssa_name (s);
8590 : }
8591 : }
8592 5889 : if (optimize)
8593 3105 : disable_ranger (cfun);
8594 : }
8595 :
8596 7366 : if (edge_insertions)
8597 56 : gsi_commit_edge_inserts ();
8598 :
8599 : /* Fix up arguments of ECF_RETURNS_TWICE calls. Those were temporarily
8600 : inserted before the call, but that is invalid IL, so move them to the
8601 : right place and add corresponding PHIs. */
8602 7366 : if (!large_huge.m_returns_twice_calls.is_empty ())
8603 : {
8604 9 : auto_vec<gimple *, 16> arg_stmts;
8605 29 : while (!large_huge.m_returns_twice_calls.is_empty ())
8606 : {
8607 11 : gimple *stmt = large_huge.m_returns_twice_calls.pop ();
8608 11 : gimple_stmt_iterator gsi = gsi_after_labels (gimple_bb (stmt));
8609 36 : while (gsi_stmt (gsi) != stmt)
8610 : {
8611 25 : if (is_gimple_debug (gsi_stmt (gsi)))
8612 2 : gsi_next (&gsi);
8613 : else
8614 : {
8615 23 : arg_stmts.safe_push (gsi_stmt (gsi));
8616 23 : gsi_remove (&gsi, false);
8617 : }
8618 : }
8619 11 : gimple *g;
8620 11 : basic_block bb = NULL;
8621 11 : edge e = NULL, ead = NULL;
8622 34 : FOR_EACH_VEC_ELT (arg_stmts, i, g)
8623 : {
8624 23 : gsi_safe_insert_before (&gsi, g);
8625 23 : if (i == 0)
8626 : {
8627 11 : bb = gimple_bb (stmt);
8628 11 : gcc_checking_assert (EDGE_COUNT (bb->preds) == 2);
8629 11 : e = EDGE_PRED (bb, 0);
8630 11 : ead = EDGE_PRED (bb, 1);
8631 11 : if ((ead->flags & EDGE_ABNORMAL) == 0)
8632 0 : std::swap (e, ead);
8633 11 : gcc_checking_assert ((e->flags & EDGE_ABNORMAL) == 0
8634 : && (ead->flags & EDGE_ABNORMAL));
8635 : }
8636 23 : tree lhs = gimple_assign_lhs (g);
8637 23 : tree arg = lhs;
8638 23 : gphi *phi = create_phi_node (copy_ssa_name (arg), bb);
8639 23 : add_phi_arg (phi, arg, e, UNKNOWN_LOCATION);
8640 23 : tree var = create_tmp_reg (TREE_TYPE (arg));
8641 23 : suppress_warning (var, OPT_Wuninitialized);
8642 23 : arg = get_or_create_ssa_default_def (cfun, var);
8643 23 : SSA_NAME_OCCURS_IN_ABNORMAL_PHI (arg) = 1;
8644 23 : add_phi_arg (phi, arg, ead, UNKNOWN_LOCATION);
8645 23 : arg = gimple_phi_result (phi);
8646 23 : SSA_NAME_OCCURS_IN_ABNORMAL_PHI (arg) = 1;
8647 23 : imm_use_iterator iter;
8648 23 : gimple *use_stmt;
8649 69 : FOR_EACH_IMM_USE_STMT (use_stmt, iter, lhs)
8650 : {
8651 46 : if (use_stmt == phi)
8652 23 : continue;
8653 23 : gcc_checking_assert (use_stmt == stmt);
8654 23 : use_operand_p use_p;
8655 46 : FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
8656 23 : SET_USE (use_p, arg);
8657 23 : }
8658 : }
8659 11 : update_stmt (stmt);
8660 11 : arg_stmts.truncate (0);
8661 : }
8662 9 : }
8663 :
8664 7366 : return ret;
8665 7366 : }
8666 :
8667 : namespace {
8668 :
8669 : const pass_data pass_data_lower_bitint =
8670 : {
8671 : GIMPLE_PASS, /* type */
8672 : "bitintlower", /* name */
8673 : OPTGROUP_NONE, /* optinfo_flags */
8674 : TV_NONE, /* tv_id */
8675 : PROP_ssa, /* properties_required */
8676 : PROP_gimple_lbitint, /* properties_provided */
8677 : 0, /* properties_destroyed */
8678 : 0, /* todo_flags_start */
8679 : 0, /* todo_flags_finish */
8680 : };
8681 :
8682 : class pass_lower_bitint : public gimple_opt_pass
8683 : {
8684 : public:
8685 589174 : pass_lower_bitint (gcc::context *ctxt)
8686 1178348 : : gimple_opt_pass (pass_data_lower_bitint, ctxt)
8687 : {}
8688 :
8689 : /* opt_pass methods: */
8690 294587 : opt_pass * clone () final override { return new pass_lower_bitint (m_ctxt); }
8691 1064832 : unsigned int execute (function *) final override
8692 : {
8693 1064832 : return gimple_lower_bitint ();
8694 : }
8695 :
8696 : }; // class pass_lower_bitint
8697 :
8698 : } // anon namespace
8699 :
8700 : gimple_opt_pass *
8701 294587 : make_pass_lower_bitint (gcc::context *ctxt)
8702 : {
8703 294587 : return new pass_lower_bitint (ctxt);
8704 : }
8705 :
8706 :
8707 : namespace {
8708 :
8709 : const pass_data pass_data_lower_bitint_O0 =
8710 : {
8711 : GIMPLE_PASS, /* type */
8712 : "bitintlower0", /* name */
8713 : OPTGROUP_NONE, /* optinfo_flags */
8714 : TV_NONE, /* tv_id */
8715 : PROP_cfg, /* properties_required */
8716 : PROP_gimple_lbitint, /* properties_provided */
8717 : 0, /* properties_destroyed */
8718 : 0, /* todo_flags_start */
8719 : 0, /* todo_flags_finish */
8720 : };
8721 :
8722 : class pass_lower_bitint_O0 : public gimple_opt_pass
8723 : {
8724 : public:
8725 294587 : pass_lower_bitint_O0 (gcc::context *ctxt)
8726 589174 : : gimple_opt_pass (pass_data_lower_bitint_O0, ctxt)
8727 : {}
8728 :
8729 : /* opt_pass methods: */
8730 1512174 : bool gate (function *fun) final override
8731 : {
8732 : /* With errors, normal optimization passes are not run. If we don't
8733 : lower bitint operations at all, rtl expansion will abort. */
8734 1512174 : return !(fun->curr_properties & PROP_gimple_lbitint);
8735 : }
8736 :
8737 447447 : unsigned int execute (function *) final override
8738 : {
8739 447447 : return gimple_lower_bitint ();
8740 : }
8741 :
8742 : }; // class pass_lower_bitint_O0
8743 :
8744 : } // anon namespace
8745 :
8746 : gimple_opt_pass *
8747 294587 : make_pass_lower_bitint_O0 (gcc::context *ctxt)
8748 : {
8749 294587 : return new pass_lower_bitint_O0 (ctxt);
8750 : }
|
