Line data Source code
1 : /* Instruction scheduling pass.
2 : Copyright (C) 1992-2026 Free Software Foundation, Inc.
3 : Contributed by Michael Tiemann (tiemann@cygnus.com) Enhanced by,
4 : and currently maintained by, Jim Wilson (wilson@cygnus.com)
5 :
6 : This file is part of GCC.
7 :
8 : GCC is free software; you can redistribute it and/or modify it under
9 : the terms of the GNU General Public License as published by the Free
10 : Software Foundation; either version 3, or (at your option) any later
11 : version.
12 :
13 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 : for more details.
17 :
18 : You should have received a copy of the GNU General Public License
19 : along with GCC; see the file COPYING3. If not see
20 : <http://www.gnu.org/licenses/>. */
21 :
22 : /* Instruction scheduling pass. This file, along with sched-deps.cc,
23 : contains the generic parts. The actual entry point for
24 : the normal instruction scheduling pass is found in sched-rgn.cc.
25 :
26 : We compute insn priorities based on data dependencies. Flow
27 : analysis only creates a fraction of the data-dependencies we must
28 : observe: namely, only those dependencies which the combiner can be
29 : expected to use. For this pass, we must therefore create the
30 : remaining dependencies we need to observe: register dependencies,
31 : memory dependencies, dependencies to keep function calls in order,
32 : and the dependence between a conditional branch and the setting of
33 : condition codes are all dealt with here.
34 :
35 : The scheduler first traverses the data flow graph, starting with
36 : the last instruction, and proceeding to the first, assigning values
37 : to insn_priority as it goes. This sorts the instructions
38 : topologically by data dependence.
39 :
40 : Once priorities have been established, we order the insns using
41 : list scheduling. This works as follows: starting with a list of
42 : all the ready insns, and sorted according to priority number, we
43 : schedule the insn from the end of the list by placing its
44 : predecessors in the list according to their priority order. We
45 : consider this insn scheduled by setting the pointer to the "end" of
46 : the list to point to the previous insn. When an insn has no
47 : predecessors, we either queue it until sufficient time has elapsed
48 : or add it to the ready list. As the instructions are scheduled or
49 : when stalls are introduced, the queue advances and dumps insns into
50 : the ready list. When all insns down to the lowest priority have
51 : been scheduled, the critical path of the basic block has been made
52 : as short as possible. The remaining insns are then scheduled in
53 : remaining slots.
54 :
55 : The following list shows the order in which we want to break ties
56 : among insns in the ready list:
57 :
58 : 1. choose insn with the longest path to end of bb, ties
59 : broken by
60 : 2. choose insn with least contribution to register pressure,
61 : ties broken by
62 : 3. prefer in-block upon interblock motion, ties broken by
63 : 4. prefer useful upon speculative motion, ties broken by
64 : 5. choose insn with largest control flow probability, ties
65 : broken by
66 : 6. choose insn with the least dependences upon the previously
67 : scheduled insn, or finally
68 : 7 choose the insn which has the most insns dependent on it.
69 : 8. choose insn with lowest UID.
70 :
71 : Memory references complicate matters. Only if we can be certain
72 : that memory references are not part of the data dependency graph
73 : (via true, anti, or output dependence), can we move operations past
74 : memory references. To first approximation, reads can be done
75 : independently, while writes introduce dependencies. Better
76 : approximations will yield fewer dependencies.
77 :
78 : Before reload, an extended analysis of interblock data dependences
79 : is required for interblock scheduling. This is performed in
80 : compute_block_dependences ().
81 :
82 : Dependencies set up by memory references are treated in exactly the
83 : same way as other dependencies, by using insn backward dependences
84 : INSN_BACK_DEPS. INSN_BACK_DEPS are translated into forward dependences
85 : INSN_FORW_DEPS for the purpose of forward list scheduling.
86 :
87 : Having optimized the critical path, we may have also unduly
88 : extended the lifetimes of some registers. If an operation requires
89 : that constants be loaded into registers, it is certainly desirable
90 : to load those constants as early as necessary, but no earlier.
91 : I.e., it will not do to load up a bunch of registers at the
92 : beginning of a basic block only to use them at the end, if they
93 : could be loaded later, since this may result in excessive register
94 : utilization.
95 :
96 : Note that since branches are never in basic blocks, but only end
97 : basic blocks, this pass will not move branches. But that is ok,
98 : since we can use GNU's delayed branch scheduling pass to take care
99 : of this case.
100 :
101 : Also note that no further optimizations based on algebraic
102 : identities are performed, so this pass would be a good one to
103 : perform instruction splitting, such as breaking up a multiply
104 : instruction into shifts and adds where that is profitable.
105 :
106 : Given the memory aliasing analysis that this pass should perform,
107 : it should be possible to remove redundant stores to memory, and to
108 : load values from registers instead of hitting memory.
109 :
110 : Before reload, speculative insns are moved only if a 'proof' exists
111 : that no exception will be caused by this, and if no live registers
112 : exist that inhibit the motion (live registers constraints are not
113 : represented by data dependence edges).
114 :
115 : This pass must update information that subsequent passes expect to
116 : be correct. Namely: reg_n_refs, reg_n_sets, reg_n_deaths,
117 : reg_n_calls_crossed, and reg_live_length. Also, BB_HEAD, BB_END.
118 :
119 : The information in the line number notes is carefully retained by
120 : this pass. Notes that refer to the starting and ending of
121 : exception regions are also carefully retained by this pass. All
122 : other NOTE insns are grouped in their same relative order at the
123 : beginning of basic blocks and regions that have been scheduled. */
124 :
125 : #include "config.h"
126 : #include "system.h"
127 : #include "coretypes.h"
128 : #include "backend.h"
129 : #include "target.h"
130 : #include "rtl.h"
131 : #include "cfghooks.h"
132 : #include "df.h"
133 : #include "memmodel.h"
134 : #include "tm_p.h"
135 : #include "insn-config.h"
136 : #include "regs.h"
137 : #include "ira.h"
138 : #include "recog.h"
139 : #include "insn-attr.h"
140 : #include "cfgrtl.h"
141 : #include "cfgbuild.h"
142 : #include "sched-int.h"
143 : #include "common/common-target.h"
144 : #include "dbgcnt.h"
145 : #include "cfgloop.h"
146 : #include "dumpfile.h"
147 : #include "print-rtl.h"
148 : #include "function-abi.h"
149 :
150 : #ifdef INSN_SCHEDULING
151 :
152 : /* True if we do register pressure relief through live-range
153 : shrinkage. */
154 : static bool live_range_shrinkage_p;
155 :
156 : /* Switch on live range shrinkage. */
157 : void
158 35 : initialize_live_range_shrinkage (void)
159 : {
160 35 : live_range_shrinkage_p = true;
161 35 : }
162 :
163 : /* Switch off live range shrinkage. */
164 : void
165 35 : finish_live_range_shrinkage (void)
166 : {
167 35 : live_range_shrinkage_p = false;
168 35 : }
169 :
170 : /* issue_rate is the number of insns that can be scheduled in the same
171 : machine cycle. It can be defined in the config/mach/mach.h file,
172 : otherwise we set it to 1. */
173 :
174 : int issue_rate;
175 :
176 : /* This can be set to true by a backend if the scheduler should not
177 : enable a DCE pass. */
178 : bool sched_no_dce;
179 :
180 : /* The current initiation interval used when modulo scheduling. */
181 : static int modulo_ii;
182 :
183 : /* The maximum number of stages we are prepared to handle. */
184 : static int modulo_max_stages;
185 :
186 : /* The number of insns that exist in each iteration of the loop. We use this
187 : to detect when we've scheduled all insns from the first iteration. */
188 : static int modulo_n_insns;
189 :
190 : /* The current count of insns in the first iteration of the loop that have
191 : already been scheduled. */
192 : static int modulo_insns_scheduled;
193 :
194 : /* The maximum uid of insns from the first iteration of the loop. */
195 : static int modulo_iter0_max_uid;
196 :
197 : /* The number of times we should attempt to backtrack when modulo scheduling.
198 : Decreased each time we have to backtrack. */
199 : static int modulo_backtracks_left;
200 :
201 : /* The stage in which the last insn from the original loop was
202 : scheduled. */
203 : static int modulo_last_stage;
204 :
205 : /* sched-verbose controls the amount of debugging output the
206 : scheduler prints. It is controlled by -fsched-verbose=N:
207 : N=0: no debugging output.
208 : N=1: default value.
209 : N=2: bb's probabilities, detailed ready list info, unit/insn info.
210 : N=3: rtl at abort point, control-flow, regions info.
211 : N=5: dependences info. */
212 : int sched_verbose = 0;
213 :
214 : /* Debugging file. All printouts are sent to dump. */
215 : FILE *sched_dump = 0;
216 :
217 : /* This is a placeholder for the scheduler parameters common
218 : to all schedulers. */
219 : struct common_sched_info_def *common_sched_info;
220 :
221 : #define INSN_TICK(INSN) (HID (INSN)->tick)
222 : #define INSN_EXACT_TICK(INSN) (HID (INSN)->exact_tick)
223 : #define INSN_TICK_ESTIMATE(INSN) (HID (INSN)->tick_estimate)
224 : #define INTER_TICK(INSN) (HID (INSN)->inter_tick)
225 : #define FEEDS_BACKTRACK_INSN(INSN) (HID (INSN)->feeds_backtrack_insn)
226 : #define SHADOW_P(INSN) (HID (INSN)->shadow_p)
227 : #define MUST_RECOMPUTE_SPEC_P(INSN) (HID (INSN)->must_recompute_spec)
228 : /* Cached cost of the instruction. Use insn_sched_cost to get cost of the
229 : insn. -1 here means that the field is not initialized. */
230 : #define INSN_COST(INSN) (HID (INSN)->cost)
231 :
232 : /* If INSN_TICK of an instruction is equal to INVALID_TICK,
233 : then it should be recalculated from scratch. */
234 : #define INVALID_TICK (-(max_insn_queue_index + 1))
235 : /* The minimal value of the INSN_TICK of an instruction. */
236 : #define MIN_TICK (-max_insn_queue_index)
237 :
238 : /* Original order of insns in the ready list.
239 : Used to keep order of normal insns while separating DEBUG_INSNs. */
240 : #define INSN_RFS_DEBUG_ORIG_ORDER(INSN) (HID (INSN)->rfs_debug_orig_order)
241 :
242 : /* The deciding reason for INSN's place in the ready list. */
243 : #define INSN_LAST_RFS_WIN(INSN) (HID (INSN)->last_rfs_win)
244 :
245 : /* List of important notes we must keep around. This is a pointer to the
246 : last element in the list. */
247 : rtx_insn *note_list;
248 :
249 : static struct spec_info_def spec_info_var;
250 : /* Description of the speculative part of the scheduling.
251 : If NULL - no speculation. */
252 : spec_info_t spec_info = NULL;
253 :
254 : /* True, if recovery block was added during scheduling of current block.
255 : Used to determine, if we need to fix INSN_TICKs. */
256 : static bool haifa_recovery_bb_recently_added_p;
257 :
258 : /* True, if recovery block was added during this scheduling pass.
259 : Used to determine if we should have empty memory pools of dependencies
260 : after finishing current region. */
261 : bool haifa_recovery_bb_ever_added_p;
262 :
263 : /* Counters of different types of speculative instructions. */
264 : static int nr_begin_data, nr_be_in_data, nr_begin_control, nr_be_in_control;
265 :
266 : /* Array used in {unlink, restore}_bb_notes. */
267 : static rtx_insn **bb_header = 0;
268 :
269 : /* Basic block after which recovery blocks will be created. */
270 : static basic_block before_recovery;
271 :
272 : /* Basic block just before the EXIT_BLOCK and after recovery, if we have
273 : created it. */
274 : basic_block after_recovery;
275 :
276 : /* FALSE if we add bb to another region, so we don't need to initialize it. */
277 : bool adding_bb_to_current_region_p = true;
278 :
279 : /* Queues, etc. */
280 :
281 : /* An instruction is ready to be scheduled when all insns preceding it
282 : have already been scheduled. It is important to ensure that all
283 : insns which use its result will not be executed until its result
284 : has been computed. An insn is maintained in one of four structures:
285 :
286 : (P) the "Pending" set of insns which cannot be scheduled until
287 : their dependencies have been satisfied.
288 : (Q) the "Queued" set of insns that can be scheduled when sufficient
289 : time has passed.
290 : (R) the "Ready" list of unscheduled, uncommitted insns.
291 : (S) the "Scheduled" list of insns.
292 :
293 : Initially, all insns are either "Pending" or "Ready" depending on
294 : whether their dependencies are satisfied.
295 :
296 : Insns move from the "Ready" list to the "Scheduled" list as they
297 : are committed to the schedule. As this occurs, the insns in the
298 : "Pending" list have their dependencies satisfied and move to either
299 : the "Ready" list or the "Queued" set depending on whether
300 : sufficient time has passed to make them ready. As time passes,
301 : insns move from the "Queued" set to the "Ready" list.
302 :
303 : The "Pending" list (P) are the insns in the INSN_FORW_DEPS of the
304 : unscheduled insns, i.e., those that are ready, queued, and pending.
305 : The "Queued" set (Q) is implemented by the variable `insn_queue'.
306 : The "Ready" list (R) is implemented by the variables `ready' and
307 : `n_ready'.
308 : The "Scheduled" list (S) is the new insn chain built by this pass.
309 :
310 : The transition (R->S) is implemented in the scheduling loop in
311 : `schedule_block' when the best insn to schedule is chosen.
312 : The transitions (P->R and P->Q) are implemented in `schedule_insn' as
313 : insns move from the ready list to the scheduled list.
314 : The transition (Q->R) is implemented in 'queue_to_insn' as time
315 : passes or stalls are introduced. */
316 :
317 : /* Implement a circular buffer to delay instructions until sufficient
318 : time has passed. For the new pipeline description interface,
319 : MAX_INSN_QUEUE_INDEX is a power of two minus one which is not less
320 : than maximal time of instruction execution computed by genattr.cc on
321 : the base maximal time of functional unit reservations and getting a
322 : result. This is the longest time an insn may be queued. */
323 :
324 : static rtx_insn_list **insn_queue;
325 : static int q_ptr = 0;
326 : static int q_size = 0;
327 : #define NEXT_Q(X) (((X)+1) & max_insn_queue_index)
328 : #define NEXT_Q_AFTER(X, C) (((X)+C) & max_insn_queue_index)
329 :
330 : #define QUEUE_SCHEDULED (-3)
331 : #define QUEUE_NOWHERE (-2)
332 : #define QUEUE_READY (-1)
333 : /* QUEUE_SCHEDULED - INSN is scheduled.
334 : QUEUE_NOWHERE - INSN isn't scheduled yet and is neither in
335 : queue or ready list.
336 : QUEUE_READY - INSN is in ready list.
337 : N >= 0 - INSN queued for X [where NEXT_Q_AFTER (q_ptr, X) == N] cycles. */
338 :
339 : #define QUEUE_INDEX(INSN) (HID (INSN)->queue_index)
340 :
341 : /* The following variable value refers for all current and future
342 : reservations of the processor units. */
343 : state_t curr_state;
344 :
345 : /* The following variable value is size of memory representing all
346 : current and future reservations of the processor units. */
347 : size_t dfa_state_size;
348 :
349 : /* The following array is used to find the best insn from ready when
350 : the automaton pipeline interface is used. */
351 : signed char *ready_try = NULL;
352 :
353 : /* The ready list. */
354 : struct ready_list ready = {NULL, 0, 0, 0, 0};
355 :
356 : /* The pointer to the ready list (to be removed). */
357 : static struct ready_list *readyp = &ready;
358 :
359 : /* Scheduling clock. */
360 : static int clock_var;
361 :
362 : /* Clock at which the previous instruction was issued. */
363 : static int last_clock_var;
364 :
365 : /* Set to true if, when queuing a shadow insn, we discover that it would be
366 : scheduled too late. */
367 : static bool must_backtrack;
368 :
369 : /* The following variable value is number of essential insns issued on
370 : the current cycle. An insn is essential one if it changes the
371 : processors state. */
372 : int cycle_issued_insns;
373 :
374 : /* This records the actual schedule. It is built up during the main phase
375 : of schedule_block, and afterwards used to reorder the insns in the RTL. */
376 : static vec<rtx_insn *> scheduled_insns;
377 :
378 : static int may_trap_exp (const_rtx, int);
379 :
380 : /* Nonzero iff the address is comprised from at most 1 register. */
381 : #define CONST_BASED_ADDRESS_P(x) \
382 : (REG_P (x) \
383 : || ((GET_CODE (x) == PLUS || GET_CODE (x) == MINUS \
384 : || (GET_CODE (x) == LO_SUM)) \
385 : && (CONSTANT_P (XEXP (x, 0)) \
386 : || CONSTANT_P (XEXP (x, 1)))))
387 :
388 : /* Returns a class that insn with GET_DEST(insn)=x may belong to,
389 : as found by analyzing insn's expression. */
390 :
391 :
392 : static int haifa_luid_for_non_insn (rtx x);
393 :
394 : /* Haifa version of sched_info hooks common to all headers. */
395 : const struct common_sched_info_def haifa_common_sched_info =
396 : {
397 : NULL, /* fix_recovery_cfg */
398 : NULL, /* add_block */
399 : NULL, /* estimate_number_of_insns */
400 : haifa_luid_for_non_insn, /* luid_for_non_insn */
401 : SCHED_PASS_UNKNOWN /* sched_pass_id */
402 : };
403 :
404 : /* Mapping from instruction UID to its Logical UID. */
405 : vec<int> sched_luids;
406 :
407 : /* Next LUID to assign to an instruction. */
408 : int sched_max_luid = 1;
409 :
410 : /* Haifa Instruction Data. */
411 : vec<haifa_insn_data_def> h_i_d;
412 :
413 : void (* sched_init_only_bb) (basic_block, basic_block);
414 :
415 : /* Split block function. Different schedulers might use different functions
416 : to handle their internal data consistent. */
417 : basic_block (* sched_split_block) (basic_block, rtx);
418 :
419 : /* Create empty basic block after the specified block. */
420 : basic_block (* sched_create_empty_bb) (basic_block);
421 :
422 : /* Return the number of cycles until INSN is expected to be ready.
423 : Return zero if it already is. */
424 : static int
425 27322 : insn_delay (rtx_insn *insn)
426 : {
427 27322 : return MAX (INSN_TICK (insn) - clock_var, 0);
428 : }
429 :
430 : static int
431 27525 : may_trap_exp (const_rtx x, int is_store)
432 : {
433 27525 : enum rtx_code code;
434 :
435 27525 : if (x == 0)
436 : return TRAP_FREE;
437 27525 : code = GET_CODE (x);
438 27525 : if (is_store)
439 : {
440 5653 : if (code == MEM && may_trap_p (x))
441 : return TRAP_RISKY;
442 : else
443 5401 : return TRAP_FREE;
444 : }
445 21872 : if (code == MEM)
446 : {
447 : /* The insn uses memory: a volatile load. */
448 1676 : if (MEM_VOLATILE_P (x))
449 : return IRISKY;
450 : /* An exception-free load. */
451 1654 : if (!may_trap_p (x))
452 : return IFREE;
453 : /* A load with 1 base register, to be further checked. */
454 382 : if (CONST_BASED_ADDRESS_P (XEXP (x, 0)))
455 : return PFREE_CANDIDATE;
456 : /* No info on the load, to be further checked. */
457 24 : return PRISKY_CANDIDATE;
458 : }
459 : else
460 : {
461 20196 : const char *fmt;
462 20196 : int i, insn_class = TRAP_FREE;
463 :
464 : /* Neither store nor load, check if it may cause a trap. */
465 20196 : if (may_trap_p (x))
466 : return TRAP_RISKY;
467 : /* Recursive step: walk the insn... */
468 19620 : fmt = GET_RTX_FORMAT (code);
469 44860 : for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
470 : {
471 25244 : if (fmt[i] == 'e')
472 : {
473 12086 : int tmp_class = may_trap_exp (XEXP (x, i), is_store);
474 12086 : insn_class = WORST_CLASS (insn_class, tmp_class);
475 : }
476 13158 : else if (fmt[i] == 'E')
477 : {
478 : int j;
479 308 : for (j = 0; j < XVECLEN (x, i); j++)
480 : {
481 266 : int tmp_class = may_trap_exp (XVECEXP (x, i, j), is_store);
482 266 : insn_class = WORST_CLASS (insn_class, tmp_class);
483 266 : if (insn_class == TRAP_RISKY || insn_class == IRISKY)
484 : break;
485 : }
486 : }
487 25244 : if (insn_class == TRAP_RISKY || insn_class == IRISKY)
488 : break;
489 : }
490 : return insn_class;
491 : }
492 : }
493 :
494 : /* Classifies rtx X of an insn for the purpose of verifying that X can be
495 : executed speculatively (and consequently the insn can be moved
496 : speculatively), by examining X, returning:
497 : TRAP_RISKY: store, or risky non-load insn (e.g. division by variable).
498 : TRAP_FREE: non-load insn.
499 : IFREE: load from a globally safe location.
500 : IRISKY: volatile load.
501 : PFREE_CANDIDATE, PRISKY_CANDIDATE: load that need to be checked for
502 : being either PFREE or PRISKY. */
503 :
504 : static int
505 6762 : haifa_classify_rtx (const_rtx x)
506 : {
507 6762 : int tmp_class = TRAP_FREE;
508 6762 : int insn_class = TRAP_FREE;
509 6762 : enum rtx_code code;
510 :
511 6762 : if (GET_CODE (x) == PARALLEL)
512 : {
513 782 : int i, len = XVECLEN (x, 0);
514 :
515 2132 : for (i = len - 1; i >= 0; i--)
516 : {
517 1511 : tmp_class = haifa_classify_rtx (XVECEXP (x, 0, i));
518 1511 : insn_class = WORST_CLASS (insn_class, tmp_class);
519 1511 : if (insn_class == TRAP_RISKY || insn_class == IRISKY)
520 : break;
521 : }
522 : }
523 : else
524 : {
525 5980 : code = GET_CODE (x);
526 5980 : switch (code)
527 : {
528 735 : case CLOBBER:
529 : /* Test if it is a 'store'. */
530 735 : tmp_class = may_trap_exp (XEXP (x, 0), 1);
531 735 : break;
532 4918 : case SET:
533 : /* Test if it is a store. */
534 4918 : tmp_class = may_trap_exp (SET_DEST (x), 1);
535 4918 : if (tmp_class == TRAP_RISKY)
536 : break;
537 : /* Test if it is a load. */
538 4760 : tmp_class =
539 4760 : WORST_CLASS (tmp_class,
540 : may_trap_exp (SET_SRC (x), 0));
541 : break;
542 0 : case COND_EXEC:
543 0 : tmp_class = haifa_classify_rtx (COND_EXEC_CODE (x));
544 0 : if (tmp_class == TRAP_RISKY)
545 : break;
546 0 : tmp_class = WORST_CLASS (tmp_class,
547 : may_trap_exp (COND_EXEC_TEST (x), 0));
548 : break;
549 : case TRAP_IF:
550 169 : tmp_class = TRAP_RISKY;
551 : break;
552 158 : default:;
553 : }
554 : insn_class = tmp_class;
555 : }
556 :
557 6762 : return insn_class;
558 : }
559 :
560 : int
561 5251 : haifa_classify_insn (const_rtx insn)
562 : {
563 5251 : return haifa_classify_rtx (PATTERN (insn));
564 : }
565 :
566 : /* After the scheduler initialization function has been called, this function
567 : can be called to enable modulo scheduling. II is the initiation interval
568 : we should use, it affects the delays for delay_pairs that were recorded as
569 : separated by a given number of stages.
570 :
571 : MAX_STAGES provides us with a limit
572 : after which we give up scheduling; the caller must have unrolled at least
573 : as many copies of the loop body and recorded delay_pairs for them.
574 :
575 : INSNS is the number of real (non-debug) insns in one iteration of
576 : the loop. MAX_UID can be used to test whether an insn belongs to
577 : the first iteration of the loop; all of them have a uid lower than
578 : MAX_UID. */
579 : void
580 0 : set_modulo_params (int ii, int max_stages, int insns, int max_uid)
581 : {
582 0 : modulo_ii = ii;
583 0 : modulo_max_stages = max_stages;
584 0 : modulo_n_insns = insns;
585 0 : modulo_iter0_max_uid = max_uid;
586 0 : modulo_backtracks_left = param_max_modulo_backtrack_attempts;
587 0 : }
588 :
589 : /* A structure to record a pair of insns where the first one is a real
590 : insn that has delay slots, and the second is its delayed shadow.
591 : I1 is scheduled normally and will emit an assembly instruction,
592 : while I2 describes the side effect that takes place at the
593 : transition between cycles CYCLES and (CYCLES + 1) after I1. */
594 : struct delay_pair
595 : {
596 : struct delay_pair *next_same_i1;
597 : rtx_insn *i1, *i2;
598 : int cycles;
599 : /* When doing modulo scheduling, we a delay_pair can also be used to
600 : show that I1 and I2 are the same insn in a different stage. If that
601 : is the case, STAGES will be nonzero. */
602 : int stages;
603 : };
604 :
605 : /* Helpers for delay hashing. */
606 :
607 : struct delay_i1_hasher : nofree_ptr_hash <delay_pair>
608 : {
609 : typedef void *compare_type;
610 : static inline hashval_t hash (const delay_pair *);
611 : static inline bool equal (const delay_pair *, const void *);
612 : };
613 :
614 : /* Returns a hash value for X, based on hashing just I1. */
615 :
616 : inline hashval_t
617 0 : delay_i1_hasher::hash (const delay_pair *x)
618 : {
619 0 : return htab_hash_pointer (x->i1);
620 : }
621 :
622 : /* Return true if I1 of pair X is the same as that of pair Y. */
623 :
624 : inline bool
625 0 : delay_i1_hasher::equal (const delay_pair *x, const void *y)
626 : {
627 0 : return x->i1 == y;
628 : }
629 :
630 : struct delay_i2_hasher : free_ptr_hash <delay_pair>
631 : {
632 : typedef void *compare_type;
633 : static inline hashval_t hash (const delay_pair *);
634 : static inline bool equal (const delay_pair *, const void *);
635 : };
636 :
637 : /* Returns a hash value for X, based on hashing just I2. */
638 :
639 : inline hashval_t
640 0 : delay_i2_hasher::hash (const delay_pair *x)
641 : {
642 0 : return htab_hash_pointer (x->i2);
643 : }
644 :
645 : /* Return true if I2 of pair X is the same as that of pair Y. */
646 :
647 : inline bool
648 0 : delay_i2_hasher::equal (const delay_pair *x, const void *y)
649 : {
650 0 : return x->i2 == y;
651 : }
652 :
653 : /* Two hash tables to record delay_pairs, one indexed by I1 and the other
654 : indexed by I2. */
655 : static hash_table<delay_i1_hasher> *delay_htab;
656 : static hash_table<delay_i2_hasher> *delay_htab_i2;
657 :
658 : /* Called through htab_traverse. Walk the hashtable using I2 as
659 : index, and delete all elements involving an UID higher than
660 : that pointed to by *DATA. */
661 : int
662 0 : haifa_htab_i2_traverse (delay_pair **slot, int *data)
663 : {
664 0 : int maxuid = *data;
665 0 : struct delay_pair *p = *slot;
666 0 : if (INSN_UID (p->i2) >= maxuid || INSN_UID (p->i1) >= maxuid)
667 : {
668 0 : delay_htab_i2->clear_slot (slot);
669 : }
670 0 : return 1;
671 : }
672 :
673 : /* Called through htab_traverse. Walk the hashtable using I2 as
674 : index, and delete all elements involving an UID higher than
675 : that pointed to by *DATA. */
676 : int
677 0 : haifa_htab_i1_traverse (delay_pair **pslot, int *data)
678 : {
679 0 : int maxuid = *data;
680 0 : struct delay_pair *p, *first, **pprev;
681 :
682 0 : if (INSN_UID ((*pslot)->i1) >= maxuid)
683 : {
684 0 : delay_htab->clear_slot (pslot);
685 0 : return 1;
686 : }
687 : pprev = &first;
688 0 : for (p = *pslot; p; p = p->next_same_i1)
689 : {
690 0 : if (INSN_UID (p->i2) < maxuid)
691 : {
692 0 : *pprev = p;
693 0 : pprev = &p->next_same_i1;
694 : }
695 : }
696 0 : *pprev = NULL;
697 0 : if (first == NULL)
698 0 : delay_htab->clear_slot (pslot);
699 : else
700 0 : *pslot = first;
701 : return 1;
702 : }
703 :
704 : /* Discard all delay pairs which involve an insn with an UID higher
705 : than MAX_UID. */
706 : void
707 0 : discard_delay_pairs_above (int max_uid)
708 : {
709 0 : delay_htab->traverse <int *, haifa_htab_i1_traverse> (&max_uid);
710 0 : delay_htab_i2->traverse <int *, haifa_htab_i2_traverse> (&max_uid);
711 0 : }
712 :
713 : /* This function can be called by a port just before it starts the final
714 : scheduling pass. It records the fact that an instruction with delay
715 : slots has been split into two insns, I1 and I2. The first one will be
716 : scheduled normally and initiates the operation. The second one is a
717 : shadow which must follow a specific number of cycles after I1; its only
718 : purpose is to show the side effect that occurs at that cycle in the RTL.
719 : If a JUMP_INSN or a CALL_INSN has been split, I1 should be a normal INSN,
720 : while I2 retains the original insn type.
721 :
722 : There are two ways in which the number of cycles can be specified,
723 : involving the CYCLES and STAGES arguments to this function. If STAGES
724 : is zero, we just use the value of CYCLES. Otherwise, STAGES is a factor
725 : which is multiplied by MODULO_II to give the number of cycles. This is
726 : only useful if the caller also calls set_modulo_params to enable modulo
727 : scheduling. */
728 :
729 : void
730 0 : record_delay_slot_pair (rtx_insn *i1, rtx_insn *i2, int cycles, int stages)
731 : {
732 0 : struct delay_pair *p = XNEW (struct delay_pair);
733 0 : struct delay_pair **slot;
734 :
735 0 : p->i1 = i1;
736 0 : p->i2 = i2;
737 0 : p->cycles = cycles;
738 0 : p->stages = stages;
739 :
740 0 : if (!delay_htab)
741 : {
742 0 : delay_htab = new hash_table<delay_i1_hasher> (10);
743 0 : delay_htab_i2 = new hash_table<delay_i2_hasher> (10);
744 : }
745 0 : slot = delay_htab->find_slot_with_hash (i1, htab_hash_pointer (i1), INSERT);
746 0 : p->next_same_i1 = *slot;
747 0 : *slot = p;
748 0 : slot = delay_htab_i2->find_slot (p, INSERT);
749 0 : *slot = p;
750 0 : }
751 :
752 : /* Examine the delay pair hashtable to see if INSN is a shadow for another,
753 : and return the other insn if so. Return NULL otherwise. */
754 : rtx_insn *
755 0 : real_insn_for_shadow (rtx_insn *insn)
756 : {
757 0 : struct delay_pair *pair;
758 :
759 0 : if (!delay_htab)
760 : return NULL;
761 :
762 0 : pair = delay_htab_i2->find_with_hash (insn, htab_hash_pointer (insn));
763 0 : if (!pair || pair->stages > 0)
764 : return NULL;
765 0 : return pair->i1;
766 : }
767 :
768 : /* For a pair P of insns, return the fixed distance in cycles from the first
769 : insn after which the second must be scheduled. */
770 : static int
771 0 : pair_delay (struct delay_pair *p)
772 : {
773 0 : if (p->stages == 0)
774 0 : return p->cycles;
775 : else
776 0 : return p->stages * modulo_ii;
777 : }
778 :
779 : /* Given an insn INSN, add a dependence on its delayed shadow if it
780 : has one. Also try to find situations where shadows depend on each other
781 : and add dependencies to the real insns to limit the amount of backtracking
782 : needed. */
783 : void
784 1535 : add_delay_dependencies (rtx_insn *insn)
785 : {
786 1535 : struct delay_pair *pair;
787 1535 : sd_iterator_def sd_it;
788 1535 : dep_t dep;
789 :
790 1535 : if (!delay_htab)
791 1535 : return;
792 :
793 0 : pair = delay_htab_i2->find_with_hash (insn, htab_hash_pointer (insn));
794 0 : if (!pair)
795 : return;
796 0 : add_dependence (insn, pair->i1, REG_DEP_ANTI);
797 0 : if (pair->stages)
798 : return;
799 :
800 0 : FOR_EACH_DEP (pair->i2, SD_LIST_BACK, sd_it, dep)
801 : {
802 0 : rtx_insn *pro = DEP_PRO (dep);
803 0 : struct delay_pair *other_pair
804 0 : = delay_htab_i2->find_with_hash (pro, htab_hash_pointer (pro));
805 0 : if (!other_pair || other_pair->stages)
806 0 : continue;
807 0 : if (pair_delay (other_pair) >= pair_delay (pair))
808 : {
809 0 : if (sched_verbose >= 4)
810 : {
811 0 : fprintf (sched_dump, ";;\tadding dependence %d <- %d\n",
812 0 : INSN_UID (other_pair->i1),
813 0 : INSN_UID (pair->i1));
814 0 : fprintf (sched_dump, ";;\tpair1 %d <- %d, cost %d\n",
815 0 : INSN_UID (pair->i1),
816 0 : INSN_UID (pair->i2),
817 : pair_delay (pair));
818 0 : fprintf (sched_dump, ";;\tpair2 %d <- %d, cost %d\n",
819 0 : INSN_UID (other_pair->i1),
820 0 : INSN_UID (other_pair->i2),
821 : pair_delay (other_pair));
822 : }
823 0 : add_dependence (pair->i1, other_pair->i1, REG_DEP_ANTI);
824 : }
825 : }
826 : }
827 :
828 : /* Forward declarations. */
829 :
830 : static int priority (rtx_insn *, bool force_recompute = false);
831 : static int autopref_rank_for_schedule (const rtx_insn *, const rtx_insn *);
832 : static int rank_for_schedule (const void *, const void *);
833 : static void swap_sort (rtx_insn **, int);
834 : static void queue_insn (rtx_insn *, int, const char *);
835 : static int schedule_insn (rtx_insn *);
836 : static void adjust_priority (rtx_insn *);
837 : static void advance_one_cycle (void);
838 : static void extend_h_i_d (void);
839 :
840 :
841 : /* Notes handling mechanism:
842 : =========================
843 : Generally, NOTES are saved before scheduling and restored after scheduling.
844 : The scheduler distinguishes between two types of notes:
845 :
846 : (1) LOOP_BEGIN, LOOP_END, SETJMP, EHREGION_BEG, EHREGION_END notes:
847 : Before scheduling a region, a pointer to the note is added to the insn
848 : that follows or precedes it. (This happens as part of the data dependence
849 : computation). After scheduling an insn, the pointer contained in it is
850 : used for regenerating the corresponding note (in reemit_notes).
851 :
852 : (2) All other notes (e.g. INSN_DELETED): Before scheduling a block,
853 : these notes are put in a list (in rm_other_notes() and
854 : unlink_other_notes ()). After scheduling the block, these notes are
855 : inserted at the beginning of the block (in schedule_block()). */
856 :
857 : static void ready_add (struct ready_list *, rtx_insn *, bool);
858 : static rtx_insn *ready_remove_first (struct ready_list *);
859 : static rtx_insn *ready_remove_first_dispatch (struct ready_list *ready);
860 :
861 : static void queue_to_ready (struct ready_list *);
862 : static int early_queue_to_ready (state_t, struct ready_list *);
863 :
864 : /* The following functions are used to implement multi-pass scheduling
865 : on the first cycle. */
866 : static rtx_insn *ready_remove (struct ready_list *, int);
867 : static void ready_remove_insn (rtx_insn *);
868 :
869 : static void fix_inter_tick (rtx_insn *, rtx_insn *);
870 : static int fix_tick_ready (rtx_insn *);
871 : static void change_queue_index (rtx_insn *, int);
872 :
873 : /* The following functions are used to implement scheduling of data/control
874 : speculative instructions. */
875 :
876 : static void extend_h_i_d (void);
877 : static void init_h_i_d (rtx_insn *);
878 : static int haifa_speculate_insn (rtx_insn *, ds_t, rtx *);
879 : static void generate_recovery_code (rtx_insn *);
880 : static void process_insn_forw_deps_be_in_spec (rtx_insn *, rtx_insn *, ds_t);
881 : static void begin_speculative_block (rtx_insn *);
882 : static void add_to_speculative_block (rtx_insn *);
883 : static void init_before_recovery (basic_block *);
884 : static void create_check_block_twin (rtx_insn *, bool);
885 : static void fix_recovery_deps (basic_block);
886 : static bool haifa_change_pattern (rtx_insn *, rtx);
887 : static void dump_new_block_header (int, basic_block, rtx_insn *, rtx_insn *);
888 : static void restore_bb_notes (basic_block);
889 : static void fix_jump_move (rtx_insn *);
890 : static void move_block_after_check (rtx_insn *);
891 : static void move_succs (vec<edge, va_gc> **, basic_block);
892 : static void sched_remove_insn (rtx_insn *);
893 : static void clear_priorities (rtx_insn *, rtx_vec_t *);
894 : static void calc_priorities (const rtx_vec_t &);
895 : static void add_jump_dependencies (rtx_insn *, rtx_insn *);
896 :
897 : #endif /* INSN_SCHEDULING */
898 :
899 : /* Point to state used for the current scheduling pass. */
900 : struct haifa_sched_info *current_sched_info;
901 :
902 : #ifndef INSN_SCHEDULING
903 : void
904 : schedule_insns (void)
905 : {
906 : }
907 : #else
908 :
909 : /* Do register pressure sensitive insn scheduling if the flag is set
910 : up. */
911 : enum sched_pressure_algorithm sched_pressure;
912 :
913 : /* Map regno -> its pressure class. The map defined only when
914 : SCHED_PRESSURE != SCHED_PRESSURE_NONE. */
915 : enum reg_class *sched_regno_pressure_class;
916 :
917 : /* The current register pressure. Only elements corresponding pressure
918 : classes are defined. */
919 : static int curr_reg_pressure[N_REG_CLASSES];
920 :
921 : /* Saved value of the previous array. */
922 : static int saved_reg_pressure[N_REG_CLASSES];
923 :
924 : /* Register living at given scheduling point. */
925 : static bitmap curr_reg_live;
926 :
927 : /* Saved value of the previous array. */
928 : static bitmap saved_reg_live;
929 :
930 : /* Registers mentioned in the current region. */
931 : static bitmap region_ref_regs;
932 :
933 : /* Temporary bitmap used for SCHED_PRESSURE_MODEL. */
934 : static bitmap tmp_bitmap;
935 :
936 : /* Effective number of available registers of a given class (see comment
937 : in sched_pressure_start_bb). */
938 : static int sched_class_regs_num[N_REG_CLASSES];
939 : /* The number of registers that the function would need to save before it
940 : uses them, and the number of fixed_regs. Helpers for calculating of
941 : sched_class_regs_num. */
942 : static int call_saved_regs_num[N_REG_CLASSES];
943 : static int fixed_regs_num[N_REG_CLASSES];
944 :
945 : /* Initiate register pressure relative info for scheduling the current
946 : region. Currently it is only clearing register mentioned in the
947 : current region. */
948 : void
949 679 : sched_init_region_reg_pressure_info (void)
950 : {
951 679 : bitmap_clear (region_ref_regs);
952 679 : }
953 :
954 : /* PRESSURE[CL] describes the pressure on register class CL. Update it
955 : for the birth (if BIRTH_P) or death (if !BIRTH_P) of register REGNO.
956 : LIVE tracks the set of live registers; if it is null, assume that
957 : every birth or death is genuine. */
958 : static inline void
959 44518 : mark_regno_birth_or_death (bitmap live, int *pressure, int regno, bool birth_p)
960 : {
961 44518 : enum reg_class pressure_class;
962 :
963 44518 : pressure_class = sched_regno_pressure_class[regno];
964 44518 : if (regno >= FIRST_PSEUDO_REGISTER)
965 : {
966 36078 : if (pressure_class != NO_REGS)
967 : {
968 36034 : if (birth_p)
969 : {
970 22718 : if (!live || bitmap_set_bit (live, regno))
971 22327 : pressure[pressure_class]
972 22327 : += (ira_reg_class_max_nregs
973 22327 : [pressure_class][PSEUDO_REGNO_MODE (regno)]);
974 : }
975 : else
976 : {
977 13316 : if (!live || bitmap_clear_bit (live, regno))
978 12949 : pressure[pressure_class]
979 12949 : -= (ira_reg_class_max_nregs
980 12949 : [pressure_class][PSEUDO_REGNO_MODE (regno)]);
981 : }
982 : }
983 : }
984 8440 : else if (pressure_class != NO_REGS
985 8440 : && ! TEST_HARD_REG_BIT (ira_no_alloc_regs, regno))
986 : {
987 6455 : if (birth_p)
988 : {
989 4487 : if (!live || bitmap_set_bit (live, regno))
990 4486 : pressure[pressure_class]++;
991 : }
992 : else
993 : {
994 1968 : if (!live || bitmap_clear_bit (live, regno))
995 1968 : pressure[pressure_class]--;
996 : }
997 : }
998 44518 : }
999 :
1000 : /* Initiate current register pressure related info from living
1001 : registers given by LIVE. */
1002 : static void
1003 706 : initiate_reg_pressure_info (bitmap live)
1004 : {
1005 706 : int i;
1006 706 : unsigned int j;
1007 706 : bitmap_iterator bi;
1008 :
1009 3534 : for (i = 0; i < ira_pressure_classes_num; i++)
1010 2828 : curr_reg_pressure[ira_pressure_classes[i]] = 0;
1011 706 : bitmap_clear (curr_reg_live);
1012 7460 : EXECUTE_IF_SET_IN_BITMAP (live, 0, j, bi)
1013 6754 : if (sched_pressure == SCHED_PRESSURE_MODEL
1014 6754 : || current_nr_blocks == 1
1015 7510 : || bitmap_bit_p (region_ref_regs, j))
1016 6346 : mark_regno_birth_or_death (curr_reg_live, curr_reg_pressure, j, true);
1017 706 : }
1018 :
1019 : /* Mark registers in X as mentioned in the current region. */
1020 : static void
1021 2988 : setup_ref_regs (rtx x)
1022 : {
1023 2988 : int i, j;
1024 2988 : const RTX_CODE code = GET_CODE (x);
1025 2988 : const char *fmt;
1026 :
1027 2988 : if (REG_P (x))
1028 : {
1029 1020 : bitmap_set_range (region_ref_regs, REGNO (x), REG_NREGS (x));
1030 1020 : return;
1031 : }
1032 1968 : fmt = GET_RTX_FORMAT (code);
1033 5184 : for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
1034 3216 : if (fmt[i] == 'e')
1035 2292 : setup_ref_regs (XEXP (x, i));
1036 924 : else if (fmt[i] == 'E')
1037 : {
1038 120 : for (j = 0; j < XVECLEN (x, i); j++)
1039 72 : setup_ref_regs (XVECEXP (x, i, j));
1040 : }
1041 : }
1042 :
1043 : /* Initiate current register pressure related info at the start of
1044 : basic block BB. */
1045 : static void
1046 706 : initiate_bb_reg_pressure_info (basic_block bb)
1047 : {
1048 706 : unsigned int i ATTRIBUTE_UNUSED;
1049 706 : rtx_insn *insn;
1050 :
1051 706 : if (current_nr_blocks > 1)
1052 1268 : FOR_BB_INSNS (bb, insn)
1053 1208 : if (NONDEBUG_INSN_P (insn))
1054 624 : setup_ref_regs (PATTERN (insn));
1055 706 : initiate_reg_pressure_info (df_get_live_in (bb));
1056 706 : if (bb_has_eh_pred (bb))
1057 0 : for (i = 0; ; ++i)
1058 : {
1059 0 : unsigned int regno = EH_RETURN_DATA_REGNO (i);
1060 :
1061 0 : if (regno == INVALID_REGNUM)
1062 : break;
1063 0 : if (! bitmap_bit_p (df_get_live_in (bb), regno))
1064 0 : mark_regno_birth_or_death (curr_reg_live, curr_reg_pressure,
1065 : regno, true);
1066 0 : }
1067 706 : }
1068 :
1069 : /* Save current register pressure related info. */
1070 : static void
1071 2866 : save_reg_pressure (void)
1072 : {
1073 2866 : int i;
1074 :
1075 14506 : for (i = 0; i < ira_pressure_classes_num; i++)
1076 11640 : saved_reg_pressure[ira_pressure_classes[i]]
1077 11640 : = curr_reg_pressure[ira_pressure_classes[i]];
1078 2866 : bitmap_copy (saved_reg_live, curr_reg_live);
1079 2866 : }
1080 :
1081 : /* Restore saved register pressure related info. */
1082 : static void
1083 2866 : restore_reg_pressure (void)
1084 : {
1085 2866 : int i;
1086 :
1087 14506 : for (i = 0; i < ira_pressure_classes_num; i++)
1088 11640 : curr_reg_pressure[ira_pressure_classes[i]]
1089 11640 : = saved_reg_pressure[ira_pressure_classes[i]];
1090 2866 : bitmap_copy (curr_reg_live, saved_reg_live);
1091 2866 : }
1092 :
1093 : /* Return TRUE if the register is dying after its USE. */
1094 : static bool
1095 32408 : dying_use_p (struct reg_use_data *use)
1096 : {
1097 32408 : struct reg_use_data *next;
1098 :
1099 39055 : for (next = use->next_regno_use; next != use; next = next->next_regno_use)
1100 15286 : if (NONDEBUG_INSN_P (next->insn)
1101 15286 : && QUEUE_INDEX (next->insn) != QUEUE_SCHEDULED)
1102 : return false;
1103 : return true;
1104 : }
1105 :
1106 : /* Print info about the current register pressure and its excess for
1107 : each pressure class. */
1108 : static void
1109 0 : print_curr_reg_pressure (void)
1110 : {
1111 0 : int i;
1112 0 : enum reg_class cl;
1113 :
1114 0 : fprintf (sched_dump, ";;\t");
1115 0 : for (i = 0; i < ira_pressure_classes_num; i++)
1116 : {
1117 0 : cl = ira_pressure_classes[i];
1118 0 : gcc_assert (curr_reg_pressure[cl] >= 0);
1119 0 : fprintf (sched_dump, " %s:%d(%d)", reg_class_names[cl],
1120 : curr_reg_pressure[cl],
1121 0 : curr_reg_pressure[cl] - sched_class_regs_num[cl]);
1122 : }
1123 0 : fprintf (sched_dump, "\n");
1124 0 : }
1125 :
1126 : /* Determine if INSN has a condition that is clobbered if a register
1127 : in SET_REGS is modified. */
1128 : static bool
1129 0 : cond_clobbered_p (rtx_insn *insn, HARD_REG_SET set_regs)
1130 : {
1131 0 : rtx pat = PATTERN (insn);
1132 0 : gcc_assert (GET_CODE (pat) == COND_EXEC);
1133 0 : if (TEST_HARD_REG_BIT (set_regs, REGNO (XEXP (COND_EXEC_TEST (pat), 0))))
1134 : {
1135 0 : sd_iterator_def sd_it;
1136 0 : dep_t dep;
1137 0 : haifa_change_pattern (insn, ORIG_PAT (insn));
1138 0 : FOR_EACH_DEP (insn, SD_LIST_BACK, sd_it, dep)
1139 0 : DEP_STATUS (dep) &= ~DEP_CANCELLED;
1140 0 : TODO_SPEC (insn) = HARD_DEP;
1141 0 : if (sched_verbose >= 2)
1142 0 : fprintf (sched_dump,
1143 : ";;\t\tdequeue insn %s because of clobbered condition\n",
1144 0 : (*current_sched_info->print_insn) (insn, 0));
1145 0 : return true;
1146 : }
1147 :
1148 : return false;
1149 : }
1150 :
1151 : /* This function should be called after modifying the pattern of INSN,
1152 : to update scheduler data structures as needed. */
1153 : static void
1154 669784 : update_insn_after_change (rtx_insn *insn)
1155 : {
1156 669784 : sd_iterator_def sd_it;
1157 669784 : dep_t dep;
1158 :
1159 669784 : dfa_clear_single_insn_cache (insn);
1160 :
1161 669784 : sd_it = sd_iterator_start (insn,
1162 : SD_LIST_FORW | SD_LIST_BACK | SD_LIST_RES_BACK);
1163 7350911 : while (sd_iterator_cond (&sd_it, &dep))
1164 : {
1165 6681127 : DEP_COST (dep) = UNKNOWN_DEP_COST;
1166 6681127 : sd_iterator_next (&sd_it);
1167 : }
1168 :
1169 : /* Invalidate INSN_COST, so it'll be recalculated. */
1170 669784 : INSN_COST (insn) = -1;
1171 : /* Invalidate INSN_TICK, so it'll be recalculated. */
1172 669784 : INSN_TICK (insn) = INVALID_TICK;
1173 :
1174 : /* Invalidate autoprefetch data entry. */
1175 669784 : INSN_AUTOPREF_MULTIPASS_DATA (insn)[0].status
1176 669784 : = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED;
1177 669784 : INSN_AUTOPREF_MULTIPASS_DATA (insn)[1].status
1178 669784 : = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED;
1179 669784 : }
1180 :
1181 :
1182 : /* Two VECs, one to hold dependencies for which pattern replacements
1183 : need to be applied or restored at the start of the next cycle, and
1184 : another to hold an integer that is either one, to apply the
1185 : corresponding replacement, or zero to restore it. */
1186 : static vec<dep_t> next_cycle_replace_deps;
1187 : static vec<int> next_cycle_apply;
1188 :
1189 : static void apply_replacement (dep_t, bool);
1190 : static void restore_pattern (dep_t, bool);
1191 :
1192 : /* Look at the remaining dependencies for insn NEXT, and compute and return
1193 : the TODO_SPEC value we should use for it. This is called after one of
1194 : NEXT's dependencies has been resolved.
1195 : We also perform pattern replacements for predication, and for broken
1196 : replacement dependencies. The latter is only done if FOR_BACKTRACK is
1197 : false. */
1198 :
1199 : static ds_t
1200 325168464 : recompute_todo_spec (rtx_insn *next, bool for_backtrack)
1201 : {
1202 325168464 : ds_t new_ds;
1203 325168464 : sd_iterator_def sd_it;
1204 325168464 : dep_t dep, modify_dep = NULL;
1205 325168464 : int n_spec = 0;
1206 325168464 : int n_control = 0;
1207 325168464 : int n_replace = 0;
1208 325168464 : bool first_p = true;
1209 :
1210 325168464 : if (sd_lists_empty_p (next, SD_LIST_BACK))
1211 : /* NEXT has all its dependencies resolved. */
1212 : return 0;
1213 :
1214 211257385 : if (!sd_lists_empty_p (next, SD_LIST_HARD_BACK))
1215 : return HARD_DEP;
1216 :
1217 : /* If NEXT is intended to sit adjacent to this instruction, we don't
1218 : want to try to break any dependencies. Treat it as a HARD_DEP. */
1219 402442 : if (SCHED_GROUP_P (next))
1220 : return HARD_DEP;
1221 :
1222 : /* Now we've got NEXT with speculative deps only.
1223 : 1. Look at the deps to see what we have to do.
1224 : 2. Check if we can do 'todo'. */
1225 402442 : new_ds = 0;
1226 :
1227 1392161 : FOR_EACH_DEP (next, SD_LIST_BACK, sd_it, dep)
1228 : {
1229 989719 : rtx_insn *pro = DEP_PRO (dep);
1230 989719 : ds_t ds = DEP_STATUS (dep) & SPECULATIVE;
1231 :
1232 989719 : if (DEBUG_INSN_P (pro) && !DEBUG_INSN_P (next))
1233 139263 : continue;
1234 :
1235 850456 : if (ds)
1236 : {
1237 0 : n_spec++;
1238 0 : if (first_p)
1239 : {
1240 : first_p = false;
1241 :
1242 : new_ds = ds;
1243 : }
1244 : else
1245 0 : new_ds = ds_merge (new_ds, ds);
1246 : }
1247 850456 : else if (DEP_TYPE (dep) == REG_DEP_CONTROL)
1248 : {
1249 0 : if (QUEUE_INDEX (pro) != QUEUE_SCHEDULED)
1250 : {
1251 0 : n_control++;
1252 0 : modify_dep = dep;
1253 : }
1254 0 : DEP_STATUS (dep) &= ~DEP_CANCELLED;
1255 : }
1256 850456 : else if (DEP_REPLACE (dep) != NULL)
1257 : {
1258 850456 : if (QUEUE_INDEX (pro) != QUEUE_SCHEDULED)
1259 : {
1260 850456 : n_replace++;
1261 850456 : modify_dep = dep;
1262 : }
1263 850456 : DEP_STATUS (dep) &= ~DEP_CANCELLED;
1264 : }
1265 : }
1266 :
1267 402442 : if (n_replace > 0 && n_control == 0 && n_spec == 0)
1268 : {
1269 402442 : if (!dbg_cnt (sched_breakdep))
1270 : return HARD_DEP;
1271 1392161 : FOR_EACH_DEP (next, SD_LIST_BACK, sd_it, dep)
1272 : {
1273 989719 : struct dep_replacement *desc = DEP_REPLACE (dep);
1274 989719 : if (desc != NULL)
1275 : {
1276 850456 : if (desc->insn == next && !for_backtrack)
1277 : {
1278 89475 : gcc_assert (n_replace == 1);
1279 89475 : apply_replacement (dep, true);
1280 : }
1281 850456 : DEP_STATUS (dep) |= DEP_CANCELLED;
1282 : }
1283 : }
1284 : return 0;
1285 : }
1286 :
1287 0 : else if (n_control == 1 && n_replace == 0 && n_spec == 0)
1288 : {
1289 0 : rtx_insn *pro, *other;
1290 0 : rtx new_pat;
1291 0 : rtx cond = NULL_RTX;
1292 0 : bool success;
1293 0 : rtx_insn *prev = NULL;
1294 0 : int i;
1295 0 : unsigned regno;
1296 :
1297 0 : if ((current_sched_info->flags & DO_PREDICATION) == 0
1298 0 : || (ORIG_PAT (next) != NULL_RTX
1299 0 : && PREDICATED_PAT (next) == NULL_RTX))
1300 : return HARD_DEP;
1301 :
1302 0 : pro = DEP_PRO (modify_dep);
1303 0 : other = real_insn_for_shadow (pro);
1304 0 : if (other != NULL_RTX)
1305 0 : pro = other;
1306 :
1307 0 : cond = sched_get_reverse_condition_uncached (pro);
1308 0 : regno = REGNO (XEXP (cond, 0));
1309 :
1310 : /* Find the last scheduled insn that modifies the condition register.
1311 : We can stop looking once we find the insn we depend on through the
1312 : REG_DEP_CONTROL; if the condition register isn't modified after it,
1313 : we know that it still has the right value. */
1314 0 : if (QUEUE_INDEX (pro) == QUEUE_SCHEDULED)
1315 0 : FOR_EACH_VEC_ELT_REVERSE (scheduled_insns, i, prev)
1316 : {
1317 0 : HARD_REG_SET t;
1318 :
1319 0 : find_all_hard_reg_sets (prev, &t, true);
1320 0 : if (TEST_HARD_REG_BIT (t, regno))
1321 0 : return HARD_DEP;
1322 0 : if (prev == pro)
1323 : break;
1324 : }
1325 0 : if (ORIG_PAT (next) == NULL_RTX)
1326 : {
1327 0 : ORIG_PAT (next) = PATTERN (next);
1328 :
1329 0 : new_pat = gen_rtx_COND_EXEC (VOIDmode, cond, PATTERN (next));
1330 0 : success = haifa_change_pattern (next, new_pat);
1331 0 : if (!success)
1332 : return HARD_DEP;
1333 0 : PREDICATED_PAT (next) = new_pat;
1334 : }
1335 0 : else if (PATTERN (next) != PREDICATED_PAT (next))
1336 : {
1337 0 : bool success = haifa_change_pattern (next,
1338 0 : PREDICATED_PAT (next));
1339 0 : gcc_assert (success);
1340 : }
1341 0 : DEP_STATUS (modify_dep) |= DEP_CANCELLED;
1342 0 : return DEP_CONTROL;
1343 : }
1344 :
1345 0 : if (PREDICATED_PAT (next) != NULL_RTX)
1346 : {
1347 0 : int tick = INSN_TICK (next);
1348 0 : bool success = haifa_change_pattern (next,
1349 0 : ORIG_PAT (next));
1350 0 : INSN_TICK (next) = tick;
1351 0 : gcc_assert (success);
1352 : }
1353 :
1354 : /* We can't handle the case where there are both speculative and control
1355 : dependencies, so we return HARD_DEP in such a case. Also fail if
1356 : we have speculative dependencies with not enough points, or more than
1357 : one control dependency. */
1358 0 : if ((n_spec > 0 && (n_control > 0 || n_replace > 0))
1359 0 : || (n_spec > 0
1360 : /* Too few points? */
1361 0 : && ds_weak (new_ds) < spec_info->data_weakness_cutoff)
1362 0 : || n_control > 0
1363 0 : || n_replace > 0)
1364 0 : return HARD_DEP;
1365 :
1366 : return new_ds;
1367 : }
1368 :
1369 : /* Pointer to the last instruction scheduled. */
1370 : static rtx_insn *last_scheduled_insn;
1371 :
1372 : /* Pointer to the last nondebug instruction scheduled within the
1373 : block, or the prev_head of the scheduling block. Used by
1374 : rank_for_schedule, so that insns independent of the last scheduled
1375 : insn will be preferred over dependent instructions. */
1376 : static rtx_insn *last_nondebug_scheduled_insn;
1377 :
1378 : /* Pointer that iterates through the list of unscheduled insns if we
1379 : have a dbg_cnt enabled. It always points at an insn prior to the
1380 : first unscheduled one. */
1381 : static rtx_insn *nonscheduled_insns_begin;
1382 :
1383 : /* Compute cost of executing INSN.
1384 : This is the number of cycles between instruction issue and
1385 : instruction results. */
1386 : int
1387 219431874 : insn_sched_cost (rtx_insn *insn)
1388 : {
1389 219431874 : int cost;
1390 :
1391 219431874 : if (sched_fusion)
1392 : return 0;
1393 :
1394 219431874 : if (sel_sched_p ())
1395 : {
1396 36651 : if (recog_memoized (insn) < 0)
1397 : return 0;
1398 :
1399 36042 : cost = insn_default_latency (insn);
1400 36042 : if (cost < 0)
1401 : cost = 0;
1402 :
1403 36042 : return cost;
1404 : }
1405 :
1406 219395223 : cost = INSN_COST (insn);
1407 :
1408 219395223 : if (cost < 0)
1409 : {
1410 : /* A USE insn, or something else we don't need to
1411 : understand. We can't pass these directly to
1412 : result_ready_cost or insn_default_latency because it will
1413 : trigger a fatal error for unrecognizable insns. */
1414 114902052 : if (recog_memoized (insn) < 0)
1415 : {
1416 53703967 : INSN_COST (insn) = 0;
1417 53703967 : return 0;
1418 : }
1419 : else
1420 : {
1421 61198085 : cost = insn_default_latency (insn);
1422 61198085 : if (cost < 0)
1423 : cost = 0;
1424 :
1425 61198085 : INSN_COST (insn) = cost;
1426 : }
1427 : }
1428 :
1429 : return cost;
1430 : }
1431 :
1432 : /* Compute cost of dependence LINK.
1433 : This is the number of cycles between instruction issue and
1434 : instruction results.
1435 : ??? We also use this function to call recog_memoized on all insns. */
1436 : int
1437 335761444 : dep_cost_1 (dep_t link, dw_t dw)
1438 : {
1439 335761444 : rtx_insn *insn = DEP_PRO (link);
1440 335761444 : rtx_insn *used = DEP_CON (link);
1441 335761444 : int cost;
1442 :
1443 335761444 : if (DEP_COST (link) != UNKNOWN_DEP_COST)
1444 179449669 : return DEP_COST (link);
1445 :
1446 156311775 : if (delay_htab)
1447 : {
1448 0 : struct delay_pair *delay_entry;
1449 0 : delay_entry
1450 0 : = delay_htab_i2->find_with_hash (used, htab_hash_pointer (used));
1451 0 : if (delay_entry)
1452 : {
1453 0 : if (delay_entry->i1 == insn)
1454 : {
1455 0 : DEP_COST (link) = pair_delay (delay_entry);
1456 0 : return DEP_COST (link);
1457 : }
1458 : }
1459 : }
1460 :
1461 : /* A USE insn should never require the value used to be computed.
1462 : This allows the computation of a function's result and parameter
1463 : values to overlap the return and call. We don't care about the
1464 : dependence cost when only decreasing register pressure. */
1465 156311775 : if (recog_memoized (used) < 0)
1466 : {
1467 1352419 : cost = 0;
1468 1352419 : recog_memoized (insn);
1469 : }
1470 : else
1471 : {
1472 154959356 : enum reg_note dep_type = DEP_TYPE (link);
1473 :
1474 154959356 : cost = insn_sched_cost (insn);
1475 :
1476 154959356 : if (INSN_CODE (insn) >= 0)
1477 : {
1478 149752962 : if (dep_type == REG_DEP_ANTI)
1479 : cost = 0;
1480 79434966 : else if (dep_type == REG_DEP_OUTPUT)
1481 : {
1482 24648088 : cost = (insn_default_latency (insn)
1483 24648088 : - insn_default_latency (used));
1484 24648088 : if (cost <= 0)
1485 : cost = 1;
1486 : }
1487 54786878 : else if (bypass_p (insn))
1488 1568 : cost = insn_latency (insn, used);
1489 : }
1490 :
1491 :
1492 154959356 : if (targetm.sched.adjust_cost)
1493 154959356 : cost = targetm.sched.adjust_cost (used, (int) dep_type, insn, cost,
1494 : dw);
1495 :
1496 154959356 : if (cost < 0)
1497 1352419 : cost = 0;
1498 : }
1499 :
1500 156311775 : DEP_COST (link) = cost;
1501 156311775 : return cost;
1502 : }
1503 :
1504 : /* Compute cost of dependence LINK.
1505 : This is the number of cycles between instruction issue and
1506 : instruction results. */
1507 : int
1508 335732071 : dep_cost (dep_t link)
1509 : {
1510 335732071 : return dep_cost_1 (link, 0);
1511 : }
1512 :
1513 : /* Use this sel-sched.cc friendly function in reorder2 instead of increasing
1514 : INSN_PRIORITY explicitly. */
1515 : void
1516 0 : increase_insn_priority (rtx_insn *insn, int amount)
1517 : {
1518 0 : if (!sel_sched_p ())
1519 : {
1520 : /* We're dealing with haifa-sched.cc INSN_PRIORITY. */
1521 0 : if (INSN_PRIORITY_KNOWN (insn))
1522 0 : INSN_PRIORITY (insn) += amount;
1523 : }
1524 : else
1525 : {
1526 : /* In sel-sched.cc INSN_PRIORITY is not kept up to date.
1527 : Use EXPR_PRIORITY instead. */
1528 0 : sel_add_to_insn_priority (insn, amount);
1529 : }
1530 0 : }
1531 :
1532 : /* Return 'true' if DEP should be included in priority calculations. */
1533 : static bool
1534 159847013 : contributes_to_priority_p (dep_t dep)
1535 : {
1536 159847013 : if (DEBUG_INSN_P (DEP_CON (dep))
1537 146172553 : || DEBUG_INSN_P (DEP_PRO (dep)))
1538 : return false;
1539 :
1540 : /* Critical path is meaningful in block boundaries only. */
1541 146172553 : if (!current_sched_info->contributes_to_priority (DEP_CON (dep),
1542 : DEP_PRO (dep)))
1543 : return false;
1544 :
1545 146168658 : if (DEP_REPLACE (dep) != NULL)
1546 : return false;
1547 :
1548 : /* If flag COUNT_SPEC_IN_CRITICAL_PATH is set,
1549 : then speculative instructions will less likely be
1550 : scheduled. That is because the priority of
1551 : their producers will increase, and, thus, the
1552 : producers will more likely be scheduled, thus,
1553 : resolving the dependence. */
1554 145507948 : if (sched_deps_info->generate_spec_deps
1555 0 : && !(spec_info->flags & COUNT_SPEC_IN_CRITICAL_PATH)
1556 0 : && (DEP_STATUS (dep) & SPECULATIVE))
1557 0 : return false;
1558 :
1559 : return true;
1560 : }
1561 :
1562 : /* Compute the number of nondebug deps in list LIST for INSN. */
1563 : int
1564 673939386 : dep_list_size (rtx_insn *insn, sd_list_types_def list)
1565 : {
1566 673939386 : sd_iterator_def sd_it;
1567 673939386 : dep_t dep;
1568 673939386 : int dbgcount = 0, nodbgcount = 0;
1569 :
1570 673939386 : if (!MAY_HAVE_DEBUG_INSNS)
1571 285225285 : return sd_lists_size (insn, list);
1572 :
1573 : /* TODO: We should split normal and debug insns into separate SD_LIST_*
1574 : sub-lists, and then we'll be able to use something like
1575 : sd_lists_size(insn, list & SD_LIST_NON_DEBUG)
1576 : instead of walking dependencies below. */
1577 :
1578 1353219174 : FOR_EACH_DEP (insn, list, sd_it, dep)
1579 : {
1580 964505073 : if (DEBUG_INSN_P (DEP_CON (dep)))
1581 103937798 : dbgcount++;
1582 860567275 : else if (!DEBUG_INSN_P (DEP_PRO (dep)))
1583 852783289 : nodbgcount++;
1584 : }
1585 :
1586 388714101 : gcc_assert (dbgcount + nodbgcount == sd_lists_size (insn, list));
1587 :
1588 : return nodbgcount;
1589 : }
1590 :
1591 : bool sched_fusion;
1592 :
1593 : /* Compute the priority number for INSN. */
1594 : static int
1595 260434455 : priority (rtx_insn *insn, bool force_recompute)
1596 : {
1597 260434455 : if (! INSN_P (insn))
1598 : return 0;
1599 :
1600 : /* We should not be interested in priority of an already scheduled insn. */
1601 260434455 : gcc_assert (QUEUE_INDEX (insn) != QUEUE_SCHEDULED);
1602 :
1603 260434455 : if (force_recompute || !INSN_PRIORITY_KNOWN (insn))
1604 : {
1605 114926387 : int this_priority = -1;
1606 :
1607 114926387 : if (sched_fusion)
1608 : {
1609 0 : int this_fusion_priority;
1610 :
1611 0 : targetm.sched.fusion_priority (insn, FUSION_MAX_PRIORITY,
1612 : &this_fusion_priority, &this_priority);
1613 0 : INSN_FUSION_PRIORITY (insn) = this_fusion_priority;
1614 : }
1615 114926387 : else if (dep_list_size (insn, SD_LIST_FORW) == 0)
1616 : /* ??? We should set INSN_PRIORITY to insn_sched_cost when and insn
1617 : has some forward deps but all of them are ignored by
1618 : contributes_to_priority hook. At the moment we set priority of
1619 : such insn to 0. */
1620 64462248 : this_priority = insn_sched_cost (insn);
1621 : else
1622 : {
1623 50464139 : rtx_insn *prev_first, *twin;
1624 50464139 : basic_block rec;
1625 :
1626 : /* For recovery check instructions we calculate priority slightly
1627 : different than that of normal instructions. Instead of walking
1628 : through INSN_FORW_DEPS (check) list, we walk through
1629 : INSN_FORW_DEPS list of each instruction in the corresponding
1630 : recovery block. */
1631 :
1632 : /* Selective scheduling does not define RECOVERY_BLOCK macro. */
1633 50464139 : rec = sel_sched_p () ? NULL : RECOVERY_BLOCK (insn);
1634 50460828 : if (!rec || rec == EXIT_BLOCK_PTR_FOR_FN (cfun))
1635 : {
1636 50464139 : prev_first = PREV_INSN (insn);
1637 50464139 : twin = insn;
1638 : }
1639 : else
1640 : {
1641 0 : prev_first = NEXT_INSN (BB_HEAD (rec));
1642 0 : twin = PREV_INSN (BB_END (rec));
1643 : }
1644 :
1645 50464139 : do
1646 : {
1647 50464139 : sd_iterator_def sd_it;
1648 50464139 : dep_t dep;
1649 :
1650 210311152 : FOR_EACH_DEP (twin, SD_LIST_FORW, sd_it, dep)
1651 : {
1652 159847013 : rtx_insn *next;
1653 159847013 : int next_priority;
1654 :
1655 159847013 : next = DEP_CON (dep);
1656 :
1657 159847013 : if (BLOCK_FOR_INSN (next) != rec)
1658 : {
1659 159847013 : int cost;
1660 :
1661 159847013 : if (!contributes_to_priority_p (dep))
1662 14339065 : continue;
1663 :
1664 145507948 : if (twin == insn)
1665 145507948 : cost = dep_cost (dep);
1666 : else
1667 : {
1668 0 : struct _dep _dep1, *dep1 = &_dep1;
1669 :
1670 0 : init_dep (dep1, insn, next, REG_DEP_ANTI);
1671 :
1672 0 : cost = dep_cost (dep1);
1673 : }
1674 :
1675 145507948 : next_priority = cost + priority (next);
1676 :
1677 145507948 : if (next_priority > this_priority)
1678 104151830 : this_priority = next_priority;
1679 : }
1680 : }
1681 :
1682 50464139 : twin = PREV_INSN (twin);
1683 : }
1684 50464139 : while (twin != prev_first);
1685 : }
1686 :
1687 114926387 : if (this_priority < 0)
1688 : {
1689 10270 : gcc_assert (this_priority == -1);
1690 :
1691 10270 : this_priority = insn_sched_cost (insn);
1692 : }
1693 :
1694 114926387 : INSN_PRIORITY (insn) = this_priority;
1695 114926387 : INSN_PRIORITY_STATUS (insn) = 1;
1696 : }
1697 :
1698 260434455 : return INSN_PRIORITY (insn);
1699 : }
1700 :
1701 : /* Macros and functions for keeping the priority queue sorted, and
1702 : dealing with queuing and dequeuing of instructions. */
1703 :
1704 : /* For each pressure class CL, set DEATH[CL] to the number of registers
1705 : in that class that die in INSN. */
1706 :
1707 : static void
1708 21766 : calculate_reg_deaths (rtx_insn *insn, int *death)
1709 : {
1710 21766 : int i;
1711 21766 : struct reg_use_data *use;
1712 :
1713 117664 : for (i = 0; i < ira_pressure_classes_num; i++)
1714 95898 : death[ira_pressure_classes[i]] = 0;
1715 36602 : for (use = INSN_REG_USE_LIST (insn); use != NULL; use = use->next_insn_use)
1716 14836 : if (dying_use_p (use))
1717 8449 : mark_regno_birth_or_death (0, death, use->regno, true);
1718 21766 : }
1719 :
1720 : /* Setup info about the current register pressure impact of scheduling
1721 : INSN at the current scheduling point. */
1722 : static void
1723 21766 : setup_insn_reg_pressure_info (rtx_insn *insn)
1724 : {
1725 21766 : int i, change, before, after, hard_regno;
1726 21766 : int excess_cost_change;
1727 21766 : machine_mode mode;
1728 21766 : enum reg_class cl;
1729 21766 : struct reg_pressure_data *pressure_info;
1730 21766 : int *max_reg_pressure;
1731 21766 : static int death[N_REG_CLASSES];
1732 :
1733 21766 : gcc_checking_assert (!DEBUG_INSN_P (insn));
1734 :
1735 21766 : excess_cost_change = 0;
1736 21766 : calculate_reg_deaths (insn, death);
1737 21766 : pressure_info = INSN_REG_PRESSURE (insn);
1738 21766 : max_reg_pressure = INSN_MAX_REG_PRESSURE (insn);
1739 21766 : gcc_assert (pressure_info != NULL && max_reg_pressure != NULL);
1740 117664 : for (i = 0; i < ira_pressure_classes_num; i++)
1741 : {
1742 95898 : cl = ira_pressure_classes[i];
1743 95898 : gcc_assert (curr_reg_pressure[cl] >= 0);
1744 95898 : change = (int) pressure_info[i].set_increase - death[cl];
1745 95898 : before = MAX (0, max_reg_pressure[i] - sched_class_regs_num[cl]);
1746 95898 : after = MAX (0, max_reg_pressure[i] + change
1747 : - sched_class_regs_num[cl]);
1748 95898 : hard_regno = ira_class_hard_regs[cl][0];
1749 95898 : gcc_assert (hard_regno >= 0);
1750 95898 : mode = reg_raw_mode[hard_regno];
1751 95898 : excess_cost_change += ((after - before)
1752 95898 : * (ira_memory_move_cost[mode][cl][0]
1753 95898 : + ira_memory_move_cost[mode][cl][1]));
1754 : }
1755 21766 : INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insn) = excess_cost_change;
1756 21766 : }
1757 :
1758 : /* This is the first page of code related to SCHED_PRESSURE_MODEL.
1759 : It tries to make the scheduler take register pressure into account
1760 : without introducing too many unnecessary stalls. It hooks into the
1761 : main scheduling algorithm at several points:
1762 :
1763 : - Before scheduling starts, model_start_schedule constructs a
1764 : "model schedule" for the current block. This model schedule is
1765 : chosen solely to keep register pressure down. It does not take the
1766 : target's pipeline or the original instruction order into account,
1767 : except as a tie-breaker. It also doesn't work to a particular
1768 : pressure limit.
1769 :
1770 : This model schedule gives us an idea of what pressure can be
1771 : achieved for the block and gives us an example of a schedule that
1772 : keeps to that pressure. It also makes the final schedule less
1773 : dependent on the original instruction order. This is important
1774 : because the original order can either be "wide" (many values live
1775 : at once, such as in user-scheduled code) or "narrow" (few values
1776 : live at once, such as after loop unrolling, where several
1777 : iterations are executed sequentially).
1778 :
1779 : We do not apply this model schedule to the rtx stream. We simply
1780 : record it in model_schedule. We also compute the maximum pressure,
1781 : MP, that was seen during this schedule.
1782 :
1783 : - Instructions are added to the ready queue even if they require
1784 : a stall. The length of the stall is instead computed as:
1785 :
1786 : MAX (INSN_TICK (INSN) - clock_var, 0)
1787 :
1788 : (= insn_delay). This allows rank_for_schedule to choose between
1789 : introducing a deliberate stall or increasing pressure.
1790 :
1791 : - Before sorting the ready queue, model_set_excess_costs assigns
1792 : a pressure-based cost to each ready instruction in the queue.
1793 : This is the instruction's INSN_REG_PRESSURE_EXCESS_COST_CHANGE
1794 : (ECC for short) and is effectively measured in cycles.
1795 :
1796 : - rank_for_schedule ranks instructions based on:
1797 :
1798 : ECC (insn) + insn_delay (insn)
1799 :
1800 : then as:
1801 :
1802 : insn_delay (insn)
1803 :
1804 : So, for example, an instruction X1 with an ECC of 1 that can issue
1805 : now will win over an instruction X0 with an ECC of zero that would
1806 : introduce a stall of one cycle. However, an instruction X2 with an
1807 : ECC of 2 that can issue now will lose to both X0 and X1.
1808 :
1809 : - When an instruction is scheduled, model_recompute updates the model
1810 : schedule with the new pressures (some of which might now exceed the
1811 : original maximum pressure MP). model_update_limit_points then searches
1812 : for the new point of maximum pressure, if not already known. */
1813 :
1814 : /* Used to separate high-verbosity debug information for SCHED_PRESSURE_MODEL
1815 : from surrounding debug information. */
1816 : #define MODEL_BAR \
1817 : ";;\t\t+------------------------------------------------------\n"
1818 :
1819 : /* Information about the pressure on a particular register class at a
1820 : particular point of the model schedule. */
1821 : struct model_pressure_data {
1822 : /* The pressure at this point of the model schedule, or -1 if the
1823 : point is associated with an instruction that has already been
1824 : scheduled. */
1825 : int ref_pressure;
1826 :
1827 : /* The maximum pressure during or after this point of the model schedule. */
1828 : int max_pressure;
1829 : };
1830 :
1831 : /* Per-instruction information that is used while building the model
1832 : schedule. Here, "schedule" refers to the model schedule rather
1833 : than the main schedule. */
1834 : struct model_insn_info {
1835 : /* The instruction itself. */
1836 : rtx_insn *insn;
1837 :
1838 : /* If this instruction is in model_worklist, these fields link to the
1839 : previous (higher-priority) and next (lower-priority) instructions
1840 : in the list. */
1841 : struct model_insn_info *prev;
1842 : struct model_insn_info *next;
1843 :
1844 : /* While constructing the schedule, QUEUE_INDEX describes whether an
1845 : instruction has already been added to the schedule (QUEUE_SCHEDULED),
1846 : is in model_worklist (QUEUE_READY), or neither (QUEUE_NOWHERE).
1847 : old_queue records the value that QUEUE_INDEX had before scheduling
1848 : started, so that we can restore it once the schedule is complete. */
1849 : int old_queue;
1850 :
1851 : /* The relative importance of an unscheduled instruction. Higher
1852 : values indicate greater importance. */
1853 : unsigned int model_priority;
1854 :
1855 : /* The length of the longest path of satisfied true dependencies
1856 : that leads to this instruction. */
1857 : unsigned int depth;
1858 :
1859 : /* The length of the longest path of dependencies of any kind
1860 : that leads from this instruction. */
1861 : unsigned int alap;
1862 :
1863 : /* The number of predecessor nodes that must still be scheduled. */
1864 : int unscheduled_preds;
1865 : };
1866 :
1867 : /* Information about the pressure limit for a particular register class.
1868 : This structure is used when applying a model schedule to the main
1869 : schedule. */
1870 : struct model_pressure_limit {
1871 : /* The maximum register pressure seen in the original model schedule. */
1872 : int orig_pressure;
1873 :
1874 : /* The maximum register pressure seen in the current model schedule
1875 : (which excludes instructions that have already been scheduled). */
1876 : int pressure;
1877 :
1878 : /* The point of the current model schedule at which PRESSURE is first
1879 : reached. It is set to -1 if the value needs to be recomputed. */
1880 : int point;
1881 : };
1882 :
1883 : /* Describes a particular way of measuring register pressure. */
1884 : struct model_pressure_group {
1885 : /* Index PCI describes the maximum pressure on ira_pressure_classes[PCI]. */
1886 : struct model_pressure_limit limits[N_REG_CLASSES];
1887 :
1888 : /* Index (POINT * ira_num_pressure_classes + PCI) describes the pressure
1889 : on register class ira_pressure_classes[PCI] at point POINT of the
1890 : current model schedule. A POINT of model_num_insns describes the
1891 : pressure at the end of the schedule. */
1892 : struct model_pressure_data *model;
1893 : };
1894 :
1895 : /* Index POINT gives the instruction at point POINT of the model schedule.
1896 : This array doesn't change during main scheduling. */
1897 : static vec<rtx_insn *> model_schedule;
1898 :
1899 : /* The list of instructions in the model worklist, sorted in order of
1900 : decreasing priority. */
1901 : static struct model_insn_info *model_worklist;
1902 :
1903 : /* Index I describes the instruction with INSN_LUID I. */
1904 : static struct model_insn_info *model_insns;
1905 :
1906 : /* The number of instructions in the model schedule. */
1907 : static int model_num_insns;
1908 :
1909 : /* The index of the first instruction in model_schedule that hasn't yet been
1910 : added to the main schedule, or model_num_insns if all of them have. */
1911 : static int model_curr_point;
1912 :
1913 : /* Describes the pressure before each instruction in the model schedule. */
1914 : static struct model_pressure_group model_before_pressure;
1915 :
1916 : /* The first unused model_priority value (as used in model_insn_info). */
1917 : static unsigned int model_next_priority;
1918 :
1919 :
1920 : /* The model_pressure_data for ira_pressure_classes[PCI] in GROUP
1921 : at point POINT of the model schedule. */
1922 : #define MODEL_PRESSURE_DATA(GROUP, POINT, PCI) \
1923 : (&(GROUP)->model[(POINT) * ira_pressure_classes_num + (PCI)])
1924 :
1925 : /* The maximum pressure on ira_pressure_classes[PCI] in GROUP at or
1926 : after point POINT of the model schedule. */
1927 : #define MODEL_MAX_PRESSURE(GROUP, POINT, PCI) \
1928 : (MODEL_PRESSURE_DATA (GROUP, POINT, PCI)->max_pressure)
1929 :
1930 : /* The pressure on ira_pressure_classes[PCI] in GROUP at point POINT
1931 : of the model schedule. */
1932 : #define MODEL_REF_PRESSURE(GROUP, POINT, PCI) \
1933 : (MODEL_PRESSURE_DATA (GROUP, POINT, PCI)->ref_pressure)
1934 :
1935 : /* Information about INSN that is used when creating the model schedule. */
1936 : #define MODEL_INSN_INFO(INSN) \
1937 : (&model_insns[INSN_LUID (INSN)])
1938 :
1939 : /* The instruction at point POINT of the model schedule. */
1940 : #define MODEL_INSN(POINT) \
1941 : (model_schedule[POINT])
1942 :
1943 :
1944 : /* Return INSN's index in the model schedule, or model_num_insns if it
1945 : doesn't belong to that schedule. */
1946 :
1947 : static int
1948 0 : model_index (rtx_insn *insn)
1949 : {
1950 0 : if (INSN_MODEL_INDEX (insn) == 0)
1951 0 : return model_num_insns;
1952 0 : return INSN_MODEL_INDEX (insn) - 1;
1953 : }
1954 :
1955 : /* Make sure that GROUP->limits is up-to-date for the current point
1956 : of the model schedule. */
1957 :
1958 : static void
1959 0 : model_update_limit_points_in_group (struct model_pressure_group *group)
1960 : {
1961 0 : int pci, max_pressure, point;
1962 :
1963 0 : for (pci = 0; pci < ira_pressure_classes_num; pci++)
1964 : {
1965 : /* We may have passed the final point at which the pressure in
1966 : group->limits[pci].pressure was reached. Update the limit if so. */
1967 0 : max_pressure = MODEL_MAX_PRESSURE (group, model_curr_point, pci);
1968 0 : group->limits[pci].pressure = max_pressure;
1969 :
1970 : /* Find the point at which MAX_PRESSURE is first reached. We need
1971 : to search in three cases:
1972 :
1973 : - We've already moved past the previous pressure point.
1974 : In this case we search forward from model_curr_point.
1975 :
1976 : - We scheduled the previous point of maximum pressure ahead of
1977 : its position in the model schedule, but doing so didn't bring
1978 : the pressure point earlier. In this case we search forward
1979 : from that previous pressure point.
1980 :
1981 : - Scheduling an instruction early caused the maximum pressure
1982 : to decrease. In this case we will have set the pressure
1983 : point to -1, and we search forward from model_curr_point. */
1984 0 : point = MAX (group->limits[pci].point, model_curr_point);
1985 0 : while (point < model_num_insns
1986 0 : && MODEL_REF_PRESSURE (group, point, pci) < max_pressure)
1987 0 : point++;
1988 0 : group->limits[pci].point = point;
1989 :
1990 0 : gcc_assert (MODEL_REF_PRESSURE (group, point, pci) == max_pressure);
1991 0 : gcc_assert (MODEL_MAX_PRESSURE (group, point, pci) == max_pressure);
1992 : }
1993 0 : }
1994 :
1995 : /* Make sure that all register-pressure limits are up-to-date for the
1996 : current position in the model schedule. */
1997 :
1998 : static void
1999 0 : model_update_limit_points (void)
2000 : {
2001 0 : model_update_limit_points_in_group (&model_before_pressure);
2002 0 : }
2003 :
2004 : /* Return the model_index of the last unscheduled use in chain USE
2005 : outside of USE's instruction. Return -1 if there are no other uses,
2006 : or model_num_insns if the register is live at the end of the block. */
2007 :
2008 : static int
2009 0 : model_last_use_except (struct reg_use_data *use)
2010 : {
2011 0 : struct reg_use_data *next;
2012 0 : int last, index;
2013 :
2014 0 : last = -1;
2015 0 : for (next = use->next_regno_use; next != use; next = next->next_regno_use)
2016 0 : if (NONDEBUG_INSN_P (next->insn)
2017 0 : && QUEUE_INDEX (next->insn) != QUEUE_SCHEDULED)
2018 : {
2019 0 : index = model_index (next->insn);
2020 0 : if (index == model_num_insns)
2021 : return model_num_insns;
2022 0 : if (last < index)
2023 0 : last = index;
2024 : }
2025 : return last;
2026 : }
2027 :
2028 : /* An instruction with model_index POINT has just been scheduled, and it
2029 : adds DELTA to the pressure on ira_pressure_classes[PCI] after POINT - 1.
2030 : Update MODEL_REF_PRESSURE (GROUP, POINT, PCI) and
2031 : MODEL_MAX_PRESSURE (GROUP, POINT, PCI) accordingly. */
2032 :
2033 : static void
2034 0 : model_start_update_pressure (struct model_pressure_group *group,
2035 : int point, int pci, int delta)
2036 : {
2037 0 : int next_max_pressure;
2038 :
2039 0 : if (point == model_num_insns)
2040 : {
2041 : /* The instruction wasn't part of the model schedule; it was moved
2042 : from a different block. Update the pressure for the end of
2043 : the model schedule. */
2044 0 : MODEL_REF_PRESSURE (group, point, pci) += delta;
2045 0 : MODEL_MAX_PRESSURE (group, point, pci) += delta;
2046 : }
2047 : else
2048 : {
2049 : /* Record that this instruction has been scheduled. Nothing now
2050 : changes between POINT and POINT + 1, so get the maximum pressure
2051 : from the latter. If the maximum pressure decreases, the new
2052 : pressure point may be before POINT. */
2053 0 : MODEL_REF_PRESSURE (group, point, pci) = -1;
2054 0 : next_max_pressure = MODEL_MAX_PRESSURE (group, point + 1, pci);
2055 0 : if (MODEL_MAX_PRESSURE (group, point, pci) > next_max_pressure)
2056 : {
2057 0 : MODEL_MAX_PRESSURE (group, point, pci) = next_max_pressure;
2058 0 : if (group->limits[pci].point == point)
2059 0 : group->limits[pci].point = -1;
2060 : }
2061 : }
2062 0 : }
2063 :
2064 : /* Record that scheduling a later instruction has changed the pressure
2065 : at point POINT of the model schedule by DELTA (which might be 0).
2066 : Update GROUP accordingly. Return nonzero if these changes might
2067 : trigger changes to previous points as well. */
2068 :
2069 : static int
2070 0 : model_update_pressure (struct model_pressure_group *group,
2071 : int point, int pci, int delta)
2072 : {
2073 0 : int ref_pressure, max_pressure, next_max_pressure;
2074 :
2075 : /* If POINT hasn't yet been scheduled, update its pressure. */
2076 0 : ref_pressure = MODEL_REF_PRESSURE (group, point, pci);
2077 0 : if (ref_pressure >= 0 && delta != 0)
2078 : {
2079 0 : ref_pressure += delta;
2080 0 : MODEL_REF_PRESSURE (group, point, pci) = ref_pressure;
2081 :
2082 : /* Check whether the maximum pressure in the overall schedule
2083 : has increased. (This means that the MODEL_MAX_PRESSURE of
2084 : every point <= POINT will need to increase too; see below.) */
2085 0 : if (group->limits[pci].pressure < ref_pressure)
2086 0 : group->limits[pci].pressure = ref_pressure;
2087 :
2088 : /* If we are at maximum pressure, and the maximum pressure
2089 : point was previously unknown or later than POINT,
2090 : bring it forward. */
2091 0 : if (group->limits[pci].pressure == ref_pressure
2092 0 : && !IN_RANGE (group->limits[pci].point, 0, point))
2093 0 : group->limits[pci].point = point;
2094 :
2095 : /* If POINT used to be the point of maximum pressure, but isn't
2096 : any longer, we need to recalculate it using a forward walk. */
2097 0 : if (group->limits[pci].pressure > ref_pressure
2098 0 : && group->limits[pci].point == point)
2099 0 : group->limits[pci].point = -1;
2100 : }
2101 :
2102 : /* Update the maximum pressure at POINT. Changes here might also
2103 : affect the maximum pressure at POINT - 1. */
2104 0 : next_max_pressure = MODEL_MAX_PRESSURE (group, point + 1, pci);
2105 0 : max_pressure = MAX (ref_pressure, next_max_pressure);
2106 0 : if (MODEL_MAX_PRESSURE (group, point, pci) != max_pressure)
2107 : {
2108 0 : MODEL_MAX_PRESSURE (group, point, pci) = max_pressure;
2109 0 : return 1;
2110 : }
2111 : return 0;
2112 : }
2113 :
2114 : /* INSN has just been scheduled. Update the model schedule accordingly. */
2115 :
2116 : static void
2117 0 : model_recompute (rtx_insn *insn)
2118 : {
2119 0 : struct {
2120 : int last_use;
2121 : int regno;
2122 : } uses[FIRST_PSEUDO_REGISTER + MAX_RECOG_OPERANDS];
2123 0 : struct reg_use_data *use;
2124 0 : struct reg_pressure_data *reg_pressure;
2125 0 : int delta[N_REG_CLASSES];
2126 0 : int pci, point, mix, new_last, cl, ref_pressure, queue;
2127 0 : unsigned int i, num_uses, num_pending_births;
2128 0 : bool print_p;
2129 :
2130 : /* The destinations of INSN were previously live from POINT onwards, but are
2131 : now live from model_curr_point onwards. Set up DELTA accordingly. */
2132 0 : point = model_index (insn);
2133 0 : reg_pressure = INSN_REG_PRESSURE (insn);
2134 0 : for (pci = 0; pci < ira_pressure_classes_num; pci++)
2135 : {
2136 0 : cl = ira_pressure_classes[pci];
2137 0 : delta[cl] = reg_pressure[pci].set_increase;
2138 : }
2139 :
2140 : /* Record which registers previously died at POINT, but which now die
2141 : before POINT. Adjust DELTA so that it represents the effect of
2142 : this change after POINT - 1. Set NUM_PENDING_BIRTHS to the number of
2143 : registers that will be born in the range [model_curr_point, POINT). */
2144 0 : num_uses = 0;
2145 0 : num_pending_births = 0;
2146 0 : bitmap_clear (tmp_bitmap);
2147 0 : for (use = INSN_REG_USE_LIST (insn); use != NULL; use = use->next_insn_use)
2148 : {
2149 0 : new_last = model_last_use_except (use);
2150 0 : if (new_last < point
2151 0 : && bitmap_set_bit (tmp_bitmap, use->regno)
2152 : /* df_get_live_in has not necessarily been updated to reflect the
2153 : effect of inter-block movement performed by earlier schedules.
2154 : Cope with stale live-in sets by ignoring registers that are not
2155 : currently assumed to be live. */
2156 0 : && bitmap_bit_p (curr_reg_live, use->regno))
2157 : {
2158 0 : gcc_assert (num_uses < ARRAY_SIZE (uses));
2159 0 : uses[num_uses].last_use = new_last;
2160 0 : uses[num_uses].regno = use->regno;
2161 : /* This register is no longer live after POINT - 1. */
2162 0 : mark_regno_birth_or_death (NULL, delta, use->regno, false);
2163 0 : num_uses++;
2164 0 : if (new_last >= 0)
2165 0 : num_pending_births++;
2166 : }
2167 : }
2168 :
2169 : /* Update the MODEL_REF_PRESSURE and MODEL_MAX_PRESSURE for POINT.
2170 : Also set each group pressure limit for POINT. */
2171 0 : for (pci = 0; pci < ira_pressure_classes_num; pci++)
2172 : {
2173 0 : cl = ira_pressure_classes[pci];
2174 0 : model_start_update_pressure (&model_before_pressure,
2175 : point, pci, delta[cl]);
2176 : }
2177 :
2178 : /* Walk the model schedule backwards, starting immediately before POINT. */
2179 0 : print_p = false;
2180 0 : if (point != model_curr_point)
2181 0 : do
2182 : {
2183 0 : point--;
2184 0 : insn = MODEL_INSN (point);
2185 0 : queue = QUEUE_INDEX (insn);
2186 :
2187 0 : if (queue != QUEUE_SCHEDULED)
2188 : {
2189 : /* DELTA describes the effect of the move on the register pressure
2190 : after POINT. Make it describe the effect on the pressure
2191 : before POINT. */
2192 : i = 0;
2193 0 : while (i < num_uses)
2194 : {
2195 0 : if (uses[i].last_use == point)
2196 : {
2197 : /* This register is now live again. */
2198 0 : mark_regno_birth_or_death (NULL, delta,
2199 : uses[i].regno, true);
2200 :
2201 : /* Remove this use from the array. */
2202 0 : uses[i] = uses[num_uses - 1];
2203 0 : num_uses--;
2204 0 : num_pending_births--;
2205 : }
2206 : else
2207 0 : i++;
2208 : }
2209 :
2210 0 : if (sched_verbose >= 5)
2211 : {
2212 0 : if (!print_p)
2213 : {
2214 0 : fprintf (sched_dump, MODEL_BAR);
2215 0 : fprintf (sched_dump, ";;\t\t| New pressure for model"
2216 : " schedule\n");
2217 0 : fprintf (sched_dump, MODEL_BAR);
2218 0 : print_p = true;
2219 : }
2220 :
2221 0 : fprintf (sched_dump, ";;\t\t| %3d %4d %-30s ",
2222 0 : point, INSN_UID (insn),
2223 0 : str_pattern_slim (PATTERN (insn)));
2224 0 : for (pci = 0; pci < ira_pressure_classes_num; pci++)
2225 : {
2226 0 : cl = ira_pressure_classes[pci];
2227 0 : ref_pressure = MODEL_REF_PRESSURE (&model_before_pressure,
2228 : point, pci);
2229 0 : fprintf (sched_dump, " %s:[%d->%d]",
2230 : reg_class_names[ira_pressure_classes[pci]],
2231 0 : ref_pressure, ref_pressure + delta[cl]);
2232 : }
2233 0 : fprintf (sched_dump, "\n");
2234 : }
2235 : }
2236 :
2237 : /* Adjust the pressure at POINT. Set MIX to nonzero if POINT - 1
2238 : might have changed as well. */
2239 0 : mix = num_pending_births;
2240 0 : for (pci = 0; pci < ira_pressure_classes_num; pci++)
2241 : {
2242 0 : cl = ira_pressure_classes[pci];
2243 0 : mix |= delta[cl];
2244 0 : mix |= model_update_pressure (&model_before_pressure,
2245 : point, pci, delta[cl]);
2246 : }
2247 : }
2248 0 : while (mix && point > model_curr_point);
2249 :
2250 0 : if (print_p)
2251 0 : fprintf (sched_dump, MODEL_BAR);
2252 0 : }
2253 :
2254 : /* After DEP, which was cancelled, has been resolved for insn NEXT,
2255 : check whether the insn's pattern needs restoring. */
2256 : static bool
2257 850456 : must_restore_pattern_p (rtx_insn *next, dep_t dep)
2258 : {
2259 850456 : if (QUEUE_INDEX (next) == QUEUE_SCHEDULED)
2260 : return false;
2261 :
2262 302886 : if (DEP_TYPE (dep) == REG_DEP_CONTROL)
2263 : {
2264 0 : gcc_assert (ORIG_PAT (next) != NULL_RTX);
2265 0 : gcc_assert (next == DEP_CON (dep));
2266 : }
2267 : else
2268 : {
2269 302886 : struct dep_replacement *desc = DEP_REPLACE (dep);
2270 302886 : if (desc->insn != next)
2271 : {
2272 241779 : gcc_assert (*desc->loc == desc->orig);
2273 : return false;
2274 : }
2275 : }
2276 : return true;
2277 : }
2278 :
2279 : /* model_spill_cost (CL, P, P') returns the cost of increasing the
2280 : pressure on CL from P to P'. We use this to calculate a "base ECC",
2281 : baseECC (CL, X), for each pressure class CL and each instruction X.
2282 : Supposing X changes the pressure on CL from P to P', and that the
2283 : maximum pressure on CL in the current model schedule is MP', then:
2284 :
2285 : * if X occurs before or at the next point of maximum pressure in
2286 : the model schedule and P' > MP', then:
2287 :
2288 : baseECC (CL, X) = model_spill_cost (CL, MP, P')
2289 :
2290 : The idea is that the pressure after scheduling a fixed set of
2291 : instructions -- in this case, the set up to and including the
2292 : next maximum pressure point -- is going to be the same regardless
2293 : of the order; we simply want to keep the intermediate pressure
2294 : under control. Thus X has a cost of zero unless scheduling it
2295 : now would exceed MP'.
2296 :
2297 : If all increases in the set are by the same amount, no zero-cost
2298 : instruction will ever cause the pressure to exceed MP'. However,
2299 : if X is instead moved past an instruction X' with pressure in the
2300 : range (MP' - (P' - P), MP'), the pressure at X' will increase
2301 : beyond MP'. Since baseECC is very much a heuristic anyway,
2302 : it doesn't seem worth the overhead of tracking cases like these.
2303 :
2304 : The cost of exceeding MP' is always based on the original maximum
2305 : pressure MP. This is so that going 2 registers over the original
2306 : limit has the same cost regardless of whether it comes from two
2307 : separate +1 deltas or from a single +2 delta.
2308 :
2309 : * if X occurs after the next point of maximum pressure in the model
2310 : schedule and P' > P, then:
2311 :
2312 : baseECC (CL, X) = model_spill_cost (CL, MP, MP' + (P' - P))
2313 :
2314 : That is, if we move X forward across a point of maximum pressure,
2315 : and if X increases the pressure by P' - P, then we conservatively
2316 : assume that scheduling X next would increase the maximum pressure
2317 : by P' - P. Again, the cost of doing this is based on the original
2318 : maximum pressure MP, for the same reason as above.
2319 :
2320 : * if P' < P, P > MP, and X occurs at or after the next point of
2321 : maximum pressure, then:
2322 :
2323 : baseECC (CL, X) = -model_spill_cost (CL, MAX (MP, P'), P)
2324 :
2325 : That is, if we have already exceeded the original maximum pressure MP,
2326 : and if X might reduce the maximum pressure again -- or at least push
2327 : it further back, and thus allow more scheduling freedom -- it is given
2328 : a negative cost to reflect the improvement.
2329 :
2330 : * otherwise,
2331 :
2332 : baseECC (CL, X) = 0
2333 :
2334 : In this case, X is not expected to affect the maximum pressure MP',
2335 : so it has zero cost.
2336 :
2337 : We then create a combined value baseECC (X) that is the sum of
2338 : baseECC (CL, X) for each pressure class CL.
2339 :
2340 : baseECC (X) could itself be used as the ECC value described above.
2341 : However, this is often too conservative, in the sense that it
2342 : tends to make high-priority instructions that increase pressure
2343 : wait too long in cases where introducing a spill would be better.
2344 : For this reason the final ECC is a priority-adjusted form of
2345 : baseECC (X). Specifically, we calculate:
2346 :
2347 : P (X) = INSN_PRIORITY (X) - insn_delay (X) - baseECC (X)
2348 : baseP = MAX { P (X) | baseECC (X) <= 0 }
2349 :
2350 : Then:
2351 :
2352 : ECC (X) = MAX (MIN (baseP - P (X), baseECC (X)), 0)
2353 :
2354 : Thus an instruction's effect on pressure is ignored if it has a high
2355 : enough priority relative to the ones that don't increase pressure.
2356 : Negative values of baseECC (X) do not increase the priority of X
2357 : itself, but they do make it harder for other instructions to
2358 : increase the pressure further.
2359 :
2360 : This pressure cost is deliberately timid. The intention has been
2361 : to choose a heuristic that rarely interferes with the normal list
2362 : scheduler in cases where that scheduler would produce good code.
2363 : We simply want to curb some of its worst excesses. */
2364 :
2365 : /* Return the cost of increasing the pressure in class CL from FROM to TO.
2366 :
2367 : Here we use the very simplistic cost model that every register above
2368 : sched_class_regs_num[CL] has a spill cost of 1. We could use other
2369 : measures instead, such as one based on MEMORY_MOVE_COST. However:
2370 :
2371 : (1) In order for an instruction to be scheduled, the higher cost
2372 : would need to be justified in a single saving of that many stalls.
2373 : This is overly pessimistic, because the benefit of spilling is
2374 : often to avoid a sequence of several short stalls rather than
2375 : a single long one.
2376 :
2377 : (2) The cost is still arbitrary. Because we are not allocating
2378 : registers during scheduling, we have no way of knowing for
2379 : sure how many memory accesses will be required by each spill,
2380 : where the spills will be placed within the block, or even
2381 : which block(s) will contain the spills.
2382 :
2383 : So a higher cost than 1 is often too conservative in practice,
2384 : forcing blocks to contain unnecessary stalls instead of spill code.
2385 : The simple cost below seems to be the best compromise. It reduces
2386 : the interference with the normal list scheduler, which helps make
2387 : it more suitable for a default-on option. */
2388 :
2389 : static int
2390 0 : model_spill_cost (int cl, int from, int to)
2391 : {
2392 0 : from = MAX (from, sched_class_regs_num[cl]);
2393 0 : return MAX (to, from) - from;
2394 : }
2395 :
2396 : /* Return baseECC (ira_pressure_classes[PCI], POINT), given that
2397 : P = curr_reg_pressure[ira_pressure_classes[PCI]] and that
2398 : P' = P + DELTA. */
2399 :
2400 : static int
2401 0 : model_excess_group_cost (struct model_pressure_group *group,
2402 : int point, int pci, int delta)
2403 : {
2404 0 : int pressure, cl;
2405 :
2406 0 : cl = ira_pressure_classes[pci];
2407 0 : if (delta < 0)
2408 : {
2409 0 : if (point >= group->limits[pci].point)
2410 : {
2411 0 : pressure = MAX (group->limits[pci].orig_pressure,
2412 : curr_reg_pressure[cl] + delta);
2413 0 : return -model_spill_cost (cl, pressure, curr_reg_pressure[cl]);
2414 : }
2415 : /* if target prefers fewer spills, return the -ve delta indicating
2416 : pressure reduction. */
2417 0 : else if (!param_cycle_accurate_model)
2418 : return delta;
2419 : }
2420 :
2421 0 : if (delta > 0)
2422 : {
2423 0 : if (point > group->limits[pci].point)
2424 0 : pressure = group->limits[pci].pressure + delta;
2425 : else
2426 0 : pressure = curr_reg_pressure[cl] + delta;
2427 :
2428 0 : if (pressure > group->limits[pci].pressure)
2429 0 : return model_spill_cost (cl, group->limits[pci].orig_pressure,
2430 : pressure);
2431 : }
2432 :
2433 : return 0;
2434 : }
2435 :
2436 : /* Return baseECC (MODEL_INSN (INSN)). Dump the costs to sched_dump
2437 : if PRINT_P. */
2438 :
2439 : static int
2440 0 : model_excess_cost (rtx_insn *insn, bool print_p)
2441 : {
2442 0 : int point, pci, cl, cost, this_cost, delta;
2443 0 : struct reg_pressure_data *insn_reg_pressure;
2444 0 : int insn_death[N_REG_CLASSES];
2445 :
2446 0 : calculate_reg_deaths (insn, insn_death);
2447 0 : point = model_index (insn);
2448 0 : insn_reg_pressure = INSN_REG_PRESSURE (insn);
2449 0 : cost = 0;
2450 :
2451 0 : if (print_p)
2452 0 : fprintf (sched_dump, ";;\t\t| %3d %4d | %4d %+3d |", point,
2453 0 : INSN_UID (insn), INSN_PRIORITY (insn), insn_delay (insn));
2454 :
2455 : /* Sum up the individual costs for each register class. */
2456 0 : for (pci = 0; pci < ira_pressure_classes_num; pci++)
2457 : {
2458 0 : cl = ira_pressure_classes[pci];
2459 0 : delta = insn_reg_pressure[pci].set_increase - insn_death[cl];
2460 0 : this_cost = model_excess_group_cost (&model_before_pressure,
2461 : point, pci, delta);
2462 0 : cost += this_cost;
2463 0 : if (print_p)
2464 0 : fprintf (sched_dump, " %s:[%d base cost %d]",
2465 : reg_class_names[cl], delta, this_cost);
2466 : }
2467 :
2468 0 : if (print_p)
2469 0 : fprintf (sched_dump, " ECC %d\n", cost);
2470 :
2471 0 : return cost;
2472 : }
2473 :
2474 : /* Dump the next points of maximum pressure for GROUP. */
2475 :
2476 : static void
2477 0 : model_dump_pressure_points (struct model_pressure_group *group)
2478 : {
2479 0 : int pci, cl;
2480 :
2481 0 : fprintf (sched_dump, ";;\t\t| pressure points");
2482 0 : for (pci = 0; pci < ira_pressure_classes_num; pci++)
2483 : {
2484 0 : cl = ira_pressure_classes[pci];
2485 0 : fprintf (sched_dump, " %s:[%d->%d at ", reg_class_names[cl],
2486 : curr_reg_pressure[cl], group->limits[pci].pressure);
2487 0 : if (group->limits[pci].point < model_num_insns)
2488 0 : fprintf (sched_dump, "%d:%d]", group->limits[pci].point,
2489 0 : INSN_UID (MODEL_INSN (group->limits[pci].point)));
2490 : else
2491 0 : fprintf (sched_dump, "end]");
2492 : }
2493 0 : fprintf (sched_dump, "\n");
2494 0 : }
2495 :
2496 : /* Set INSN_REG_PRESSURE_EXCESS_COST_CHANGE for INSNS[0...COUNT-1]. */
2497 :
2498 : static void
2499 0 : model_set_excess_costs (rtx_insn **insns, int count)
2500 : {
2501 0 : int i, cost, priority_base, priority;
2502 0 : bool print_p;
2503 :
2504 : /* Record the baseECC value for each instruction in the model schedule,
2505 : except that for targets which prefer wider schedules (more spills)
2506 : negative costs are converted to zero ones now rather than later.
2507 : Do not assign a cost to debug instructions, since they must
2508 : not change code-generation decisions. Experiments suggest we also
2509 : get better results by not assigning a cost to instructions from
2510 : a different block.
2511 :
2512 : Set PRIORITY_BASE to baseP in the block comment above. This is the
2513 : maximum priority of the "cheap" instructions, which should always
2514 : include the next model instruction. */
2515 0 : priority_base = 0;
2516 0 : print_p = false;
2517 0 : for (i = 0; i < count; i++)
2518 0 : if (INSN_MODEL_INDEX (insns[i]))
2519 : {
2520 0 : if (sched_verbose >= 6 && !print_p)
2521 : {
2522 0 : fprintf (sched_dump, MODEL_BAR);
2523 0 : fprintf (sched_dump, ";;\t\t| Pressure costs for ready queue\n");
2524 0 : model_dump_pressure_points (&model_before_pressure);
2525 0 : fprintf (sched_dump, MODEL_BAR);
2526 0 : print_p = true;
2527 : }
2528 0 : cost = model_excess_cost (insns[i], print_p);
2529 0 : if (param_cycle_accurate_model && cost <= 0)
2530 : {
2531 0 : priority = INSN_PRIORITY (insns[i]) - insn_delay (insns[i]) - cost;
2532 0 : priority_base = MAX (priority_base, priority);
2533 : cost = 0;
2534 : }
2535 0 : INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insns[i]) = cost;
2536 : }
2537 0 : if (print_p)
2538 0 : fprintf (sched_dump, MODEL_BAR);
2539 :
2540 : /* Typically in-order cores have a good pipeline scheduling model and the
2541 : algorithm would try to use that to minimize bubbles, favoring spills.
2542 : MAX (baseECC, 0) below changes negative baseECC (pressure reduction)
2543 : to 0 (pressure neutral) thus tending to more spills.
2544 : Otherwise return. */
2545 0 : if (!param_cycle_accurate_model)
2546 : return;
2547 :
2548 : /* Use MAX (baseECC, 0) and baseP to calculate ECC for each
2549 : instruction. */
2550 0 : for (i = 0; i < count; i++)
2551 : {
2552 0 : cost = INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insns[i]);
2553 0 : priority = INSN_PRIORITY (insns[i]) - insn_delay (insns[i]);
2554 0 : if (cost > 0 && priority > priority_base)
2555 : {
2556 0 : cost += priority_base - priority;
2557 0 : INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insns[i]) = MAX (cost, 0);
2558 : }
2559 : }
2560 : }
2561 :
2562 :
2563 : /* Enum of rank_for_schedule heuristic decisions. */
2564 : enum rfs_decision {
2565 : RFS_LIVE_RANGE_SHRINK1, RFS_LIVE_RANGE_SHRINK2,
2566 : RFS_SCHED_GROUP, RFS_PRESSURE_DELAY, RFS_PRESSURE_TICK,
2567 : RFS_FEEDS_BACKTRACK_INSN, RFS_PRIORITY, RFS_AUTOPREF, RFS_SPECULATION,
2568 : RFS_SCHED_RANK, RFS_LAST_INSN, RFS_PRESSURE_INDEX,
2569 : RFS_DEP_COUNT, RFS_TIE, RFS_FUSION, RFS_COST, RFS_N };
2570 :
2571 : /* Corresponding strings for print outs. */
2572 : static const char *rfs_str[RFS_N] = {
2573 : "RFS_LIVE_RANGE_SHRINK1", "RFS_LIVE_RANGE_SHRINK2",
2574 : "RFS_SCHED_GROUP", "RFS_PRESSURE_DELAY", "RFS_PRESSURE_TICK",
2575 : "RFS_FEEDS_BACKTRACK_INSN", "RFS_PRIORITY", "RFS_AUTOPREF", "RFS_SPECULATION",
2576 : "RFS_SCHED_RANK", "RFS_LAST_INSN", "RFS_PRESSURE_INDEX",
2577 : "RFS_DEP_COUNT", "RFS_TIE", "RFS_FUSION", "RFS_COST" };
2578 :
2579 : /* Statistical breakdown of rank_for_schedule decisions. */
2580 : struct rank_for_schedule_stats_t { unsigned stats[RFS_N]; };
2581 : static rank_for_schedule_stats_t rank_for_schedule_stats;
2582 :
2583 : /* Return the result of comparing insns TMP and TMP2 and update
2584 : Rank_For_Schedule statistics. */
2585 : static int
2586 342291585 : rfs_result (enum rfs_decision decision, int result, rtx tmp, rtx tmp2)
2587 : {
2588 342291585 : ++rank_for_schedule_stats.stats[decision];
2589 342291585 : if (result < 0)
2590 187765290 : INSN_LAST_RFS_WIN (tmp) = decision;
2591 154526295 : else if (result > 0)
2592 154526295 : INSN_LAST_RFS_WIN (tmp2) = decision;
2593 : else
2594 0 : gcc_unreachable ();
2595 342291585 : return result;
2596 : }
2597 :
2598 : /* Sorting predicate to move DEBUG_INSNs to the top of ready list, while
2599 : keeping normal insns in original order. */
2600 :
2601 : static int
2602 428409603 : rank_for_schedule_debug (const void *x, const void *y)
2603 : {
2604 428409603 : rtx_insn *tmp = *(rtx_insn * const *) y;
2605 428409603 : rtx_insn *tmp2 = *(rtx_insn * const *) x;
2606 :
2607 : /* Schedule debug insns as early as possible. */
2608 428409603 : if (DEBUG_INSN_P (tmp) && !DEBUG_INSN_P (tmp2))
2609 : return -1;
2610 269377980 : else if (!DEBUG_INSN_P (tmp) && DEBUG_INSN_P (tmp2))
2611 : return 1;
2612 201024862 : else if (DEBUG_INSN_P (tmp) && DEBUG_INSN_P (tmp2))
2613 0 : return INSN_LUID (tmp) - INSN_LUID (tmp2);
2614 : else
2615 201024862 : return INSN_RFS_DEBUG_ORIG_ORDER (tmp2) - INSN_RFS_DEBUG_ORIG_ORDER (tmp);
2616 : }
2617 :
2618 : /* Returns a positive value if x is preferred; returns a negative value if
2619 : y is preferred. Should never return 0, since that will make the sort
2620 : unstable. */
2621 :
2622 : static int
2623 342291585 : rank_for_schedule (const void *x, const void *y)
2624 : {
2625 342291585 : rtx_insn *tmp = *(rtx_insn * const *) y;
2626 342291585 : rtx_insn *tmp2 = *(rtx_insn * const *) x;
2627 342291585 : int tmp_class, tmp2_class;
2628 342291585 : int val, priority_val, info_val, diff;
2629 :
2630 342291585 : if (live_range_shrinkage_p)
2631 : {
2632 : /* Don't use SCHED_PRESSURE_MODEL -- it results in much worse
2633 : code. */
2634 219999 : gcc_assert (sched_pressure == SCHED_PRESSURE_WEIGHTED);
2635 219999 : if ((INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp) < 0
2636 215111 : || INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp2) < 0)
2637 219999 : && (diff = (INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp)
2638 8630 : - INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp2))) != 0)
2639 8522 : return rfs_result (RFS_LIVE_RANGE_SHRINK1, diff, tmp, tmp2);
2640 : /* Sort by INSN_LUID (original insn order), so that we make the
2641 : sort stable. This minimizes instruction movement, thus
2642 : minimizing sched's effect on debugging and cross-jumping. */
2643 634431 : return rfs_result (RFS_LIVE_RANGE_SHRINK2,
2644 211477 : INSN_LUID (tmp) - INSN_LUID (tmp2), tmp, tmp2);
2645 : }
2646 :
2647 : /* The insn in a schedule group should be issued the first. */
2648 342071586 : if (flag_sched_group_heuristic &&
2649 342071586 : SCHED_GROUP_P (tmp) != SCHED_GROUP_P (tmp2))
2650 212 : return rfs_result (RFS_SCHED_GROUP, SCHED_GROUP_P (tmp2) ? 1 : -1,
2651 152 : tmp, tmp2);
2652 :
2653 : /* Make sure that priority of TMP and TMP2 are initialized. */
2654 342071434 : gcc_assert (INSN_PRIORITY_KNOWN (tmp) && INSN_PRIORITY_KNOWN (tmp2));
2655 :
2656 342071434 : if (sched_fusion)
2657 : {
2658 : /* The instruction that has the same fusion priority as the last
2659 : instruction is the instruction we picked next. If that is not
2660 : the case, we sort ready list firstly by fusion priority, then
2661 : by priority, and at last by INSN_LUID. */
2662 0 : int a = INSN_FUSION_PRIORITY (tmp);
2663 0 : int b = INSN_FUSION_PRIORITY (tmp2);
2664 0 : int last = -1;
2665 :
2666 0 : if (last_nondebug_scheduled_insn
2667 0 : && !NOTE_P (last_nondebug_scheduled_insn)
2668 0 : && BLOCK_FOR_INSN (tmp)
2669 0 : == BLOCK_FOR_INSN (last_nondebug_scheduled_insn))
2670 0 : last = INSN_FUSION_PRIORITY (last_nondebug_scheduled_insn);
2671 :
2672 0 : if (a != last && b != last)
2673 : {
2674 0 : if (a == b)
2675 : {
2676 0 : a = INSN_PRIORITY (tmp);
2677 0 : b = INSN_PRIORITY (tmp2);
2678 : }
2679 0 : if (a != b)
2680 0 : return rfs_result (RFS_FUSION, b - a, tmp, tmp2);
2681 : else
2682 0 : return rfs_result (RFS_FUSION,
2683 0 : INSN_LUID (tmp) - INSN_LUID (tmp2), tmp, tmp2);
2684 : }
2685 0 : else if (a == b)
2686 : {
2687 0 : gcc_assert (last_nondebug_scheduled_insn
2688 : && !NOTE_P (last_nondebug_scheduled_insn));
2689 0 : last = INSN_PRIORITY (last_nondebug_scheduled_insn);
2690 :
2691 0 : a = abs (INSN_PRIORITY (tmp) - last);
2692 0 : b = abs (INSN_PRIORITY (tmp2) - last);
2693 0 : if (a != b)
2694 0 : return rfs_result (RFS_FUSION, a - b, tmp, tmp2);
2695 : else
2696 0 : return rfs_result (RFS_FUSION,
2697 0 : INSN_LUID (tmp) - INSN_LUID (tmp2), tmp, tmp2);
2698 : }
2699 0 : else if (a == last)
2700 0 : return rfs_result (RFS_FUSION, -1, tmp, tmp2);
2701 : else
2702 0 : return rfs_result (RFS_FUSION, 1, tmp, tmp2);
2703 : }
2704 :
2705 342071434 : if (sched_pressure != SCHED_PRESSURE_NONE)
2706 : {
2707 : /* Prefer insn whose scheduling results in the smallest register
2708 : pressure excess. */
2709 13661 : if ((diff = (INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp)
2710 13661 : + insn_delay (tmp)
2711 13661 : - INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp2)
2712 13661 : - insn_delay (tmp2))))
2713 5021 : return rfs_result (RFS_PRESSURE_DELAY, diff, tmp, tmp2);
2714 : }
2715 :
2716 342066413 : if (sched_pressure != SCHED_PRESSURE_NONE
2717 8640 : && (INSN_TICK (tmp2) > clock_var || INSN_TICK (tmp) > clock_var)
2718 342067358 : && INSN_TICK (tmp2) != INSN_TICK (tmp))
2719 : {
2720 0 : diff = INSN_TICK (tmp) - INSN_TICK (tmp2);
2721 0 : return rfs_result (RFS_PRESSURE_TICK, diff, tmp, tmp2);
2722 : }
2723 :
2724 : /* If we are doing backtracking in this schedule, prefer insns that
2725 : have forward dependencies with negative cost against an insn that
2726 : was already scheduled. */
2727 342066413 : if (current_sched_info->flags & DO_BACKTRACKING)
2728 : {
2729 0 : priority_val = FEEDS_BACKTRACK_INSN (tmp2) - FEEDS_BACKTRACK_INSN (tmp);
2730 0 : if (priority_val)
2731 0 : return rfs_result (RFS_FEEDS_BACKTRACK_INSN, priority_val, tmp, tmp2);
2732 : }
2733 :
2734 : /* Prefer insn with higher priority. */
2735 342066413 : priority_val = INSN_PRIORITY (tmp2) - INSN_PRIORITY (tmp);
2736 :
2737 342066413 : if (flag_sched_critical_path_heuristic && priority_val)
2738 92577220 : return rfs_result (RFS_PRIORITY, priority_val, tmp, tmp2);
2739 :
2740 249489193 : if (param_sched_autopref_queue_depth >= 0)
2741 : {
2742 12 : int autopref = autopref_rank_for_schedule (tmp, tmp2);
2743 12 : if (autopref != 0)
2744 12 : return rfs_result (RFS_AUTOPREF, autopref, tmp, tmp2);
2745 : }
2746 :
2747 : /* Prefer speculative insn with greater dependencies weakness. */
2748 249489181 : if (flag_sched_spec_insn_heuristic && spec_info)
2749 : {
2750 0 : ds_t ds1, ds2;
2751 0 : dw_t dw1, dw2;
2752 0 : int dw;
2753 :
2754 0 : ds1 = TODO_SPEC (tmp) & SPECULATIVE;
2755 0 : if (ds1)
2756 0 : dw1 = ds_weak (ds1);
2757 : else
2758 : dw1 = NO_DEP_WEAK;
2759 :
2760 0 : ds2 = TODO_SPEC (tmp2) & SPECULATIVE;
2761 0 : if (ds2)
2762 0 : dw2 = ds_weak (ds2);
2763 : else
2764 : dw2 = NO_DEP_WEAK;
2765 :
2766 0 : dw = dw2 - dw1;
2767 0 : if (dw > (NO_DEP_WEAK / 8) || dw < -(NO_DEP_WEAK / 8))
2768 0 : return rfs_result (RFS_SPECULATION, dw, tmp, tmp2);
2769 : }
2770 :
2771 249489181 : info_val = (*current_sched_info->rank) (tmp, tmp2);
2772 249489181 : if (flag_sched_rank_heuristic && info_val)
2773 0 : return rfs_result (RFS_SCHED_RANK, info_val, tmp, tmp2);
2774 :
2775 : /* Compare insns based on their relation to the last scheduled
2776 : non-debug insn. */
2777 249489181 : if (flag_sched_last_insn_heuristic && last_nondebug_scheduled_insn)
2778 : {
2779 221870413 : dep_t dep1;
2780 221870413 : dep_t dep2;
2781 221870413 : rtx_insn *last = last_nondebug_scheduled_insn;
2782 :
2783 : /* Classify the instructions into three classes:
2784 : 1) Data dependent on last schedule insn.
2785 : 2) Anti/Output dependent on last scheduled insn.
2786 : 3) Independent of last scheduled insn, or has latency of one.
2787 : Choose the insn from the highest numbered class if different. */
2788 221870413 : dep1 = sd_find_dep_between (last, tmp, true);
2789 :
2790 221870413 : if (dep1 == NULL || dep_cost (dep1) == 1)
2791 : tmp_class = 3;
2792 16590076 : else if (/* Data dependence. */
2793 16590076 : DEP_TYPE (dep1) == REG_DEP_TRUE)
2794 : tmp_class = 1;
2795 : else
2796 15089552 : tmp_class = 2;
2797 :
2798 221870413 : dep2 = sd_find_dep_between (last, tmp2, true);
2799 :
2800 221870413 : if (dep2 == NULL || dep_cost (dep2) == 1)
2801 : tmp2_class = 3;
2802 16949073 : else if (/* Data dependence. */
2803 16949073 : DEP_TYPE (dep2) == REG_DEP_TRUE)
2804 : tmp2_class = 1;
2805 : else
2806 15235682 : tmp2_class = 2;
2807 :
2808 221870413 : if ((val = tmp2_class - tmp_class))
2809 3380379 : return rfs_result (RFS_LAST_INSN, val, tmp, tmp2);
2810 : }
2811 :
2812 : /* Prefer instructions that occur earlier in the model schedule. */
2813 246108802 : if (sched_pressure == SCHED_PRESSURE_MODEL)
2814 : {
2815 0 : diff = model_index (tmp) - model_index (tmp2);
2816 0 : if (diff != 0)
2817 0 : return rfs_result (RFS_PRESSURE_INDEX, diff, tmp, tmp2);
2818 : }
2819 :
2820 : /* Prefer the insn which has more later insns that depend on it.
2821 : This gives the scheduler more freedom when scheduling later
2822 : instructions at the expense of added register pressure. */
2823 :
2824 246108802 : val = (dep_list_size (tmp2, SD_LIST_FORW)
2825 246108802 : - dep_list_size (tmp, SD_LIST_FORW));
2826 :
2827 246108802 : if (flag_sched_dep_count_heuristic && val != 0)
2828 30010361 : return rfs_result (RFS_DEP_COUNT, val, tmp, tmp2);
2829 :
2830 : /* Sort by INSN_COST rather than INSN_LUID. This means that instructions
2831 : which take longer to execute are prioritised and it leads to more
2832 : dual-issue opportunities on in-order cores which have this feature. */
2833 :
2834 216098441 : if (INSN_COST (tmp) != INSN_COST (tmp2))
2835 13825805 : return rfs_result (RFS_COST, INSN_COST (tmp2) - INSN_COST (tmp),
2836 13825805 : tmp, tmp2);
2837 :
2838 : /* If insns are equally good, sort by INSN_LUID (original insn order),
2839 : so that we make the sort stable. This minimizes instruction movement,
2840 : thus minimizing sched's effect on debugging and cross-jumping. */
2841 202272636 : return rfs_result (RFS_TIE, INSN_LUID (tmp) - INSN_LUID (tmp2), tmp, tmp2);
2842 : }
2843 :
2844 : /* Resort the array A in which only element at index N may be out of order. */
2845 :
2846 : HAIFA_INLINE static void
2847 8154096 : swap_sort (rtx_insn **a, int n)
2848 : {
2849 8154096 : rtx_insn *insn = a[n - 1];
2850 8154096 : int i = n - 2;
2851 :
2852 10963149 : while (i >= 0 && rank_for_schedule (a + i, &insn) >= 0)
2853 : {
2854 2809053 : a[i + 1] = a[i];
2855 2809053 : i -= 1;
2856 : }
2857 8154096 : a[i + 1] = insn;
2858 8154096 : }
2859 :
2860 : /* Add INSN to the insn queue so that it can be executed at least
2861 : N_CYCLES after the currently executing insn. Preserve insns
2862 : chain for debugging purposes. REASON will be printed in debugging
2863 : output. */
2864 :
2865 : HAIFA_INLINE static void
2866 39770954 : queue_insn (rtx_insn *insn, int n_cycles, const char *reason)
2867 : {
2868 39770954 : int next_q = NEXT_Q_AFTER (q_ptr, n_cycles);
2869 39770954 : rtx_insn_list *link = alloc_INSN_LIST (insn, insn_queue[next_q]);
2870 39770954 : int new_tick;
2871 :
2872 39770954 : gcc_assert (n_cycles <= max_insn_queue_index);
2873 39770954 : gcc_assert (!DEBUG_INSN_P (insn));
2874 :
2875 39770954 : insn_queue[next_q] = link;
2876 39770954 : q_size += 1;
2877 :
2878 39770954 : if (sched_verbose >= 2)
2879 : {
2880 0 : fprintf (sched_dump, ";;\t\tReady-->Q: insn %s: ",
2881 0 : (*current_sched_info->print_insn) (insn, 0));
2882 :
2883 0 : fprintf (sched_dump, "queued for %d cycles (%s).\n", n_cycles, reason);
2884 : }
2885 :
2886 39770954 : QUEUE_INDEX (insn) = next_q;
2887 :
2888 39770954 : if (current_sched_info->flags & DO_BACKTRACKING)
2889 : {
2890 0 : new_tick = clock_var + n_cycles;
2891 0 : if (INSN_TICK (insn) == INVALID_TICK || INSN_TICK (insn) < new_tick)
2892 0 : INSN_TICK (insn) = new_tick;
2893 :
2894 0 : if (INSN_EXACT_TICK (insn) != INVALID_TICK
2895 0 : && INSN_EXACT_TICK (insn) < clock_var + n_cycles)
2896 : {
2897 0 : must_backtrack = true;
2898 0 : if (sched_verbose >= 2)
2899 0 : fprintf (sched_dump, ";;\t\tcausing a backtrack.\n");
2900 : }
2901 : }
2902 39770954 : }
2903 :
2904 : /* Remove INSN from queue. */
2905 : static void
2906 139610 : queue_remove (rtx_insn *insn)
2907 : {
2908 139610 : gcc_assert (QUEUE_INDEX (insn) >= 0);
2909 139610 : remove_free_INSN_LIST_elem (insn, &insn_queue[QUEUE_INDEX (insn)]);
2910 139610 : q_size--;
2911 139610 : QUEUE_INDEX (insn) = QUEUE_NOWHERE;
2912 139610 : }
2913 :
2914 : /* Return a pointer to the bottom of the ready list, i.e. the insn
2915 : with the lowest priority. */
2916 :
2917 : rtx_insn **
2918 150082288 : ready_lastpos (struct ready_list *ready)
2919 : {
2920 150082288 : gcc_assert (ready->n_ready >= 1);
2921 150082288 : return ready->vec + ready->first - ready->n_ready + 1;
2922 : }
2923 :
2924 : /* Add an element INSN to the ready list so that it ends up with the
2925 : lowest/highest priority depending on FIRST_P. */
2926 :
2927 : HAIFA_INLINE static void
2928 134217576 : ready_add (struct ready_list *ready, rtx_insn *insn, bool first_p)
2929 : {
2930 134217576 : if (!first_p)
2931 : {
2932 134215361 : if (ready->first == ready->n_ready)
2933 : {
2934 148 : memmove (ready->vec + ready->veclen - ready->n_ready,
2935 74 : ready_lastpos (ready),
2936 74 : ready->n_ready * sizeof (rtx));
2937 74 : ready->first = ready->veclen - 1;
2938 : }
2939 134215361 : ready->vec[ready->first - ready->n_ready] = insn;
2940 : }
2941 : else
2942 : {
2943 2215 : if (ready->first == ready->veclen - 1)
2944 : {
2945 950 : if (ready->n_ready)
2946 : /* ready_lastpos() fails when called with (ready->n_ready == 0). */
2947 0 : memmove (ready->vec + ready->veclen - ready->n_ready - 1,
2948 0 : ready_lastpos (ready),
2949 0 : ready->n_ready * sizeof (rtx));
2950 950 : ready->first = ready->veclen - 2;
2951 : }
2952 2215 : ready->vec[++(ready->first)] = insn;
2953 : }
2954 :
2955 134217576 : ready->n_ready++;
2956 134217576 : if (DEBUG_INSN_P (insn))
2957 52852642 : ready->n_debug++;
2958 :
2959 134217576 : gcc_assert (QUEUE_INDEX (insn) != QUEUE_READY);
2960 134217576 : QUEUE_INDEX (insn) = QUEUE_READY;
2961 :
2962 134217576 : if (INSN_EXACT_TICK (insn) != INVALID_TICK
2963 134217576 : && INSN_EXACT_TICK (insn) < clock_var)
2964 : {
2965 0 : must_backtrack = true;
2966 : }
2967 134217576 : }
2968 :
2969 : /* Remove the element with the highest priority from the ready list and
2970 : return it. */
2971 :
2972 : HAIFA_INLINE static rtx_insn *
2973 131918401 : ready_remove_first (struct ready_list *ready)
2974 : {
2975 131918401 : rtx_insn *t;
2976 :
2977 131918401 : gcc_assert (ready->n_ready);
2978 131918401 : t = ready->vec[ready->first--];
2979 131918401 : ready->n_ready--;
2980 131918401 : if (DEBUG_INSN_P (t))
2981 52852642 : ready->n_debug--;
2982 : /* If the queue becomes empty, reset it. */
2983 131918401 : if (ready->n_ready == 0)
2984 68252605 : ready->first = ready->veclen - 1;
2985 :
2986 131918401 : gcc_assert (QUEUE_INDEX (t) == QUEUE_READY);
2987 131918401 : QUEUE_INDEX (t) = QUEUE_NOWHERE;
2988 :
2989 131918401 : return t;
2990 : }
2991 :
2992 : /* The following code implements multi-pass scheduling for the first
2993 : cycle. In other words, we will try to choose ready insn which
2994 : permits to start maximum number of insns on the same cycle. */
2995 :
2996 : /* Return a pointer to the element INDEX from the ready. INDEX for
2997 : insn with the highest priority is 0, and the lowest priority has
2998 : N_READY - 1. */
2999 :
3000 : rtx_insn *
3001 1140067026 : ready_element (struct ready_list *ready, int index)
3002 : {
3003 1140067026 : gcc_assert (ready->n_ready && index < ready->n_ready);
3004 :
3005 1140067026 : return ready->vec[ready->first - index];
3006 : }
3007 :
3008 : /* Remove the element INDEX from the ready list and return it. INDEX
3009 : for insn with the highest priority is 0, and the lowest priority
3010 : has N_READY - 1. */
3011 :
3012 : HAIFA_INLINE static rtx_insn *
3013 74040464 : ready_remove (struct ready_list *ready, int index)
3014 : {
3015 74040464 : rtx_insn *t;
3016 74040464 : int i;
3017 :
3018 74040464 : if (index == 0)
3019 71741301 : return ready_remove_first (ready);
3020 2299163 : gcc_assert (ready->n_ready && index < ready->n_ready);
3021 2299163 : t = ready->vec[ready->first - index];
3022 2299163 : ready->n_ready--;
3023 2299163 : if (DEBUG_INSN_P (t))
3024 0 : ready->n_debug--;
3025 7533919 : for (i = index; i < ready->n_ready; i++)
3026 5234756 : ready->vec[ready->first - i] = ready->vec[ready->first - i - 1];
3027 2299163 : QUEUE_INDEX (t) = QUEUE_NOWHERE;
3028 2299163 : return t;
3029 : }
3030 :
3031 : /* Remove INSN from the ready list. */
3032 : static void
3033 0 : ready_remove_insn (rtx_insn *insn)
3034 : {
3035 0 : int i;
3036 :
3037 0 : for (i = 0; i < readyp->n_ready; i++)
3038 0 : if (ready_element (readyp, i) == insn)
3039 : {
3040 0 : ready_remove (readyp, i);
3041 0 : return;
3042 : }
3043 0 : gcc_unreachable ();
3044 : }
3045 :
3046 : /* Calculate difference of two statistics set WAS and NOW.
3047 : Result returned in WAS. */
3048 : static void
3049 0 : rank_for_schedule_stats_diff (rank_for_schedule_stats_t *was,
3050 : const rank_for_schedule_stats_t *now)
3051 : {
3052 0 : for (int i = 0; i < RFS_N; ++i)
3053 0 : was->stats[i] = now->stats[i] - was->stats[i];
3054 0 : }
3055 :
3056 : /* Print rank_for_schedule statistics. */
3057 : static void
3058 0 : print_rank_for_schedule_stats (const char *prefix,
3059 : const rank_for_schedule_stats_t *stats,
3060 : struct ready_list *ready)
3061 : {
3062 0 : for (int i = 0; i < RFS_N; ++i)
3063 0 : if (stats->stats[i])
3064 : {
3065 0 : fprintf (sched_dump, "%s%20s: %u", prefix, rfs_str[i], stats->stats[i]);
3066 :
3067 0 : if (ready != NULL)
3068 : /* Print out insns that won due to RFS_<I>. */
3069 : {
3070 0 : rtx_insn **p = ready_lastpos (ready);
3071 :
3072 0 : fprintf (sched_dump, ":");
3073 : /* Start with 1 since least-priority insn didn't have any wins. */
3074 0 : for (int j = 1; j < ready->n_ready; ++j)
3075 0 : if (INSN_LAST_RFS_WIN (p[j]) == i)
3076 0 : fprintf (sched_dump, " %s",
3077 0 : (*current_sched_info->print_insn) (p[j], 0));
3078 : }
3079 0 : fprintf (sched_dump, "\n");
3080 : }
3081 0 : }
3082 :
3083 : /* Separate DEBUG_INSNS from normal insns. DEBUG_INSNs go to the end
3084 : of array. */
3085 : static void
3086 52852646 : ready_sort_debug (struct ready_list *ready)
3087 : {
3088 52852646 : int i;
3089 52852646 : rtx_insn **first = ready_lastpos (ready);
3090 :
3091 223736163 : for (i = 0; i < ready->n_ready; ++i)
3092 118030871 : if (!DEBUG_INSN_P (first[i]))
3093 65178228 : INSN_RFS_DEBUG_ORIG_ORDER (first[i]) = i;
3094 :
3095 52852646 : qsort (first, ready->n_ready, sizeof (rtx), rank_for_schedule_debug);
3096 52852646 : }
3097 :
3098 : /* Sort non-debug insns in the ready list READY by ascending priority.
3099 : Assumes that all debug insns are separated from the real insns. */
3100 : static void
3101 61546217 : ready_sort_real (struct ready_list *ready)
3102 : {
3103 61546217 : int i;
3104 61546217 : rtx_insn **first = ready_lastpos (ready);
3105 61546217 : int n_ready_real = ready->n_ready - ready->n_debug;
3106 :
3107 61546217 : if (sched_pressure == SCHED_PRESSURE_WEIGHTED)
3108 27889 : for (i = 0; i < n_ready_real; ++i)
3109 21766 : setup_insn_reg_pressure_info (first[i]);
3110 61540094 : else if (sched_pressure == SCHED_PRESSURE_MODEL
3111 0 : && model_curr_point < model_num_insns)
3112 0 : model_set_excess_costs (first, n_ready_real);
3113 :
3114 61546217 : rank_for_schedule_stats_t stats1;
3115 61546217 : if (sched_verbose >= 4)
3116 0 : stats1 = rank_for_schedule_stats;
3117 :
3118 61546217 : if (n_ready_real == 2)
3119 8154096 : swap_sort (first, n_ready_real);
3120 53392121 : else if (n_ready_real > 2)
3121 7758044 : qsort (first, n_ready_real, sizeof (rtx), rank_for_schedule);
3122 :
3123 61546217 : if (sched_verbose >= 4)
3124 : {
3125 0 : rank_for_schedule_stats_diff (&stats1, &rank_for_schedule_stats);
3126 0 : print_rank_for_schedule_stats (";;\t\t", &stats1, ready);
3127 : }
3128 61546217 : }
3129 :
3130 : /* Sort the ready list READY by ascending priority. */
3131 : static void
3132 114398855 : ready_sort (struct ready_list *ready)
3133 : {
3134 114398855 : if (ready->n_debug > 0)
3135 52852642 : ready_sort_debug (ready);
3136 : else
3137 61546213 : ready_sort_real (ready);
3138 114398855 : }
3139 :
3140 : /* PREV is an insn that is ready to execute. Adjust its priority if that
3141 : will help shorten or lengthen register lifetimes as appropriate. Also
3142 : provide a hook for the target to tweak itself. */
3143 :
3144 : HAIFA_INLINE static void
3145 114313485 : adjust_priority (rtx_insn *prev)
3146 : {
3147 : /* ??? There used to be code here to try and estimate how an insn
3148 : affected register lifetimes, but it did it by looking at REG_DEAD
3149 : notes, which we removed in schedule_region. Nor did it try to
3150 : take into account register pressure or anything useful like that.
3151 :
3152 : Revisit when we have a machine model to work with and not before. */
3153 :
3154 114313485 : if (targetm.sched.adjust_priority)
3155 114313485 : INSN_PRIORITY (prev) =
3156 114313485 : targetm.sched.adjust_priority (prev, INSN_PRIORITY (prev));
3157 114313485 : }
3158 :
3159 : /* Advance DFA state STATE on one cycle. */
3160 : void
3161 47045946 : advance_state (state_t state)
3162 : {
3163 47045946 : if (targetm.sched.dfa_pre_advance_cycle)
3164 0 : targetm.sched.dfa_pre_advance_cycle ();
3165 :
3166 47045946 : if (targetm.sched.dfa_pre_cycle_insn)
3167 0 : state_transition (state,
3168 : targetm.sched.dfa_pre_cycle_insn ());
3169 :
3170 47045946 : state_transition (state, NULL);
3171 :
3172 47045946 : if (targetm.sched.dfa_post_cycle_insn)
3173 0 : state_transition (state,
3174 0 : targetm.sched.dfa_post_cycle_insn ());
3175 :
3176 47045946 : if (targetm.sched.dfa_post_advance_cycle)
3177 46869811 : targetm.sched.dfa_post_advance_cycle ();
3178 47045946 : }
3179 :
3180 : /* Advance time on one cycle. */
3181 : HAIFA_INLINE static void
3182 47040958 : advance_one_cycle (void)
3183 : {
3184 47040958 : int i;
3185 :
3186 47040958 : advance_state (curr_state);
3187 94081916 : for (i = 4; i <= sched_verbose; ++i)
3188 0 : fprintf (sched_dump, ";;\tAdvance the current state: %d.\n", clock_var);
3189 47040958 : }
3190 :
3191 : /* Update register pressure after scheduling INSN. */
3192 : static void
3193 20115 : update_register_pressure (rtx_insn *insn)
3194 : {
3195 20115 : struct reg_use_data *use;
3196 20115 : struct reg_set_data *set;
3197 :
3198 20115 : gcc_checking_assert (!DEBUG_INSN_P (insn));
3199 :
3200 37687 : for (use = INSN_REG_USE_LIST (insn); use != NULL; use = use->next_insn_use)
3201 17572 : if (dying_use_p (use))
3202 15320 : mark_regno_birth_or_death (curr_reg_live, curr_reg_pressure,
3203 : use->regno, false);
3204 34518 : for (set = INSN_REG_SET_LIST (insn); set != NULL; set = set->next_insn_set)
3205 14403 : mark_regno_birth_or_death (curr_reg_live, curr_reg_pressure,
3206 : set->regno, true);
3207 20115 : }
3208 :
3209 : /* Set up or update (if UPDATE_P) max register pressure (see its
3210 : meaning in sched-int.h::_haifa_insn_data) for all current BB insns
3211 : after insn AFTER. */
3212 : static void
3213 2866 : setup_insn_max_reg_pressure (rtx_insn *after, bool update_p)
3214 : {
3215 2866 : int i, p;
3216 2866 : bool eq_p;
3217 2866 : rtx_insn *insn;
3218 2866 : static int max_reg_pressure[N_REG_CLASSES];
3219 :
3220 2866 : save_reg_pressure ();
3221 17372 : for (i = 0; i < ira_pressure_classes_num; i++)
3222 11640 : max_reg_pressure[ira_pressure_classes[i]]
3223 11640 : = curr_reg_pressure[ira_pressure_classes[i]];
3224 23050 : for (insn = NEXT_INSN (after);
3225 23050 : insn != NULL_RTX && ! BARRIER_P (insn)
3226 45927 : && BLOCK_FOR_INSN (insn) == BLOCK_FOR_INSN (after);
3227 20184 : insn = NEXT_INSN (insn))
3228 21731 : if (NONDEBUG_INSN_P (insn))
3229 : {
3230 : eq_p = true;
3231 93344 : for (i = 0; i < ira_pressure_classes_num; i++)
3232 : {
3233 75446 : p = max_reg_pressure[ira_pressure_classes[i]];
3234 75446 : if (INSN_MAX_REG_PRESSURE (insn)[i] != p)
3235 : {
3236 18737 : eq_p = false;
3237 18737 : INSN_MAX_REG_PRESSURE (insn)[i]
3238 18737 : = max_reg_pressure[ira_pressure_classes[i]];
3239 : }
3240 : }
3241 17898 : if (update_p && eq_p)
3242 : break;
3243 16351 : update_register_pressure (insn);
3244 101850 : for (i = 0; i < ira_pressure_classes_num; i++)
3245 69148 : if (max_reg_pressure[ira_pressure_classes[i]]
3246 69148 : < curr_reg_pressure[ira_pressure_classes[i]])
3247 2154 : max_reg_pressure[ira_pressure_classes[i]]
3248 2154 : = curr_reg_pressure[ira_pressure_classes[i]];
3249 : }
3250 2866 : restore_reg_pressure ();
3251 2866 : }
3252 :
3253 : /* Update the current register pressure after scheduling INSN. Update
3254 : also max register pressure for unscheduled insns of the current
3255 : BB. */
3256 : static void
3257 3764 : update_reg_and_insn_max_reg_pressure (rtx_insn *insn)
3258 : {
3259 3764 : int i;
3260 3764 : int before[N_REG_CLASSES];
3261 :
3262 19097 : for (i = 0; i < ira_pressure_classes_num; i++)
3263 15333 : before[i] = curr_reg_pressure[ira_pressure_classes[i]];
3264 3764 : update_register_pressure (insn);
3265 14342 : for (i = 0; i < ira_pressure_classes_num; i++)
3266 8974 : if (curr_reg_pressure[ira_pressure_classes[i]] != before[i])
3267 : break;
3268 3764 : if (i < ira_pressure_classes_num)
3269 2160 : setup_insn_max_reg_pressure (insn, true);
3270 3764 : }
3271 :
3272 : /* Set up register pressure at the beginning of basic block BB whose
3273 : insns starting after insn AFTER. Set up also max register pressure
3274 : for all insns of the basic block. */
3275 : void
3276 706 : sched_setup_bb_reg_pressure_info (basic_block bb, rtx_insn *after)
3277 : {
3278 706 : gcc_assert (sched_pressure == SCHED_PRESSURE_WEIGHTED);
3279 706 : initiate_bb_reg_pressure_info (bb);
3280 706 : setup_insn_max_reg_pressure (after, false);
3281 706 : }
3282 :
3283 : /* If doing predication while scheduling, verify whether INSN, which
3284 : has just been scheduled, clobbers the conditions of any
3285 : instructions that must be predicated in order to break their
3286 : dependencies. If so, remove them from the queues so that they will
3287 : only be scheduled once their control dependency is resolved. */
3288 :
3289 : static void
3290 114313473 : check_clobbered_conditions (rtx_insn *insn)
3291 : {
3292 114313473 : HARD_REG_SET t;
3293 114313473 : int i;
3294 :
3295 114313473 : if ((current_sched_info->flags & DO_PREDICATION) == 0)
3296 114313473 : return;
3297 :
3298 0 : find_all_hard_reg_sets (insn, &t, true);
3299 :
3300 0 : restart:
3301 0 : for (i = 0; i < ready.n_ready; i++)
3302 : {
3303 0 : rtx_insn *x = ready_element (&ready, i);
3304 0 : if (TODO_SPEC (x) == DEP_CONTROL && cond_clobbered_p (x, t))
3305 : {
3306 0 : ready_remove_insn (x);
3307 0 : goto restart;
3308 : }
3309 : }
3310 0 : for (i = 0; i <= max_insn_queue_index; i++)
3311 : {
3312 0 : rtx_insn_list *link;
3313 0 : int q = NEXT_Q_AFTER (q_ptr, i);
3314 :
3315 0 : restart_queue:
3316 0 : for (link = insn_queue[q]; link; link = link->next ())
3317 : {
3318 0 : rtx_insn *x = link->insn ();
3319 0 : if (TODO_SPEC (x) == DEP_CONTROL && cond_clobbered_p (x, t))
3320 : {
3321 0 : queue_remove (x);
3322 0 : goto restart_queue;
3323 : }
3324 : }
3325 : }
3326 : }
3327 :
3328 : /* Return (in order):
3329 :
3330 : - positive if INSN adversely affects the pressure on one
3331 : register class
3332 :
3333 : - negative if INSN reduces the pressure on one register class
3334 :
3335 : - 0 if INSN doesn't affect the pressure on any register class. */
3336 :
3337 : static int
3338 0 : model_classify_pressure (struct model_insn_info *insn)
3339 : {
3340 0 : struct reg_pressure_data *reg_pressure;
3341 0 : int death[N_REG_CLASSES];
3342 0 : int pci, cl, sum;
3343 :
3344 0 : calculate_reg_deaths (insn->insn, death);
3345 0 : reg_pressure = INSN_REG_PRESSURE (insn->insn);
3346 0 : sum = 0;
3347 0 : for (pci = 0; pci < ira_pressure_classes_num; pci++)
3348 : {
3349 0 : cl = ira_pressure_classes[pci];
3350 0 : if (death[cl] < reg_pressure[pci].set_increase)
3351 : return 1;
3352 0 : sum += reg_pressure[pci].set_increase - death[cl];
3353 : }
3354 : return sum;
3355 : }
3356 :
3357 : /* Return true if INSN1 should come before INSN2 in the model schedule. */
3358 :
3359 : static int
3360 0 : model_order_p (struct model_insn_info *insn1, struct model_insn_info *insn2)
3361 : {
3362 0 : unsigned int height1, height2;
3363 0 : unsigned int priority1, priority2;
3364 :
3365 : /* Prefer instructions with a higher model priority. */
3366 0 : if (insn1->model_priority != insn2->model_priority)
3367 0 : return insn1->model_priority > insn2->model_priority;
3368 :
3369 : /* Combine the length of the longest path of satisfied true dependencies
3370 : that leads to each instruction (depth) with the length of the longest
3371 : path of any dependencies that leads from the instruction (alap).
3372 : Prefer instructions with the greatest combined length. If the combined
3373 : lengths are equal, prefer instructions with the greatest depth.
3374 :
3375 : The idea is that, if we have a set S of "equal" instructions that each
3376 : have ALAP value X, and we pick one such instruction I, any true-dependent
3377 : successors of I that have ALAP value X - 1 should be preferred over S.
3378 : This encourages the schedule to be "narrow" rather than "wide".
3379 : However, if I is a low-priority instruction that we decided to
3380 : schedule because of its model_classify_pressure, and if there
3381 : is a set of higher-priority instructions T, the aforementioned
3382 : successors of I should not have the edge over T. */
3383 0 : height1 = insn1->depth + insn1->alap;
3384 0 : height2 = insn2->depth + insn2->alap;
3385 0 : if (height1 != height2)
3386 0 : return height1 > height2;
3387 0 : if (insn1->depth != insn2->depth)
3388 0 : return insn1->depth > insn2->depth;
3389 :
3390 : /* We have no real preference between INSN1 an INSN2 as far as attempts
3391 : to reduce pressure go. Prefer instructions with higher priorities. */
3392 0 : priority1 = INSN_PRIORITY (insn1->insn);
3393 0 : priority2 = INSN_PRIORITY (insn2->insn);
3394 0 : if (priority1 != priority2)
3395 0 : return priority1 > priority2;
3396 :
3397 : /* Use the original rtl sequence as a tie-breaker. */
3398 0 : return insn1 < insn2;
3399 : }
3400 :
3401 : /* Add INSN to the model worklist immediately after PREV. Add it to the
3402 : beginning of the list if PREV is null. */
3403 :
3404 : static void
3405 0 : model_add_to_worklist_at (struct model_insn_info *insn,
3406 : struct model_insn_info *prev)
3407 : {
3408 0 : gcc_assert (QUEUE_INDEX (insn->insn) == QUEUE_NOWHERE);
3409 0 : QUEUE_INDEX (insn->insn) = QUEUE_READY;
3410 :
3411 0 : insn->prev = prev;
3412 0 : if (prev)
3413 : {
3414 0 : insn->next = prev->next;
3415 0 : prev->next = insn;
3416 : }
3417 : else
3418 : {
3419 0 : insn->next = model_worklist;
3420 0 : model_worklist = insn;
3421 : }
3422 0 : if (insn->next)
3423 0 : insn->next->prev = insn;
3424 0 : }
3425 :
3426 : /* Remove INSN from the model worklist. */
3427 :
3428 : static void
3429 0 : model_remove_from_worklist (struct model_insn_info *insn)
3430 : {
3431 0 : gcc_assert (QUEUE_INDEX (insn->insn) == QUEUE_READY);
3432 0 : QUEUE_INDEX (insn->insn) = QUEUE_NOWHERE;
3433 :
3434 0 : if (insn->prev)
3435 0 : insn->prev->next = insn->next;
3436 : else
3437 0 : model_worklist = insn->next;
3438 0 : if (insn->next)
3439 0 : insn->next->prev = insn->prev;
3440 0 : }
3441 :
3442 : /* Add INSN to the model worklist. Start looking for a suitable position
3443 : between neighbors PREV and NEXT, testing at most param_max_sched_ready_insns
3444 : insns either side. A null PREV indicates the beginning of the list and
3445 : a null NEXT indicates the end. */
3446 :
3447 : static void
3448 0 : model_add_to_worklist (struct model_insn_info *insn,
3449 : struct model_insn_info *prev,
3450 : struct model_insn_info *next)
3451 : {
3452 0 : int count;
3453 :
3454 0 : count = param_max_sched_ready_insns;
3455 0 : if (count > 0 && prev && model_order_p (insn, prev))
3456 0 : do
3457 : {
3458 0 : count--;
3459 0 : prev = prev->prev;
3460 : }
3461 0 : while (count > 0 && prev && model_order_p (insn, prev));
3462 : else
3463 0 : while (count > 0 && next && model_order_p (next, insn))
3464 : {
3465 0 : count--;
3466 0 : prev = next;
3467 0 : next = next->next;
3468 : }
3469 0 : model_add_to_worklist_at (insn, prev);
3470 0 : }
3471 :
3472 : /* INSN may now have a higher priority (in the model_order_p sense)
3473 : than before. Move it up the worklist if necessary. */
3474 :
3475 : static void
3476 0 : model_promote_insn (struct model_insn_info *insn)
3477 : {
3478 0 : struct model_insn_info *prev;
3479 0 : int count;
3480 :
3481 0 : prev = insn->prev;
3482 0 : count = param_max_sched_ready_insns;
3483 0 : while (count > 0 && prev && model_order_p (insn, prev))
3484 : {
3485 0 : count--;
3486 0 : prev = prev->prev;
3487 : }
3488 0 : if (prev != insn->prev)
3489 : {
3490 0 : model_remove_from_worklist (insn);
3491 0 : model_add_to_worklist_at (insn, prev);
3492 : }
3493 0 : }
3494 :
3495 : /* Add INSN to the end of the model schedule. */
3496 :
3497 : static void
3498 0 : model_add_to_schedule (rtx_insn *insn)
3499 : {
3500 0 : unsigned int point;
3501 :
3502 0 : gcc_assert (QUEUE_INDEX (insn) == QUEUE_NOWHERE);
3503 0 : QUEUE_INDEX (insn) = QUEUE_SCHEDULED;
3504 :
3505 0 : point = model_schedule.length ();
3506 0 : model_schedule.quick_push (insn);
3507 0 : INSN_MODEL_INDEX (insn) = point + 1;
3508 0 : }
3509 :
3510 : /* Analyze the instructions that are to be scheduled, setting up
3511 : MODEL_INSN_INFO (...) and model_num_insns accordingly. Add ready
3512 : instructions to model_worklist. */
3513 :
3514 : static void
3515 0 : model_analyze_insns (void)
3516 : {
3517 0 : rtx_insn *start, *end, *iter;
3518 0 : sd_iterator_def sd_it;
3519 0 : dep_t dep;
3520 0 : struct model_insn_info *insn, *con;
3521 :
3522 0 : model_num_insns = 0;
3523 0 : start = PREV_INSN (current_sched_info->next_tail);
3524 0 : end = current_sched_info->prev_head;
3525 0 : for (iter = start; iter != end; iter = PREV_INSN (iter))
3526 0 : if (NONDEBUG_INSN_P (iter))
3527 : {
3528 0 : insn = MODEL_INSN_INFO (iter);
3529 0 : insn->insn = iter;
3530 0 : FOR_EACH_DEP (iter, SD_LIST_FORW, sd_it, dep)
3531 : {
3532 0 : con = MODEL_INSN_INFO (DEP_CON (dep));
3533 0 : if (con->insn && insn->alap < con->alap + 1)
3534 0 : insn->alap = con->alap + 1;
3535 : }
3536 :
3537 0 : insn->old_queue = QUEUE_INDEX (iter);
3538 0 : QUEUE_INDEX (iter) = QUEUE_NOWHERE;
3539 :
3540 0 : insn->unscheduled_preds = dep_list_size (iter, SD_LIST_HARD_BACK);
3541 0 : if (insn->unscheduled_preds == 0)
3542 0 : model_add_to_worklist (insn, NULL, model_worklist);
3543 :
3544 0 : model_num_insns++;
3545 : }
3546 0 : }
3547 :
3548 : /* The global state describes the register pressure at the start of the
3549 : model schedule. Initialize GROUP accordingly. */
3550 :
3551 : static void
3552 0 : model_init_pressure_group (struct model_pressure_group *group)
3553 : {
3554 0 : int pci, cl;
3555 :
3556 0 : for (pci = 0; pci < ira_pressure_classes_num; pci++)
3557 : {
3558 0 : cl = ira_pressure_classes[pci];
3559 0 : group->limits[pci].pressure = curr_reg_pressure[cl];
3560 0 : group->limits[pci].point = 0;
3561 : }
3562 : /* Use index model_num_insns to record the state after the last
3563 : instruction in the model schedule. */
3564 0 : group->model = XNEWVEC (struct model_pressure_data,
3565 : (model_num_insns + 1) * ira_pressure_classes_num);
3566 0 : }
3567 :
3568 : /* Record that MODEL_REF_PRESSURE (GROUP, POINT, PCI) is PRESSURE.
3569 : Update the maximum pressure for the whole schedule. */
3570 :
3571 : static void
3572 0 : model_record_pressure (struct model_pressure_group *group,
3573 : int point, int pci, int pressure)
3574 : {
3575 0 : MODEL_REF_PRESSURE (group, point, pci) = pressure;
3576 0 : if (group->limits[pci].pressure < pressure)
3577 : {
3578 0 : group->limits[pci].pressure = pressure;
3579 0 : group->limits[pci].point = point;
3580 : }
3581 0 : }
3582 :
3583 : /* INSN has just been added to the end of the model schedule. Record its
3584 : register-pressure information. */
3585 :
3586 : static void
3587 0 : model_record_pressures (struct model_insn_info *insn)
3588 : {
3589 0 : struct reg_pressure_data *reg_pressure;
3590 0 : int point, pci, cl, delta;
3591 0 : int death[N_REG_CLASSES];
3592 :
3593 0 : point = model_index (insn->insn);
3594 0 : if (sched_verbose >= 2)
3595 : {
3596 0 : if (point == 0)
3597 : {
3598 0 : fprintf (sched_dump, "\n;;\tModel schedule:\n;;\n");
3599 0 : fprintf (sched_dump, ";;\t| idx insn | mpri hght dpth prio |\n");
3600 : }
3601 0 : fprintf (sched_dump, ";;\t| %3d %4d | %4d %4d %4d %4d | %-30s ",
3602 : point, INSN_UID (insn->insn), insn->model_priority,
3603 0 : insn->depth + insn->alap, insn->depth,
3604 0 : INSN_PRIORITY (insn->insn),
3605 0 : str_pattern_slim (PATTERN (insn->insn)));
3606 : }
3607 0 : calculate_reg_deaths (insn->insn, death);
3608 0 : reg_pressure = INSN_REG_PRESSURE (insn->insn);
3609 0 : for (pci = 0; pci < ira_pressure_classes_num; pci++)
3610 : {
3611 0 : cl = ira_pressure_classes[pci];
3612 0 : delta = reg_pressure[pci].set_increase - death[cl];
3613 0 : if (sched_verbose >= 2)
3614 0 : fprintf (sched_dump, " %s:[%d,%+d]", reg_class_names[cl],
3615 : curr_reg_pressure[cl], delta);
3616 0 : model_record_pressure (&model_before_pressure, point, pci,
3617 : curr_reg_pressure[cl]);
3618 : }
3619 0 : if (sched_verbose >= 2)
3620 0 : fprintf (sched_dump, "\n");
3621 0 : }
3622 :
3623 : /* All instructions have been added to the model schedule. Record the
3624 : final register pressure in GROUP and set up all MODEL_MAX_PRESSUREs. */
3625 :
3626 : static void
3627 0 : model_record_final_pressures (struct model_pressure_group *group)
3628 : {
3629 0 : int point, pci, max_pressure, ref_pressure, cl;
3630 :
3631 0 : for (pci = 0; pci < ira_pressure_classes_num; pci++)
3632 : {
3633 : /* Record the final pressure for this class. */
3634 0 : cl = ira_pressure_classes[pci];
3635 0 : point = model_num_insns;
3636 0 : ref_pressure = curr_reg_pressure[cl];
3637 0 : model_record_pressure (group, point, pci, ref_pressure);
3638 :
3639 : /* Record the original maximum pressure. */
3640 0 : group->limits[pci].orig_pressure = group->limits[pci].pressure;
3641 :
3642 : /* Update the MODEL_MAX_PRESSURE for every point of the schedule. */
3643 0 : max_pressure = ref_pressure;
3644 0 : MODEL_MAX_PRESSURE (group, point, pci) = max_pressure;
3645 0 : while (point > 0)
3646 : {
3647 0 : point--;
3648 0 : ref_pressure = MODEL_REF_PRESSURE (group, point, pci);
3649 0 : max_pressure = MAX (max_pressure, ref_pressure);
3650 0 : MODEL_MAX_PRESSURE (group, point, pci) = max_pressure;
3651 : }
3652 : }
3653 0 : }
3654 :
3655 : /* Update all successors of INSN, given that INSN has just been scheduled. */
3656 :
3657 : static void
3658 0 : model_add_successors_to_worklist (struct model_insn_info *insn)
3659 : {
3660 0 : sd_iterator_def sd_it;
3661 0 : struct model_insn_info *con;
3662 0 : dep_t dep;
3663 :
3664 0 : FOR_EACH_DEP (insn->insn, SD_LIST_FORW, sd_it, dep)
3665 : {
3666 0 : con = MODEL_INSN_INFO (DEP_CON (dep));
3667 : /* Ignore debug instructions, and instructions from other blocks. */
3668 0 : if (con->insn)
3669 : {
3670 0 : con->unscheduled_preds--;
3671 :
3672 : /* Update the depth field of each true-dependent successor.
3673 : Increasing the depth gives them a higher priority than
3674 : before. */
3675 0 : if (DEP_TYPE (dep) == REG_DEP_TRUE && con->depth < insn->depth + 1)
3676 : {
3677 0 : con->depth = insn->depth + 1;
3678 0 : if (QUEUE_INDEX (con->insn) == QUEUE_READY)
3679 0 : model_promote_insn (con);
3680 : }
3681 :
3682 : /* If this is a true dependency, or if there are no remaining
3683 : dependencies for CON (meaning that CON only had non-true
3684 : dependencies), make sure that CON is on the worklist.
3685 : We don't bother otherwise because it would tend to fill the
3686 : worklist with a lot of low-priority instructions that are not
3687 : yet ready to issue. */
3688 0 : if ((con->depth > 0 || con->unscheduled_preds == 0)
3689 0 : && QUEUE_INDEX (con->insn) == QUEUE_NOWHERE)
3690 0 : model_add_to_worklist (con, insn, insn->next);
3691 : }
3692 : }
3693 0 : }
3694 :
3695 : /* Give INSN a higher priority than any current instruction, then give
3696 : unscheduled predecessors of INSN a higher priority still. If any of
3697 : those predecessors are not on the model worklist, do the same for its
3698 : predecessors, and so on. */
3699 :
3700 : static void
3701 0 : model_promote_predecessors (struct model_insn_info *insn)
3702 : {
3703 0 : struct model_insn_info *pro, *first;
3704 0 : sd_iterator_def sd_it;
3705 0 : dep_t dep;
3706 :
3707 0 : if (sched_verbose >= 7)
3708 0 : fprintf (sched_dump, ";;\t+--- priority of %d = %d, priority of",
3709 0 : INSN_UID (insn->insn), model_next_priority);
3710 0 : insn->model_priority = model_next_priority++;
3711 0 : model_remove_from_worklist (insn);
3712 0 : model_add_to_worklist_at (insn, NULL);
3713 :
3714 0 : first = NULL;
3715 0 : for (;;)
3716 : {
3717 0 : FOR_EACH_DEP (insn->insn, SD_LIST_HARD_BACK, sd_it, dep)
3718 : {
3719 0 : pro = MODEL_INSN_INFO (DEP_PRO (dep));
3720 : /* The first test is to ignore debug instructions, and instructions
3721 : from other blocks. */
3722 0 : if (pro->insn
3723 0 : && pro->model_priority != model_next_priority
3724 0 : && QUEUE_INDEX (pro->insn) != QUEUE_SCHEDULED)
3725 : {
3726 0 : pro->model_priority = model_next_priority;
3727 0 : if (sched_verbose >= 7)
3728 0 : fprintf (sched_dump, " %d", INSN_UID (pro->insn));
3729 0 : if (QUEUE_INDEX (pro->insn) == QUEUE_READY)
3730 : {
3731 : /* PRO is already in the worklist, but it now has
3732 : a higher priority than before. Move it at the
3733 : appropriate place. */
3734 0 : model_remove_from_worklist (pro);
3735 0 : model_add_to_worklist (pro, NULL, model_worklist);
3736 : }
3737 : else
3738 : {
3739 : /* PRO isn't in the worklist. Recursively process
3740 : its predecessors until we find one that is. */
3741 0 : pro->next = first;
3742 0 : first = pro;
3743 : }
3744 : }
3745 : }
3746 0 : if (!first)
3747 : break;
3748 0 : insn = first;
3749 0 : first = insn->next;
3750 : }
3751 0 : if (sched_verbose >= 7)
3752 0 : fprintf (sched_dump, " = %d\n", model_next_priority);
3753 0 : model_next_priority++;
3754 0 : }
3755 :
3756 : /* Pick one instruction from model_worklist and process it. */
3757 :
3758 : static void
3759 0 : model_choose_insn (void)
3760 : {
3761 0 : struct model_insn_info *insn, *fallback;
3762 0 : int count;
3763 :
3764 0 : if (sched_verbose >= 7)
3765 : {
3766 0 : fprintf (sched_dump, ";;\t+--- worklist:\n");
3767 0 : insn = model_worklist;
3768 0 : count = param_max_sched_ready_insns;
3769 0 : while (count > 0 && insn)
3770 : {
3771 0 : fprintf (sched_dump, ";;\t+--- %d [%d, %d, %d, %d][%d]\n",
3772 : INSN_UID (insn->insn), insn->model_priority,
3773 0 : insn->depth + insn->alap, insn->depth,
3774 0 : INSN_PRIORITY (insn->insn), insn->unscheduled_preds);
3775 0 : count--;
3776 0 : insn = insn->next;
3777 : }
3778 : }
3779 :
3780 : /* Look for a ready instruction whose model_classify_priority is zero
3781 : or negative, picking the highest-priority one. Adding such an
3782 : instruction to the schedule now should do no harm, and may actually
3783 : do some good.
3784 :
3785 : Failing that, see whether there is an instruction with the highest
3786 : extant model_priority that is not yet ready, but which would reduce
3787 : pressure if it became ready. This is designed to catch cases like:
3788 :
3789 : (set (mem (reg R1)) (reg R2))
3790 :
3791 : where the instruction is the last remaining use of R1 and where the
3792 : value of R2 is not yet available (or vice versa). The death of R1
3793 : means that this instruction already reduces pressure. It is of
3794 : course possible that the computation of R2 involves other registers
3795 : that are hard to kill, but such cases are rare enough for this
3796 : heuristic to be a win in general.
3797 :
3798 : Failing that, just pick the highest-priority instruction in the
3799 : worklist. */
3800 0 : count = param_max_sched_ready_insns;
3801 0 : insn = model_worklist;
3802 0 : fallback = 0;
3803 0 : for (;;)
3804 : {
3805 0 : if (count == 0 || !insn)
3806 : {
3807 0 : insn = fallback ? fallback : model_worklist;
3808 : break;
3809 : }
3810 0 : if (insn->unscheduled_preds)
3811 : {
3812 0 : if (model_worklist->model_priority == insn->model_priority
3813 0 : && !fallback
3814 0 : && model_classify_pressure (insn) < 0)
3815 : fallback = insn;
3816 : }
3817 : else
3818 : {
3819 0 : if (model_classify_pressure (insn) <= 0)
3820 : break;
3821 : }
3822 0 : count--;
3823 0 : insn = insn->next;
3824 : }
3825 :
3826 0 : if (sched_verbose >= 7 && insn != model_worklist)
3827 : {
3828 0 : if (insn->unscheduled_preds)
3829 0 : fprintf (sched_dump, ";;\t+--- promoting insn %d, with dependencies\n",
3830 0 : INSN_UID (insn->insn));
3831 : else
3832 0 : fprintf (sched_dump, ";;\t+--- promoting insn %d, which is ready\n",
3833 0 : INSN_UID (insn->insn));
3834 : }
3835 0 : if (insn->unscheduled_preds)
3836 : /* INSN isn't yet ready to issue. Give all its predecessors the
3837 : highest priority. */
3838 0 : model_promote_predecessors (insn);
3839 : else
3840 : {
3841 : /* INSN is ready. Add it to the end of model_schedule and
3842 : process its successors. */
3843 0 : model_add_successors_to_worklist (insn);
3844 0 : model_remove_from_worklist (insn);
3845 0 : model_add_to_schedule (insn->insn);
3846 0 : model_record_pressures (insn);
3847 0 : update_register_pressure (insn->insn);
3848 : }
3849 0 : }
3850 :
3851 : /* Restore all QUEUE_INDEXs to the values that they had before
3852 : model_start_schedule was called. */
3853 :
3854 : static void
3855 0 : model_reset_queue_indices (void)
3856 : {
3857 0 : unsigned int i;
3858 0 : rtx_insn *insn;
3859 :
3860 0 : FOR_EACH_VEC_ELT (model_schedule, i, insn)
3861 0 : QUEUE_INDEX (insn) = MODEL_INSN_INFO (insn)->old_queue;
3862 0 : }
3863 :
3864 : /* We have calculated the model schedule and spill costs. Print a summary
3865 : to sched_dump. */
3866 :
3867 : static void
3868 0 : model_dump_pressure_summary (basic_block bb)
3869 : {
3870 0 : int pci, cl;
3871 :
3872 0 : fprintf (sched_dump, ";; Pressure summary (bb %d):", bb->index);
3873 0 : for (pci = 0; pci < ira_pressure_classes_num; pci++)
3874 : {
3875 0 : cl = ira_pressure_classes[pci];
3876 0 : fprintf (sched_dump, " %s:%d", reg_class_names[cl],
3877 : model_before_pressure.limits[pci].pressure);
3878 : }
3879 0 : fprintf (sched_dump, "\n\n");
3880 0 : }
3881 :
3882 : /* Initialize the SCHED_PRESSURE_MODEL information for the current
3883 : scheduling region. */
3884 :
3885 : static void
3886 0 : model_start_schedule (basic_block bb)
3887 : {
3888 0 : model_next_priority = 1;
3889 0 : model_schedule.create (sched_max_luid);
3890 0 : model_insns = XCNEWVEC (struct model_insn_info, sched_max_luid);
3891 :
3892 0 : gcc_assert (bb == BLOCK_FOR_INSN (NEXT_INSN (current_sched_info->prev_head)));
3893 0 : initiate_reg_pressure_info (df_get_live_in (bb));
3894 :
3895 0 : model_analyze_insns ();
3896 0 : model_init_pressure_group (&model_before_pressure);
3897 0 : while (model_worklist)
3898 0 : model_choose_insn ();
3899 0 : gcc_assert (model_num_insns == (int) model_schedule.length ());
3900 0 : if (sched_verbose >= 2)
3901 0 : fprintf (sched_dump, "\n");
3902 :
3903 0 : model_record_final_pressures (&model_before_pressure);
3904 0 : model_reset_queue_indices ();
3905 :
3906 0 : XDELETEVEC (model_insns);
3907 :
3908 0 : model_curr_point = 0;
3909 0 : initiate_reg_pressure_info (df_get_live_in (bb));
3910 0 : if (sched_verbose >= 1)
3911 0 : model_dump_pressure_summary (bb);
3912 0 : }
3913 :
3914 : /* Free the information associated with GROUP. */
3915 :
3916 : static void
3917 0 : model_finalize_pressure_group (struct model_pressure_group *group)
3918 : {
3919 0 : XDELETEVEC (group->model);
3920 0 : }
3921 :
3922 : /* Free the information created by model_start_schedule. */
3923 :
3924 : static void
3925 0 : model_end_schedule (void)
3926 : {
3927 0 : model_finalize_pressure_group (&model_before_pressure);
3928 0 : model_schedule.release ();
3929 0 : }
3930 :
3931 : /* Prepare reg pressure scheduling for basic block BB. */
3932 : static void
3933 706 : sched_pressure_start_bb (basic_block bb)
3934 : {
3935 : /* Set the number of available registers for each class taking into account
3936 : relative probability of current basic block versus function prologue and
3937 : epilogue.
3938 : * If the basic block executes much more often than the prologue/epilogue
3939 : (e.g., inside a hot loop), then cost of spill in the prologue is close to
3940 : nil, so the effective number of available registers is
3941 : (ira_class_hard_regs_num[cl] - fixed_regs_num[cl] - 0).
3942 : * If the basic block executes as often as the prologue/epilogue,
3943 : then spill in the block is as costly as in the prologue, so the effective
3944 : number of available registers is
3945 : (ira_class_hard_regs_num[cl] - fixed_regs_num[cl]
3946 : - call_saved_regs_num[cl]).
3947 : Note that all-else-equal, we prefer to spill in the prologue, since that
3948 : allows "extra" registers for other basic blocks of the function.
3949 : * If the basic block is on the cold path of the function and executes
3950 : rarely, then we should always prefer to spill in the block, rather than
3951 : in the prologue/epilogue. The effective number of available register is
3952 : (ira_class_hard_regs_num[cl] - fixed_regs_num[cl]
3953 : - call_saved_regs_num[cl]). */
3954 706 : {
3955 706 : int i;
3956 706 : int entry_freq = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count.to_frequency (cfun);
3957 706 : int bb_freq = bb->count.to_frequency (cfun);
3958 :
3959 706 : if (bb_freq == 0)
3960 : {
3961 128 : if (entry_freq == 0)
3962 : entry_freq = bb_freq = 1;
3963 : }
3964 706 : if (bb_freq < entry_freq)
3965 : bb_freq = entry_freq;
3966 :
3967 3534 : for (i = 0; i < ira_pressure_classes_num; ++i)
3968 : {
3969 2828 : enum reg_class cl = ira_pressure_classes[i];
3970 2828 : sched_class_regs_num[cl] = ira_class_hard_regs_num[cl]
3971 2828 : - fixed_regs_num[cl];
3972 2828 : sched_class_regs_num[cl]
3973 2828 : -= (call_saved_regs_num[cl] * entry_freq) / bb_freq;
3974 : }
3975 : }
3976 :
3977 706 : if (sched_pressure == SCHED_PRESSURE_MODEL)
3978 0 : model_start_schedule (bb);
3979 706 : }
3980 :
3981 : /* A structure that holds local state for the loop in schedule_block. */
3982 : struct sched_block_state
3983 : {
3984 : /* True if no real insns have been scheduled in the current cycle. */
3985 : bool first_cycle_insn_p;
3986 : /* True if a shadow insn has been scheduled in the current cycle, which
3987 : means that no more normal insns can be issued. */
3988 : bool shadows_only_p;
3989 : /* True if we're winding down a modulo schedule, which means that we only
3990 : issue insns with INSN_EXACT_TICK set. */
3991 : bool modulo_epilogue;
3992 : /* Initialized with the machine's issue rate every cycle, and updated
3993 : by calls to the variable_issue hook. */
3994 : int can_issue_more;
3995 : };
3996 :
3997 : /* INSN is the "currently executing insn". Launch each insn which was
3998 : waiting on INSN. READY is the ready list which contains the insns
3999 : that are ready to fire. CLOCK is the current cycle. The function
4000 : returns necessary cycle advance after issuing the insn (it is not
4001 : zero for insns in a schedule group). */
4002 :
4003 : static int
4004 114313473 : schedule_insn (rtx_insn *insn)
4005 : {
4006 114313473 : sd_iterator_def sd_it;
4007 114313473 : dep_t dep;
4008 114313473 : int i;
4009 114313473 : int advance = 0;
4010 :
4011 114313473 : if (sched_verbose >= 1)
4012 : {
4013 1004 : struct reg_pressure_data *pressure_info;
4014 1004 : fprintf (sched_dump, ";;\t%3i--> %s %-40s:",
4015 1004 : clock_var, (*current_sched_info->print_insn) (insn, 1),
4016 1004 : str_pattern_slim (PATTERN (insn)));
4017 :
4018 1004 : if (recog_memoized (insn) < 0)
4019 134 : fprintf (sched_dump, "nothing");
4020 : else
4021 870 : print_reservation (sched_dump, insn);
4022 1004 : pressure_info = INSN_REG_PRESSURE (insn);
4023 1004 : if (pressure_info != NULL)
4024 : {
4025 0 : fputc (':', sched_dump);
4026 0 : for (i = 0; i < ira_pressure_classes_num; i++)
4027 0 : fprintf (sched_dump, "%s%s%+d(%d)",
4028 0 : scheduled_insns.length () > 1
4029 0 : && INSN_LUID (insn)
4030 0 : < INSN_LUID (scheduled_insns[scheduled_insns.length () - 2]) ? "@" : "",
4031 0 : reg_class_names[ira_pressure_classes[i]],
4032 0 : pressure_info[i].set_increase, pressure_info[i].change);
4033 : }
4034 1004 : if (sched_pressure == SCHED_PRESSURE_MODEL
4035 0 : && model_curr_point < model_num_insns
4036 1004 : && model_index (insn) == model_curr_point)
4037 0 : fprintf (sched_dump, ":model %d", model_curr_point);
4038 1004 : fputc ('\n', sched_dump);
4039 : }
4040 :
4041 114313473 : if (sched_pressure == SCHED_PRESSURE_WEIGHTED && !DEBUG_INSN_P (insn))
4042 3764 : update_reg_and_insn_max_reg_pressure (insn);
4043 :
4044 : /* Scheduling instruction should have all its dependencies resolved and
4045 : should have been removed from the ready list. */
4046 114313473 : gcc_assert (sd_lists_empty_p (insn, SD_LIST_HARD_BACK));
4047 :
4048 : /* Reset debug insns invalidated by moving this insn. */
4049 114313473 : if (MAY_HAVE_DEBUG_BIND_INSNS && !DEBUG_INSN_P (insn))
4050 42318762 : for (sd_it = sd_iterator_start (insn, SD_LIST_BACK);
4051 42851993 : sd_iterator_cond (&sd_it, &dep);)
4052 : {
4053 533231 : rtx_insn *dbg = DEP_PRO (dep);
4054 533231 : struct reg_use_data *use, *next;
4055 :
4056 533231 : if (DEP_STATUS (dep) & DEP_CANCELLED)
4057 : {
4058 382558 : sd_iterator_next (&sd_it);
4059 382558 : continue;
4060 : }
4061 :
4062 150673 : gcc_assert (DEBUG_BIND_INSN_P (dbg));
4063 :
4064 150673 : if (sched_verbose >= 6)
4065 0 : fprintf (sched_dump, ";;\t\tresetting: debug insn %d\n",
4066 0 : INSN_UID (dbg));
4067 :
4068 : /* ??? Rather than resetting the debug insn, we might be able
4069 : to emit a debug temp before the just-scheduled insn, but
4070 : this would involve checking that the expression at the
4071 : point of the debug insn is equivalent to the expression
4072 : before the just-scheduled insn. They might not be: the
4073 : expression in the debug insn may depend on other insns not
4074 : yet scheduled that set MEMs, REGs or even other debug
4075 : insns. It's not clear that attempting to preserve debug
4076 : information in these cases is worth the effort, given how
4077 : uncommon these resets are and the likelihood that the debug
4078 : temps introduced won't survive the schedule change. */
4079 150673 : INSN_VAR_LOCATION_LOC (dbg) = gen_rtx_UNKNOWN_VAR_LOC ();
4080 150673 : df_insn_rescan (dbg);
4081 :
4082 : /* Unknown location doesn't use any registers. */
4083 150673 : for (use = INSN_REG_USE_LIST (dbg); use != NULL; use = next)
4084 : {
4085 : struct reg_use_data *prev = use;
4086 :
4087 : /* Remove use from the cyclic next_regno_use chain first. */
4088 0 : while (prev->next_regno_use != use)
4089 : prev = prev->next_regno_use;
4090 0 : prev->next_regno_use = use->next_regno_use;
4091 0 : next = use->next_insn_use;
4092 0 : free (use);
4093 : }
4094 150673 : INSN_REG_USE_LIST (dbg) = NULL;
4095 :
4096 : /* We delete rather than resolve these deps, otherwise we
4097 : crash in sched_free_deps(), because forward deps are
4098 : expected to be released before backward deps. */
4099 150673 : sd_delete_dep (sd_it);
4100 : }
4101 :
4102 114313473 : gcc_assert (QUEUE_INDEX (insn) == QUEUE_NOWHERE);
4103 114313473 : QUEUE_INDEX (insn) = QUEUE_SCHEDULED;
4104 :
4105 114313473 : if (sched_pressure == SCHED_PRESSURE_MODEL
4106 0 : && model_curr_point < model_num_insns
4107 0 : && NONDEBUG_INSN_P (insn))
4108 : {
4109 0 : if (model_index (insn) == model_curr_point)
4110 0 : do
4111 0 : model_curr_point++;
4112 : while (model_curr_point < model_num_insns
4113 0 : && (QUEUE_INDEX (MODEL_INSN (model_curr_point))
4114 : == QUEUE_SCHEDULED));
4115 : else
4116 0 : model_recompute (insn);
4117 0 : model_update_limit_points ();
4118 0 : update_register_pressure (insn);
4119 0 : if (sched_verbose >= 2)
4120 0 : print_curr_reg_pressure ();
4121 : }
4122 :
4123 114313473 : gcc_assert (INSN_TICK (insn) >= MIN_TICK);
4124 114313473 : if (INSN_TICK (insn) > clock_var)
4125 : /* INSN has been prematurely moved from the queue to the ready list.
4126 : This is possible only if following flags are set. */
4127 5 : gcc_assert (flag_sched_stalled_insns || sched_fusion);
4128 :
4129 : /* ??? Probably, if INSN is scheduled prematurely, we should leave
4130 : INSN_TICK untouched. This is a machine-dependent issue, actually. */
4131 114313473 : INSN_TICK (insn) = clock_var;
4132 :
4133 114313473 : check_clobbered_conditions (insn);
4134 :
4135 : /* Update dependent instructions. First, see if by scheduling this insn
4136 : now we broke a dependence in a way that requires us to change another
4137 : insn. */
4138 114313473 : for (sd_it = sd_iterator_start (insn, SD_LIST_SPEC_BACK);
4139 114861043 : sd_iterator_cond (&sd_it, &dep); sd_iterator_next (&sd_it))
4140 : {
4141 547570 : struct dep_replacement *desc = DEP_REPLACE (dep);
4142 547570 : rtx_insn *pro = DEP_PRO (dep);
4143 547570 : if (QUEUE_INDEX (pro) != QUEUE_SCHEDULED
4144 547570 : && desc != NULL && desc->insn == pro)
4145 519202 : apply_replacement (dep, false);
4146 : }
4147 :
4148 : /* Go through and resolve forward dependencies. */
4149 114313473 : for (sd_it = sd_iterator_start (insn, SD_LIST_FORW);
4150 331091066 : sd_iterator_cond (&sd_it, &dep);)
4151 : {
4152 216777593 : rtx_insn *next = DEP_CON (dep);
4153 216777593 : bool cancelled = (DEP_STATUS (dep) & DEP_CANCELLED) != 0;
4154 :
4155 : /* Resolve the dependence between INSN and NEXT.
4156 : sd_resolve_dep () moves current dep to another list thus
4157 : advancing the iterator. */
4158 216777593 : sd_resolve_dep (sd_it);
4159 :
4160 216777593 : if (cancelled)
4161 : {
4162 850456 : if (must_restore_pattern_p (next, dep))
4163 61107 : restore_pattern (dep, false);
4164 850456 : continue;
4165 : }
4166 :
4167 : /* Don't bother trying to mark next as ready if insn is a debug
4168 : insn. If insn is the last hard dependency, it will have
4169 : already been discounted. */
4170 215927137 : if (DEBUG_INSN_P (insn) && !DEBUG_INSN_P (next))
4171 5072775 : continue;
4172 :
4173 210854362 : if (!IS_SPECULATION_BRANCHY_CHECK_P (insn))
4174 : {
4175 210854362 : int effective_cost;
4176 :
4177 210854362 : effective_cost = try_ready (next);
4178 :
4179 210854362 : if (effective_cost >= 0
4180 19868930 : && SCHED_GROUP_P (next)
4181 210854362 : && advance < effective_cost)
4182 : advance = effective_cost;
4183 : }
4184 : else
4185 : /* Check always has only one forward dependence (to the first insn in
4186 : the recovery block), therefore, this will be executed only once. */
4187 : {
4188 0 : gcc_assert (sd_lists_empty_p (insn, SD_LIST_FORW));
4189 0 : fix_recovery_deps (RECOVERY_BLOCK (insn));
4190 : }
4191 : }
4192 :
4193 : /* Annotate the instruction with issue information -- TImode
4194 : indicates that the instruction is expected not to be able
4195 : to issue on the same cycle as the previous insn. A machine
4196 : may use this information to decide how the instruction should
4197 : be aligned. */
4198 114313473 : if (issue_rate > 1
4199 114312610 : && GET_CODE (PATTERN (insn)) != USE
4200 113595627 : && GET_CODE (PATTERN (insn)) != CLOBBER
4201 227825899 : && !DEBUG_INSN_P (insn))
4202 : {
4203 60660009 : if (reload_completed)
4204 96348591 : PUT_MODE (insn, clock_var > last_clock_var ? TImode : VOIDmode);
4205 60660009 : last_clock_var = clock_var;
4206 : }
4207 :
4208 114313473 : if (nonscheduled_insns_begin != NULL_RTX)
4209 : /* Indicate to debug counters that INSN is scheduled. */
4210 0 : nonscheduled_insns_begin = insn;
4211 :
4212 114313473 : return advance;
4213 : }
4214 :
4215 : /* Functions for handling of notes. */
4216 :
4217 : /* Add note list that ends on FROM_END to the end of TO_ENDP. */
4218 : void
4219 64 : concat_note_lists (rtx_insn *from_end, rtx_insn **to_endp)
4220 : {
4221 64 : rtx_insn *from_start;
4222 :
4223 : /* It's easy when have nothing to concat. */
4224 64 : if (from_end == NULL)
4225 : return;
4226 :
4227 : /* It's also easy when destination is empty. */
4228 0 : if (*to_endp == NULL)
4229 : {
4230 0 : *to_endp = from_end;
4231 0 : return;
4232 : }
4233 :
4234 : from_start = from_end;
4235 0 : while (PREV_INSN (from_start) != NULL)
4236 : from_start = PREV_INSN (from_start);
4237 :
4238 0 : SET_PREV_INSN (from_start) = *to_endp;
4239 0 : SET_NEXT_INSN (*to_endp) = from_start;
4240 0 : *to_endp = from_end;
4241 : }
4242 :
4243 : /* Delete notes between HEAD and TAIL and put them in the chain
4244 : of notes ended by NOTE_LIST. */
4245 : void
4246 10478747 : remove_notes (rtx_insn *head, rtx_insn *tail)
4247 : {
4248 10478747 : rtx_insn *next_tail, *insn, *next;
4249 :
4250 10478747 : note_list = 0;
4251 10478747 : if (head == tail && !INSN_P (head))
4252 : return;
4253 :
4254 10478716 : next_tail = NEXT_INSN (tail);
4255 141196465 : for (insn = head; insn != next_tail; insn = next)
4256 : {
4257 120239033 : next = NEXT_INSN (insn);
4258 120239033 : if (!NOTE_P (insn))
4259 114318160 : continue;
4260 :
4261 5920873 : switch (NOTE_KIND (insn))
4262 : {
4263 1086 : case NOTE_INSN_BASIC_BLOCK:
4264 1086 : continue;
4265 :
4266 927461 : case NOTE_INSN_EPILOGUE_BEG:
4267 927461 : if (insn != tail)
4268 : {
4269 927461 : remove_insn (insn);
4270 : /* If an insn was split just before the EPILOGUE_BEG note and
4271 : that split created new basic blocks, we could have a
4272 : BASIC_BLOCK note here. Safely advance over it in that case
4273 : and assert that we land on a real insn. */
4274 927461 : if (NOTE_P (next)
4275 0 : && NOTE_KIND (next) == NOTE_INSN_BASIC_BLOCK
4276 0 : && next != next_tail)
4277 0 : next = NEXT_INSN (next);
4278 927461 : gcc_assert (INSN_P (next));
4279 927461 : add_reg_note (next, REG_SAVE_NOTE,
4280 : GEN_INT (NOTE_INSN_EPILOGUE_BEG));
4281 927461 : break;
4282 : }
4283 : /* FALLTHRU */
4284 :
4285 4992326 : default:
4286 4992326 : remove_insn (insn);
4287 :
4288 : /* Add the note to list that ends at NOTE_LIST. */
4289 4992326 : SET_PREV_INSN (insn) = note_list;
4290 4992326 : SET_NEXT_INSN (insn) = NULL_RTX;
4291 4992326 : if (note_list)
4292 2875146 : SET_NEXT_INSN (note_list) = insn;
4293 4992326 : note_list = insn;
4294 4992326 : break;
4295 : }
4296 :
4297 5919787 : gcc_assert ((sel_sched_p () || insn != tail) && insn != head);
4298 : }
4299 : }
4300 :
4301 : /* A structure to record enough data to allow us to backtrack the scheduler to
4302 : a previous state. */
4303 : struct haifa_saved_data
4304 : {
4305 : /* Next entry on the list. */
4306 : struct haifa_saved_data *next;
4307 :
4308 : /* Backtracking is associated with scheduling insns that have delay slots.
4309 : DELAY_PAIR points to the structure that contains the insns involved, and
4310 : the number of cycles between them. */
4311 : struct delay_pair *delay_pair;
4312 :
4313 : /* Data used by the frontend (e.g. sched-ebb or sched-rgn). */
4314 : void *fe_saved_data;
4315 : /* Data used by the backend. */
4316 : void *be_saved_data;
4317 :
4318 : /* Copies of global state. */
4319 : int clock_var, last_clock_var;
4320 : struct ready_list ready;
4321 : state_t curr_state;
4322 :
4323 : rtx_insn *last_scheduled_insn;
4324 : rtx_insn *last_nondebug_scheduled_insn;
4325 : rtx_insn *nonscheduled_insns_begin;
4326 : int cycle_issued_insns;
4327 :
4328 : /* Copies of state used in the inner loop of schedule_block. */
4329 : struct sched_block_state sched_block;
4330 :
4331 : /* We don't need to save q_ptr, as its value is arbitrary and we can set it
4332 : to 0 when restoring. */
4333 : int q_size;
4334 : rtx_insn_list **insn_queue;
4335 :
4336 : /* Describe pattern replacements that occurred since this backtrack point
4337 : was queued. */
4338 : vec<dep_t> replacement_deps;
4339 : vec<int> replace_apply;
4340 :
4341 : /* A copy of the next-cycle replacement vectors at the time of the backtrack
4342 : point. */
4343 : vec<dep_t> next_cycle_deps;
4344 : vec<int> next_cycle_apply;
4345 : };
4346 :
4347 : /* A record, in reverse order, of all scheduled insns which have delay slots
4348 : and may require backtracking. */
4349 : static struct haifa_saved_data *backtrack_queue;
4350 :
4351 : /* For every dependency of INSN, set the FEEDS_BACKTRACK_INSN bit according
4352 : to SET_P. */
4353 : static void
4354 0 : mark_backtrack_feeds (rtx_insn *insn, int set_p)
4355 : {
4356 0 : sd_iterator_def sd_it;
4357 0 : dep_t dep;
4358 0 : FOR_EACH_DEP (insn, SD_LIST_HARD_BACK, sd_it, dep)
4359 : {
4360 0 : FEEDS_BACKTRACK_INSN (DEP_PRO (dep)) = set_p;
4361 : }
4362 0 : }
4363 :
4364 : /* Save the current scheduler state so that we can backtrack to it
4365 : later if necessary. PAIR gives the insns that make it necessary to
4366 : save this point. SCHED_BLOCK is the local state of schedule_block
4367 : that need to be saved. */
4368 : static void
4369 0 : save_backtrack_point (struct delay_pair *pair,
4370 : struct sched_block_state sched_block)
4371 : {
4372 0 : int i;
4373 0 : struct haifa_saved_data *save = XNEW (struct haifa_saved_data);
4374 :
4375 0 : save->curr_state = xmalloc (dfa_state_size);
4376 0 : memcpy (save->curr_state, curr_state, dfa_state_size);
4377 :
4378 0 : save->ready.first = ready.first;
4379 0 : save->ready.n_ready = ready.n_ready;
4380 0 : save->ready.n_debug = ready.n_debug;
4381 0 : save->ready.veclen = ready.veclen;
4382 0 : save->ready.vec = XNEWVEC (rtx_insn *, ready.veclen);
4383 0 : memcpy (save->ready.vec, ready.vec, ready.veclen * sizeof (rtx));
4384 :
4385 0 : save->insn_queue = XNEWVEC (rtx_insn_list *, max_insn_queue_index + 1);
4386 0 : save->q_size = q_size;
4387 0 : for (i = 0; i <= max_insn_queue_index; i++)
4388 : {
4389 0 : int q = NEXT_Q_AFTER (q_ptr, i);
4390 0 : save->insn_queue[i] = copy_INSN_LIST (insn_queue[q]);
4391 : }
4392 :
4393 0 : save->clock_var = clock_var;
4394 0 : save->last_clock_var = last_clock_var;
4395 0 : save->cycle_issued_insns = cycle_issued_insns;
4396 0 : save->last_scheduled_insn = last_scheduled_insn;
4397 0 : save->last_nondebug_scheduled_insn = last_nondebug_scheduled_insn;
4398 0 : save->nonscheduled_insns_begin = nonscheduled_insns_begin;
4399 :
4400 0 : save->sched_block = sched_block;
4401 :
4402 0 : save->replacement_deps.create (0);
4403 0 : save->replace_apply.create (0);
4404 0 : save->next_cycle_deps = next_cycle_replace_deps.copy ();
4405 0 : save->next_cycle_apply = next_cycle_apply.copy ();
4406 :
4407 0 : if (current_sched_info->save_state)
4408 0 : save->fe_saved_data = (*current_sched_info->save_state) ();
4409 :
4410 0 : if (targetm.sched.alloc_sched_context)
4411 : {
4412 0 : save->be_saved_data = targetm.sched.alloc_sched_context ();
4413 0 : targetm.sched.init_sched_context (save->be_saved_data, false);
4414 : }
4415 : else
4416 0 : save->be_saved_data = NULL;
4417 :
4418 0 : save->delay_pair = pair;
4419 :
4420 0 : save->next = backtrack_queue;
4421 0 : backtrack_queue = save;
4422 :
4423 0 : while (pair)
4424 : {
4425 0 : mark_backtrack_feeds (pair->i2, 1);
4426 0 : INSN_TICK (pair->i2) = INVALID_TICK;
4427 0 : INSN_EXACT_TICK (pair->i2) = clock_var + pair_delay (pair);
4428 0 : SHADOW_P (pair->i2) = pair->stages == 0;
4429 0 : pair = pair->next_same_i1;
4430 : }
4431 0 : }
4432 :
4433 : /* Walk the ready list and all queues. If any insns have unresolved backwards
4434 : dependencies, these must be cancelled deps, broken by predication. Set or
4435 : clear (depending on SET) the DEP_CANCELLED bit in DEP_STATUS. */
4436 :
4437 : static void
4438 0 : toggle_cancelled_flags (bool set)
4439 : {
4440 0 : int i;
4441 0 : sd_iterator_def sd_it;
4442 0 : dep_t dep;
4443 :
4444 0 : if (ready.n_ready > 0)
4445 : {
4446 0 : rtx_insn **first = ready_lastpos (&ready);
4447 0 : for (i = 0; i < ready.n_ready; i++)
4448 0 : FOR_EACH_DEP (first[i], SD_LIST_BACK, sd_it, dep)
4449 0 : if (!DEBUG_INSN_P (DEP_PRO (dep)))
4450 : {
4451 0 : if (set)
4452 0 : DEP_STATUS (dep) |= DEP_CANCELLED;
4453 : else
4454 0 : DEP_STATUS (dep) &= ~DEP_CANCELLED;
4455 : }
4456 : }
4457 0 : for (i = 0; i <= max_insn_queue_index; i++)
4458 : {
4459 0 : int q = NEXT_Q_AFTER (q_ptr, i);
4460 0 : rtx_insn_list *link;
4461 0 : for (link = insn_queue[q]; link; link = link->next ())
4462 : {
4463 0 : rtx_insn *insn = link->insn ();
4464 0 : FOR_EACH_DEP (insn, SD_LIST_BACK, sd_it, dep)
4465 0 : if (!DEBUG_INSN_P (DEP_PRO (dep)))
4466 : {
4467 0 : if (set)
4468 0 : DEP_STATUS (dep) |= DEP_CANCELLED;
4469 : else
4470 0 : DEP_STATUS (dep) &= ~DEP_CANCELLED;
4471 : }
4472 : }
4473 : }
4474 0 : }
4475 :
4476 : /* Undo the replacements that have occurred after backtrack point SAVE
4477 : was placed. */
4478 : static void
4479 0 : undo_replacements_for_backtrack (struct haifa_saved_data *save)
4480 : {
4481 0 : while (!save->replacement_deps.is_empty ())
4482 : {
4483 0 : dep_t dep = save->replacement_deps.pop ();
4484 0 : int apply_p = save->replace_apply.pop ();
4485 :
4486 0 : if (apply_p)
4487 0 : restore_pattern (dep, true);
4488 : else
4489 0 : apply_replacement (dep, true);
4490 : }
4491 0 : save->replacement_deps.release ();
4492 0 : save->replace_apply.release ();
4493 0 : }
4494 :
4495 : /* Pop entries from the SCHEDULED_INSNS vector up to and including INSN.
4496 : Restore their dependencies to an unresolved state, and mark them as
4497 : queued nowhere. */
4498 :
4499 : static void
4500 0 : unschedule_insns_until (rtx_insn *insn)
4501 : {
4502 0 : auto_vec<rtx_insn *> recompute_vec;
4503 :
4504 : /* Make two passes over the insns to be unscheduled. First, we clear out
4505 : dependencies and other trivial bookkeeping. */
4506 0 : for (;;)
4507 : {
4508 0 : rtx_insn *last;
4509 0 : sd_iterator_def sd_it;
4510 0 : dep_t dep;
4511 :
4512 0 : last = scheduled_insns.pop ();
4513 :
4514 : /* This will be changed by restore_backtrack_point if the insn is in
4515 : any queue. */
4516 0 : QUEUE_INDEX (last) = QUEUE_NOWHERE;
4517 0 : if (last != insn)
4518 0 : INSN_TICK (last) = INVALID_TICK;
4519 :
4520 0 : if (modulo_ii > 0 && INSN_UID (last) < modulo_iter0_max_uid)
4521 0 : modulo_insns_scheduled--;
4522 :
4523 0 : for (sd_it = sd_iterator_start (last, SD_LIST_RES_FORW);
4524 0 : sd_iterator_cond (&sd_it, &dep);)
4525 : {
4526 0 : rtx_insn *con = DEP_CON (dep);
4527 0 : sd_unresolve_dep (sd_it);
4528 0 : if (!MUST_RECOMPUTE_SPEC_P (con))
4529 : {
4530 0 : MUST_RECOMPUTE_SPEC_P (con) = 1;
4531 0 : recompute_vec.safe_push (con);
4532 : }
4533 : }
4534 :
4535 0 : if (last == insn)
4536 : break;
4537 0 : }
4538 :
4539 : /* A second pass, to update ready and speculation status for insns
4540 : depending on the unscheduled ones. The first pass must have
4541 : popped the scheduled_insns vector up to the point where we
4542 : restart scheduling, as recompute_todo_spec requires it to be
4543 : up-to-date. */
4544 0 : while (!recompute_vec.is_empty ())
4545 : {
4546 0 : rtx_insn *con;
4547 :
4548 0 : con = recompute_vec.pop ();
4549 0 : MUST_RECOMPUTE_SPEC_P (con) = 0;
4550 0 : if (!sd_lists_empty_p (con, SD_LIST_HARD_BACK))
4551 : {
4552 0 : TODO_SPEC (con) = HARD_DEP;
4553 0 : INSN_TICK (con) = INVALID_TICK;
4554 0 : if (PREDICATED_PAT (con) != NULL_RTX)
4555 0 : haifa_change_pattern (con, ORIG_PAT (con));
4556 : }
4557 0 : else if (QUEUE_INDEX (con) != QUEUE_SCHEDULED)
4558 0 : TODO_SPEC (con) = recompute_todo_spec (con, true);
4559 : }
4560 0 : }
4561 :
4562 : /* Restore scheduler state from the topmost entry on the backtracking queue.
4563 : PSCHED_BLOCK_P points to the local data of schedule_block that we must
4564 : overwrite with the saved data.
4565 : The caller must already have called unschedule_insns_until. */
4566 :
4567 : static void
4568 0 : restore_last_backtrack_point (struct sched_block_state *psched_block)
4569 : {
4570 0 : int i;
4571 0 : struct haifa_saved_data *save = backtrack_queue;
4572 :
4573 0 : backtrack_queue = save->next;
4574 :
4575 0 : if (current_sched_info->restore_state)
4576 0 : (*current_sched_info->restore_state) (save->fe_saved_data);
4577 :
4578 0 : if (targetm.sched.alloc_sched_context)
4579 : {
4580 0 : targetm.sched.set_sched_context (save->be_saved_data);
4581 0 : targetm.sched.free_sched_context (save->be_saved_data);
4582 : }
4583 :
4584 : /* Do this first since it clobbers INSN_TICK of the involved
4585 : instructions. */
4586 0 : undo_replacements_for_backtrack (save);
4587 :
4588 : /* Clear the QUEUE_INDEX of everything in the ready list or one
4589 : of the queues. */
4590 0 : if (ready.n_ready > 0)
4591 : {
4592 0 : rtx_insn **first = ready_lastpos (&ready);
4593 0 : for (i = 0; i < ready.n_ready; i++)
4594 : {
4595 0 : rtx_insn *insn = first[i];
4596 0 : QUEUE_INDEX (insn) = QUEUE_NOWHERE;
4597 0 : INSN_TICK (insn) = INVALID_TICK;
4598 : }
4599 : }
4600 0 : for (i = 0; i <= max_insn_queue_index; i++)
4601 : {
4602 0 : int q = NEXT_Q_AFTER (q_ptr, i);
4603 :
4604 0 : for (rtx_insn_list *link = insn_queue[q]; link; link = link->next ())
4605 : {
4606 0 : rtx_insn *x = link->insn ();
4607 0 : QUEUE_INDEX (x) = QUEUE_NOWHERE;
4608 0 : INSN_TICK (x) = INVALID_TICK;
4609 : }
4610 0 : free_INSN_LIST_list (&insn_queue[q]);
4611 : }
4612 :
4613 0 : free (ready.vec);
4614 0 : ready = save->ready;
4615 :
4616 0 : if (ready.n_ready > 0)
4617 : {
4618 0 : rtx_insn **first = ready_lastpos (&ready);
4619 0 : for (i = 0; i < ready.n_ready; i++)
4620 : {
4621 0 : rtx_insn *insn = first[i];
4622 0 : QUEUE_INDEX (insn) = QUEUE_READY;
4623 0 : TODO_SPEC (insn) = recompute_todo_spec (insn, true);
4624 0 : INSN_TICK (insn) = save->clock_var;
4625 : }
4626 : }
4627 :
4628 0 : q_ptr = 0;
4629 0 : q_size = save->q_size;
4630 0 : for (i = 0; i <= max_insn_queue_index; i++)
4631 : {
4632 0 : int q = NEXT_Q_AFTER (q_ptr, i);
4633 :
4634 0 : insn_queue[q] = save->insn_queue[q];
4635 :
4636 0 : for (rtx_insn_list *link = insn_queue[q]; link; link = link->next ())
4637 : {
4638 0 : rtx_insn *x = link->insn ();
4639 0 : QUEUE_INDEX (x) = i;
4640 0 : TODO_SPEC (x) = recompute_todo_spec (x, true);
4641 0 : INSN_TICK (x) = save->clock_var + i;
4642 : }
4643 : }
4644 0 : free (save->insn_queue);
4645 :
4646 0 : toggle_cancelled_flags (true);
4647 :
4648 0 : clock_var = save->clock_var;
4649 0 : last_clock_var = save->last_clock_var;
4650 0 : cycle_issued_insns = save->cycle_issued_insns;
4651 0 : last_scheduled_insn = save->last_scheduled_insn;
4652 0 : last_nondebug_scheduled_insn = save->last_nondebug_scheduled_insn;
4653 0 : nonscheduled_insns_begin = save->nonscheduled_insns_begin;
4654 :
4655 0 : *psched_block = save->sched_block;
4656 :
4657 0 : memcpy (curr_state, save->curr_state, dfa_state_size);
4658 0 : free (save->curr_state);
4659 :
4660 0 : mark_backtrack_feeds (save->delay_pair->i2, 0);
4661 :
4662 0 : gcc_assert (next_cycle_replace_deps.is_empty ());
4663 0 : next_cycle_replace_deps = save->next_cycle_deps.copy ();
4664 0 : next_cycle_apply = save->next_cycle_apply.copy ();
4665 :
4666 0 : free (save);
4667 :
4668 0 : for (save = backtrack_queue; save; save = save->next)
4669 : {
4670 0 : mark_backtrack_feeds (save->delay_pair->i2, 1);
4671 : }
4672 0 : }
4673 :
4674 : /* Discard all data associated with the topmost entry in the backtrack
4675 : queue. If RESET_TICK is false, we just want to free the data. If true,
4676 : we are doing this because we discovered a reason to backtrack. In the
4677 : latter case, also reset the INSN_TICK for the shadow insn. */
4678 : static void
4679 0 : free_topmost_backtrack_point (bool reset_tick)
4680 : {
4681 0 : struct haifa_saved_data *save = backtrack_queue;
4682 0 : int i;
4683 :
4684 0 : backtrack_queue = save->next;
4685 :
4686 0 : if (reset_tick)
4687 : {
4688 0 : struct delay_pair *pair = save->delay_pair;
4689 0 : while (pair)
4690 : {
4691 0 : INSN_TICK (pair->i2) = INVALID_TICK;
4692 0 : INSN_EXACT_TICK (pair->i2) = INVALID_TICK;
4693 0 : pair = pair->next_same_i1;
4694 : }
4695 0 : undo_replacements_for_backtrack (save);
4696 : }
4697 : else
4698 : {
4699 0 : save->replacement_deps.release ();
4700 0 : save->replace_apply.release ();
4701 : }
4702 :
4703 0 : if (targetm.sched.free_sched_context)
4704 0 : targetm.sched.free_sched_context (save->be_saved_data);
4705 0 : if (current_sched_info->restore_state)
4706 0 : free (save->fe_saved_data);
4707 0 : for (i = 0; i <= max_insn_queue_index; i++)
4708 0 : free_INSN_LIST_list (&save->insn_queue[i]);
4709 0 : free (save->insn_queue);
4710 0 : free (save->curr_state);
4711 0 : free (save->ready.vec);
4712 0 : free (save);
4713 0 : }
4714 :
4715 : /* Free the entire backtrack queue. */
4716 : static void
4717 10477678 : free_backtrack_queue (void)
4718 : {
4719 10477678 : while (backtrack_queue)
4720 0 : free_topmost_backtrack_point (false);
4721 10477678 : }
4722 :
4723 : /* Apply a replacement described by DESC. If IMMEDIATELY is false, we
4724 : may have to postpone the replacement until the start of the next cycle,
4725 : at which point we will be called again with IMMEDIATELY true. This is
4726 : only done for machines which have instruction packets with explicit
4727 : parallelism however. */
4728 : static void
4729 608677 : apply_replacement (dep_t dep, bool immediately)
4730 : {
4731 608677 : struct dep_replacement *desc = DEP_REPLACE (dep);
4732 608677 : if (!immediately && targetm.sched.exposed_pipeline && reload_completed)
4733 : {
4734 0 : next_cycle_replace_deps.safe_push (dep);
4735 0 : next_cycle_apply.safe_push (1);
4736 : }
4737 : else
4738 : {
4739 608677 : bool success;
4740 :
4741 608677 : if (QUEUE_INDEX (desc->insn) == QUEUE_SCHEDULED)
4742 : return;
4743 :
4744 608677 : if (sched_verbose >= 5)
4745 0 : fprintf (sched_dump, "applying replacement for insn %d\n",
4746 : INSN_UID (desc->insn));
4747 :
4748 608677 : success = validate_change (desc->insn, desc->loc, desc->newval, 0);
4749 608677 : gcc_assert (success);
4750 :
4751 608677 : rtx_insn *insn = DEP_PRO (dep);
4752 :
4753 : /* Recompute priority since dependent priorities may have changed. */
4754 608677 : priority (insn, true);
4755 608677 : update_insn_after_change (desc->insn);
4756 :
4757 608677 : if ((TODO_SPEC (desc->insn) & (HARD_DEP | DEP_POSTPONED)) == 0)
4758 317814 : fix_tick_ready (desc->insn);
4759 :
4760 608677 : if (backtrack_queue != NULL)
4761 : {
4762 0 : backtrack_queue->replacement_deps.safe_push (dep);
4763 0 : backtrack_queue->replace_apply.safe_push (1);
4764 : }
4765 : }
4766 : }
4767 :
4768 : /* We have determined that a pattern involved in DEP must be restored.
4769 : If IMMEDIATELY is false, we may have to postpone the replacement
4770 : until the start of the next cycle, at which point we will be called
4771 : again with IMMEDIATELY true. */
4772 : static void
4773 61107 : restore_pattern (dep_t dep, bool immediately)
4774 : {
4775 61107 : rtx_insn *next = DEP_CON (dep);
4776 61107 : int tick = INSN_TICK (next);
4777 :
4778 : /* If we already scheduled the insn, the modified version is
4779 : correct. */
4780 61107 : if (QUEUE_INDEX (next) == QUEUE_SCHEDULED)
4781 : return;
4782 :
4783 61107 : if (!immediately && targetm.sched.exposed_pipeline && reload_completed)
4784 : {
4785 0 : next_cycle_replace_deps.safe_push (dep);
4786 0 : next_cycle_apply.safe_push (0);
4787 0 : return;
4788 : }
4789 :
4790 :
4791 61107 : if (DEP_TYPE (dep) == REG_DEP_CONTROL)
4792 : {
4793 0 : if (sched_verbose >= 5)
4794 0 : fprintf (sched_dump, "restoring pattern for insn %d\n",
4795 : INSN_UID (next));
4796 0 : haifa_change_pattern (next, ORIG_PAT (next));
4797 : }
4798 : else
4799 : {
4800 61107 : struct dep_replacement *desc = DEP_REPLACE (dep);
4801 61107 : bool success;
4802 :
4803 61107 : if (sched_verbose >= 5)
4804 0 : fprintf (sched_dump, "restoring pattern for insn %d\n",
4805 0 : INSN_UID (desc->insn));
4806 61107 : tick = INSN_TICK (desc->insn);
4807 :
4808 61107 : success = validate_change (desc->insn, desc->loc, desc->orig, 0);
4809 61107 : gcc_assert (success);
4810 :
4811 61107 : rtx_insn *insn = DEP_PRO (dep);
4812 :
4813 61107 : if (QUEUE_INDEX (insn) != QUEUE_SCHEDULED)
4814 : {
4815 : /* Recompute priority since dependent priorities may have changed. */
4816 0 : priority (insn, true);
4817 : }
4818 :
4819 61107 : update_insn_after_change (desc->insn);
4820 :
4821 61107 : if (backtrack_queue != NULL)
4822 : {
4823 0 : backtrack_queue->replacement_deps.safe_push (dep);
4824 0 : backtrack_queue->replace_apply.safe_push (0);
4825 : }
4826 : }
4827 61107 : INSN_TICK (next) = tick;
4828 61107 : if (TODO_SPEC (next) == DEP_POSTPONED)
4829 : return;
4830 :
4831 61107 : if (sd_lists_empty_p (next, SD_LIST_BACK))
4832 61107 : TODO_SPEC (next) = 0;
4833 0 : else if (!sd_lists_empty_p (next, SD_LIST_HARD_BACK))
4834 0 : TODO_SPEC (next) = HARD_DEP;
4835 : }
4836 :
4837 : /* Perform pattern replacements that were queued up until the next
4838 : cycle. */
4839 : static void
4840 46185778 : perform_replacements_new_cycle (void)
4841 : {
4842 46185778 : int i;
4843 46185778 : dep_t dep;
4844 46185778 : FOR_EACH_VEC_ELT (next_cycle_replace_deps, i, dep)
4845 : {
4846 0 : int apply_p = next_cycle_apply[i];
4847 0 : if (apply_p)
4848 0 : apply_replacement (dep, true);
4849 : else
4850 0 : restore_pattern (dep, true);
4851 : }
4852 46185778 : next_cycle_replace_deps.truncate (0);
4853 46185778 : next_cycle_apply.truncate (0);
4854 46185778 : }
4855 :
4856 : /* Compute INSN_TICK_ESTIMATE for INSN. PROCESSED is a bitmap of
4857 : instructions we've previously encountered, a set bit prevents
4858 : recursion. BUDGET is a limit on how far ahead we look, it is
4859 : reduced on recursive calls. Return true if we produced a good
4860 : estimate, or false if we exceeded the budget. */
4861 : static bool
4862 0 : estimate_insn_tick (bitmap processed, rtx_insn *insn, int budget)
4863 : {
4864 0 : sd_iterator_def sd_it;
4865 0 : dep_t dep;
4866 0 : int earliest = INSN_TICK (insn);
4867 :
4868 0 : FOR_EACH_DEP (insn, SD_LIST_BACK, sd_it, dep)
4869 : {
4870 0 : rtx_insn *pro = DEP_PRO (dep);
4871 0 : int t;
4872 :
4873 0 : if (DEP_STATUS (dep) & DEP_CANCELLED)
4874 0 : continue;
4875 :
4876 0 : if (QUEUE_INDEX (pro) == QUEUE_SCHEDULED)
4877 0 : gcc_assert (INSN_TICK (pro) + dep_cost (dep) <= INSN_TICK (insn));
4878 : else
4879 : {
4880 0 : int cost = dep_cost (dep);
4881 0 : if (cost >= budget)
4882 : return false;
4883 0 : if (!bitmap_bit_p (processed, INSN_LUID (pro)))
4884 : {
4885 0 : if (!estimate_insn_tick (processed, pro, budget - cost))
4886 : return false;
4887 : }
4888 0 : gcc_assert (INSN_TICK_ESTIMATE (pro) != INVALID_TICK);
4889 0 : t = INSN_TICK_ESTIMATE (pro) + cost;
4890 0 : if (earliest == INVALID_TICK || t > earliest)
4891 0 : earliest = t;
4892 : }
4893 : }
4894 0 : bitmap_set_bit (processed, INSN_LUID (insn));
4895 0 : INSN_TICK_ESTIMATE (insn) = earliest;
4896 0 : return true;
4897 : }
4898 :
4899 : /* Examine the pair of insns in P, and estimate (optimistically, assuming
4900 : infinite resources) the cycle in which the delayed shadow can be issued.
4901 : Return the number of cycles that must pass before the real insn can be
4902 : issued in order to meet this constraint. */
4903 : static int
4904 0 : estimate_shadow_tick (struct delay_pair *p)
4905 : {
4906 0 : auto_bitmap processed;
4907 0 : int t;
4908 0 : bool cutoff;
4909 :
4910 0 : cutoff = !estimate_insn_tick (processed, p->i2,
4911 0 : max_insn_queue_index + pair_delay (p));
4912 0 : if (cutoff)
4913 : return max_insn_queue_index;
4914 0 : t = INSN_TICK_ESTIMATE (p->i2) - (clock_var + pair_delay (p) + 1);
4915 0 : if (t > 0)
4916 : return t;
4917 : return 0;
4918 0 : }
4919 :
4920 : /* If INSN has no unresolved backwards dependencies, add it to the schedule and
4921 : recursively resolve all its forward dependencies. */
4922 : static void
4923 0 : resolve_dependencies (rtx_insn *insn)
4924 : {
4925 0 : sd_iterator_def sd_it;
4926 0 : dep_t dep;
4927 :
4928 : /* Don't use sd_lists_empty_p; it ignores debug insns. */
4929 0 : if (DEPS_LIST_FIRST (INSN_HARD_BACK_DEPS (insn)) != NULL
4930 0 : || DEPS_LIST_FIRST (INSN_SPEC_BACK_DEPS (insn)) != NULL)
4931 0 : return;
4932 :
4933 0 : if (sched_verbose >= 4)
4934 0 : fprintf (sched_dump, ";;\tquickly resolving %d\n", INSN_UID (insn));
4935 :
4936 0 : if (QUEUE_INDEX (insn) >= 0)
4937 0 : queue_remove (insn);
4938 :
4939 0 : scheduled_insns.safe_push (insn);
4940 :
4941 : /* Update dependent instructions. */
4942 0 : for (sd_it = sd_iterator_start (insn, SD_LIST_FORW);
4943 0 : sd_iterator_cond (&sd_it, &dep);)
4944 : {
4945 0 : rtx_insn *next = DEP_CON (dep);
4946 :
4947 0 : if (sched_verbose >= 4)
4948 0 : fprintf (sched_dump, ";;\t\tdep %d against %d\n", INSN_UID (insn),
4949 0 : INSN_UID (next));
4950 :
4951 : /* Resolve the dependence between INSN and NEXT.
4952 : sd_resolve_dep () moves current dep to another list thus
4953 : advancing the iterator. */
4954 0 : sd_resolve_dep (sd_it);
4955 :
4956 0 : if (!IS_SPECULATION_BRANCHY_CHECK_P (insn))
4957 : {
4958 0 : resolve_dependencies (next);
4959 : }
4960 : else
4961 : /* Check always has only one forward dependence (to the first insn in
4962 : the recovery block), therefore, this will be executed only once. */
4963 : {
4964 0 : gcc_assert (sd_lists_empty_p (insn, SD_LIST_FORW));
4965 : }
4966 : }
4967 : }
4968 :
4969 :
4970 : /* Return the head and tail pointers of ebb starting at BEG and ending
4971 : at END. */
4972 : void
4973 41989719 : get_ebb_head_tail (basic_block beg, basic_block end,
4974 : rtx_insn **headp, rtx_insn **tailp)
4975 : {
4976 41989719 : rtx_insn *beg_head = BB_HEAD (beg);
4977 41989719 : rtx_insn * beg_tail = BB_END (beg);
4978 41989719 : rtx_insn * end_head = BB_HEAD (end);
4979 41989719 : rtx_insn * end_tail = BB_END (end);
4980 :
4981 : /* Don't include any notes or labels at the beginning of the BEG
4982 : basic block, or notes at the end of the END basic blocks. */
4983 :
4984 41989719 : if (LABEL_P (beg_head))
4985 20505967 : beg_head = NEXT_INSN (beg_head);
4986 :
4987 98494777 : while (beg_head != beg_tail)
4988 94555761 : if (NOTE_P (beg_head))
4989 56505058 : beg_head = NEXT_INSN (beg_head);
4990 38050703 : else if (DEBUG_INSN_P (beg_head))
4991 : {
4992 10547195 : rtx_insn * note, *next;
4993 :
4994 10547195 : for (note = NEXT_INSN (beg_head);
4995 88761354 : note != beg_tail;
4996 78214159 : note = next)
4997 : {
4998 88440233 : next = NEXT_INSN (note);
4999 88440233 : if (NOTE_P (note))
5000 : {
5001 433526 : if (sched_verbose >= 9)
5002 0 : fprintf (sched_dump, "reorder %i\n", INSN_UID (note));
5003 :
5004 433526 : reorder_insns_nobb (note, note, PREV_INSN (beg_head));
5005 :
5006 433526 : if (BLOCK_FOR_INSN (note) != beg)
5007 0 : df_insn_change_bb (note, beg);
5008 : }
5009 88006707 : else if (!DEBUG_INSN_P (note))
5010 : break;
5011 : }
5012 :
5013 : break;
5014 : }
5015 : else
5016 : break;
5017 :
5018 41989719 : *headp = beg_head;
5019 :
5020 41989719 : if (beg == end)
5021 41989719 : end_head = beg_head;
5022 63 : else if (LABEL_P (end_head))
5023 2 : end_head = NEXT_INSN (end_head);
5024 :
5025 42149831 : while (end_head != end_tail)
5026 38197811 : if (NOTE_P (end_tail))
5027 160112 : end_tail = PREV_INSN (end_tail);
5028 38037699 : else if (DEBUG_INSN_P (end_tail))
5029 : {
5030 881819 : rtx_insn * note, *prev;
5031 :
5032 881819 : for (note = PREV_INSN (end_tail);
5033 7520527 : note != end_head;
5034 6638708 : note = prev)
5035 : {
5036 7485450 : prev = PREV_INSN (note);
5037 7485450 : if (NOTE_P (note))
5038 : {
5039 1604 : if (sched_verbose >= 9)
5040 0 : fprintf (sched_dump, "reorder %i\n", INSN_UID (note));
5041 :
5042 1604 : reorder_insns_nobb (note, note, end_tail);
5043 :
5044 1604 : if (end_tail == BB_END (end))
5045 925 : BB_END (end) = note;
5046 :
5047 1604 : if (BLOCK_FOR_INSN (note) != end)
5048 0 : df_insn_change_bb (note, end);
5049 : }
5050 7483846 : else if (!DEBUG_INSN_P (note))
5051 : break;
5052 : }
5053 :
5054 : break;
5055 : }
5056 : else
5057 : break;
5058 :
5059 41989719 : *tailp = end_tail;
5060 41989719 : }
5061 :
5062 : /* Return true if there are no real insns in the range [ HEAD, TAIL ]. */
5063 :
5064 : bool
5065 31491286 : no_real_insns_p (const rtx_insn *head, const rtx_insn *tail)
5066 : {
5067 31547621 : while (head != NEXT_INSN (tail))
5068 : {
5069 31491286 : if (!NOTE_P (head) && !LABEL_P (head))
5070 : return false;
5071 56335 : head = NEXT_INSN (head);
5072 : }
5073 : return true;
5074 : }
5075 :
5076 : /* Restore-other-notes: NOTE_LIST is the end of a chain of notes
5077 : previously found among the insns. Insert them just before HEAD. */
5078 : rtx_insn *
5079 10478683 : restore_other_notes (rtx_insn *head, basic_block head_bb)
5080 : {
5081 10478683 : if (note_list != 0)
5082 : {
5083 2117180 : rtx_insn *note_head = note_list;
5084 :
5085 2117180 : if (head)
5086 2116809 : head_bb = BLOCK_FOR_INSN (head);
5087 : else
5088 371 : head = NEXT_INSN (bb_note (head_bb));
5089 :
5090 4992326 : while (PREV_INSN (note_head))
5091 : {
5092 2875146 : set_block_for_insn (note_head, head_bb);
5093 2875146 : note_head = PREV_INSN (note_head);
5094 : }
5095 : /* In the above cycle we've missed this note. */
5096 2117180 : set_block_for_insn (note_head, head_bb);
5097 :
5098 2117180 : SET_PREV_INSN (note_head) = PREV_INSN (head);
5099 2117180 : SET_NEXT_INSN (PREV_INSN (head)) = note_head;
5100 2117180 : SET_PREV_INSN (head) = note_list;
5101 2117180 : SET_NEXT_INSN (note_list) = head;
5102 :
5103 2117180 : if (BLOCK_FOR_INSN (head) != head_bb)
5104 0 : BB_END (head_bb) = note_list;
5105 :
5106 : head = note_head;
5107 : }
5108 :
5109 10478683 : return head;
5110 : }
5111 :
5112 : /* When we know we are going to discard the schedule due to a failed attempt
5113 : at modulo scheduling, undo all replacements. */
5114 : static void
5115 0 : undo_all_replacements (void)
5116 : {
5117 0 : rtx_insn *insn;
5118 0 : int i;
5119 :
5120 0 : FOR_EACH_VEC_ELT (scheduled_insns, i, insn)
5121 : {
5122 0 : sd_iterator_def sd_it;
5123 0 : dep_t dep;
5124 :
5125 : /* See if we must undo a replacement. */
5126 0 : for (sd_it = sd_iterator_start (insn, SD_LIST_RES_FORW);
5127 0 : sd_iterator_cond (&sd_it, &dep); sd_iterator_next (&sd_it))
5128 : {
5129 0 : struct dep_replacement *desc = DEP_REPLACE (dep);
5130 0 : if (desc != NULL)
5131 0 : validate_change (desc->insn, desc->loc, desc->orig, 0);
5132 : }
5133 : }
5134 0 : }
5135 :
5136 : /* Return first non-scheduled insn in the current scheduling block.
5137 : This is mostly used for debug-counter purposes. */
5138 : static rtx_insn *
5139 0 : first_nonscheduled_insn (void)
5140 : {
5141 0 : rtx_insn *insn = (nonscheduled_insns_begin != NULL_RTX
5142 0 : ? nonscheduled_insns_begin
5143 0 : : current_sched_info->prev_head);
5144 :
5145 0 : do
5146 : {
5147 0 : insn = next_nonnote_nondebug_insn (insn);
5148 : }
5149 0 : while (QUEUE_INDEX (insn) == QUEUE_SCHEDULED);
5150 :
5151 0 : return insn;
5152 : }
5153 :
5154 : /* Move insns that became ready to fire from queue to ready list. */
5155 :
5156 : static void
5157 35708100 : queue_to_ready (struct ready_list *ready)
5158 : {
5159 35708100 : rtx_insn *insn;
5160 35708100 : rtx_insn_list *link;
5161 35708100 : rtx_insn *skip_insn;
5162 :
5163 35708100 : q_ptr = NEXT_Q (q_ptr);
5164 :
5165 35708100 : if (dbg_cnt (sched_insn) == false)
5166 : /* If debug counter is activated do not requeue the first
5167 : nonscheduled insn. */
5168 0 : skip_insn = first_nonscheduled_insn ();
5169 : else
5170 : skip_insn = NULL;
5171 :
5172 : /* Add all pending insns that can be scheduled without stalls to the
5173 : ready list. */
5174 68736734 : for (link = insn_queue[q_ptr]; link; link = link->next ())
5175 : {
5176 33028634 : insn = link->insn ();
5177 33028634 : q_size -= 1;
5178 :
5179 33028634 : if (sched_verbose >= 2)
5180 0 : fprintf (sched_dump, ";;\t\tQ-->Ready: insn %s: ",
5181 0 : (*current_sched_info->print_insn) (insn, 0));
5182 :
5183 : /* If the ready list is full, delay the insn for 1 cycle.
5184 : See the comment in schedule_block for the rationale. */
5185 33028634 : if (!reload_completed
5186 5700 : && (ready->n_ready - ready->n_debug > param_max_sched_ready_insns
5187 5553 : || (sched_pressure == SCHED_PRESSURE_MODEL
5188 : /* Limit pressure recalculations to
5189 : param_max_sched_ready_insns instructions too. */
5190 0 : && model_index (insn) > (model_curr_point
5191 0 : + param_max_sched_ready_insns)))
5192 147 : && !(sched_pressure == SCHED_PRESSURE_MODEL
5193 0 : && model_curr_point < model_num_insns
5194 : /* Always allow the next model instruction to issue. */
5195 0 : && model_index (insn) == model_curr_point)
5196 147 : && !SCHED_GROUP_P (insn)
5197 33028781 : && insn != skip_insn)
5198 : {
5199 147 : if (sched_verbose >= 2)
5200 0 : fprintf (sched_dump, "keeping in queue, ready full\n");
5201 147 : queue_insn (insn, 1, "ready full");
5202 : }
5203 : else
5204 : {
5205 33028487 : ready_add (ready, insn, false);
5206 33028487 : if (sched_verbose >= 2)
5207 0 : fprintf (sched_dump, "moving to ready without stalls\n");
5208 : }
5209 : }
5210 35708100 : free_INSN_LIST_list (&insn_queue[q_ptr]);
5211 :
5212 : /* If there are no ready insns, stall until one is ready and add all
5213 : of the pending insns at that point to the ready list. */
5214 35708100 : if (ready->n_ready == 0)
5215 : {
5216 : int stalls;
5217 :
5218 11331916 : for (stalls = 1; stalls <= max_insn_queue_index; stalls++)
5219 : {
5220 11331916 : if ((link = insn_queue[NEXT_Q_AFTER (q_ptr, stalls)]))
5221 : {
5222 12748590 : for (; link; link = link->next ())
5223 : {
5224 6602705 : insn = link->insn ();
5225 6602705 : q_size -= 1;
5226 :
5227 6602705 : if (sched_verbose >= 2)
5228 0 : fprintf (sched_dump, ";;\t\tQ-->Ready: insn %s: ",
5229 0 : (*current_sched_info->print_insn) (insn, 0));
5230 :
5231 6602705 : ready_add (ready, insn, false);
5232 6602705 : if (sched_verbose >= 2)
5233 0 : fprintf (sched_dump, "moving to ready with %d stalls\n", stalls);
5234 : }
5235 6145885 : free_INSN_LIST_list (&insn_queue[NEXT_Q_AFTER (q_ptr, stalls)]);
5236 :
5237 6145885 : advance_one_cycle ();
5238 :
5239 6145885 : break;
5240 : }
5241 :
5242 5186031 : advance_one_cycle ();
5243 : }
5244 :
5245 6145885 : q_ptr = NEXT_Q_AFTER (q_ptr, stalls);
5246 6145885 : clock_var += stalls;
5247 6145885 : if (sched_verbose >= 2)
5248 0 : fprintf (sched_dump, ";;\tAdvancing clock by %d cycle[s] to %d\n",
5249 : stalls, clock_var);
5250 : }
5251 35708100 : }
5252 :
5253 : /* Used by early_queue_to_ready. Determines whether it is "ok" to
5254 : prematurely move INSN from the queue to the ready list. Currently,
5255 : if a target defines the hook 'is_costly_dependence', this function
5256 : uses the hook to check whether there exist any dependences which are
5257 : considered costly by the target, between INSN and other insns that
5258 : have already been scheduled. Dependences are checked up to Y cycles
5259 : back, with default Y=1; The flag -fsched-stalled-insns-dep=Y allows
5260 : controlling this value.
5261 : (Other considerations could be taken into account instead (or in
5262 : addition) depending on user flags and target hooks. */
5263 :
5264 : static bool
5265 5 : ok_for_early_queue_removal (rtx_insn *insn)
5266 : {
5267 5 : if (targetm.sched.is_costly_dependence)
5268 : {
5269 0 : int n_cycles;
5270 0 : int i = scheduled_insns.length ();
5271 0 : for (n_cycles = flag_sched_stalled_insns_dep; n_cycles; n_cycles--)
5272 : {
5273 0 : while (i-- > 0)
5274 : {
5275 0 : int cost;
5276 :
5277 0 : rtx_insn *prev_insn = scheduled_insns[i];
5278 :
5279 0 : if (!NOTE_P (prev_insn))
5280 : {
5281 0 : dep_t dep;
5282 :
5283 0 : dep = sd_find_dep_between (prev_insn, insn, true);
5284 :
5285 0 : if (dep != NULL)
5286 : {
5287 0 : cost = dep_cost (dep);
5288 :
5289 0 : if (targetm.sched.is_costly_dependence (dep, cost,
5290 0 : flag_sched_stalled_insns_dep - n_cycles))
5291 : return false;
5292 : }
5293 : }
5294 :
5295 0 : if (GET_MODE (prev_insn) == TImode) /* end of dispatch group */
5296 : break;
5297 : }
5298 :
5299 0 : if (i == 0)
5300 : break;
5301 : }
5302 : }
5303 :
5304 : return true;
5305 : }
5306 :
5307 :
5308 : /* Remove insns from the queue, before they become "ready" with respect
5309 : to FU latency considerations. */
5310 :
5311 : static int
5312 33759567 : early_queue_to_ready (state_t state, struct ready_list *ready)
5313 : {
5314 33759567 : rtx_insn *insn;
5315 33759567 : rtx_insn_list *link;
5316 33759567 : rtx_insn_list *next_link;
5317 33759567 : rtx_insn_list *prev_link;
5318 33759567 : bool move_to_ready;
5319 33759567 : int cost;
5320 33759567 : state_t temp_state = alloca (dfa_state_size);
5321 33759567 : int stalls;
5322 33759567 : int insns_removed = 0;
5323 :
5324 : /*
5325 : Flag '-fsched-stalled-insns=X' determines the aggressiveness of this
5326 : function:
5327 :
5328 : X == 0: There is no limit on how many queued insns can be removed
5329 : prematurely. (flag_sched_stalled_insns = -1).
5330 :
5331 : X >= 1: Only X queued insns can be removed prematurely in each
5332 : invocation. (flag_sched_stalled_insns = X).
5333 :
5334 : Otherwise: Early queue removal is disabled.
5335 : (flag_sched_stalled_insns = 0)
5336 : */
5337 :
5338 33759567 : if (! flag_sched_stalled_insns)
5339 : return 0;
5340 :
5341 1799 : for (stalls = 0; stalls <= max_insn_queue_index; stalls++)
5342 : {
5343 1792 : if ((link = insn_queue[NEXT_Q_AFTER (q_ptr, stalls)]))
5344 : {
5345 4 : if (sched_verbose > 6)
5346 0 : fprintf (sched_dump, ";; look at index %d + %d\n", q_ptr, stalls);
5347 :
5348 4 : prev_link = 0;
5349 9 : while (link)
5350 : {
5351 5 : next_link = link->next ();
5352 5 : insn = link->insn ();
5353 5 : if (insn && sched_verbose > 6)
5354 0 : print_rtl_single (sched_dump, insn);
5355 :
5356 5 : memcpy (temp_state, state, dfa_state_size);
5357 5 : if (recog_memoized (insn) < 0)
5358 : /* non-negative to indicate that it's not ready
5359 : to avoid infinite Q->R->Q->R... */
5360 : cost = 0;
5361 : else
5362 5 : cost = state_transition (temp_state, insn);
5363 :
5364 5 : if (sched_verbose >= 6)
5365 0 : fprintf (sched_dump, "transition cost = %d\n", cost);
5366 :
5367 5 : move_to_ready = false;
5368 5 : if (cost < 0)
5369 : {
5370 5 : move_to_ready = ok_for_early_queue_removal (insn);
5371 5 : if (move_to_ready == true)
5372 : {
5373 : /* move from Q to R */
5374 5 : q_size -= 1;
5375 5 : ready_add (ready, insn, false);
5376 :
5377 5 : if (prev_link)
5378 0 : XEXP (prev_link, 1) = next_link;
5379 : else
5380 5 : insn_queue[NEXT_Q_AFTER (q_ptr, stalls)] = next_link;
5381 :
5382 5 : free_INSN_LIST_node (link);
5383 :
5384 5 : if (sched_verbose >= 2)
5385 0 : fprintf (sched_dump, ";;\t\tEarly Q-->Ready: insn %s\n",
5386 0 : (*current_sched_info->print_insn) (insn, 0));
5387 :
5388 5 : insns_removed++;
5389 5 : if (insns_removed == flag_sched_stalled_insns)
5390 : /* Remove no more than flag_sched_stalled_insns insns
5391 : from Q at a time. */
5392 : return insns_removed;
5393 : }
5394 : }
5395 :
5396 : if (move_to_ready == false)
5397 : prev_link = link;
5398 :
5399 5 : link = next_link;
5400 : } /* while link */
5401 : } /* if link */
5402 :
5403 : } /* for stalls.. */
5404 :
5405 : return insns_removed;
5406 : }
5407 :
5408 :
5409 : /* Print the ready list for debugging purposes.
5410 : If READY_TRY is non-zero then only print insns that max_issue
5411 : will consider. */
5412 : static void
5413 190 : debug_ready_list_1 (struct ready_list *ready, signed char *ready_try)
5414 : {
5415 190 : rtx_insn **p;
5416 190 : int i;
5417 :
5418 190 : if (ready->n_ready == 0)
5419 : {
5420 190 : fprintf (sched_dump, "\n");
5421 190 : return;
5422 : }
5423 :
5424 0 : p = ready_lastpos (ready);
5425 0 : for (i = 0; i < ready->n_ready; i++)
5426 : {
5427 0 : if (ready_try != NULL && ready_try[ready->n_ready - i - 1])
5428 0 : continue;
5429 :
5430 0 : fprintf (sched_dump, " %s:%d",
5431 0 : (*current_sched_info->print_insn) (p[i], 0),
5432 0 : INSN_LUID (p[i]));
5433 0 : if (sched_pressure != SCHED_PRESSURE_NONE)
5434 0 : fprintf (sched_dump, "(cost=%d",
5435 0 : INSN_REG_PRESSURE_EXCESS_COST_CHANGE (p[i]));
5436 0 : fprintf (sched_dump, ":prio=%d", INSN_PRIORITY (p[i]));
5437 0 : if (INSN_TICK (p[i]) > clock_var)
5438 0 : fprintf (sched_dump, ":delay=%d", INSN_TICK (p[i]) - clock_var);
5439 0 : if (sched_pressure == SCHED_PRESSURE_MODEL)
5440 0 : fprintf (sched_dump, ":idx=%d",
5441 : model_index (p[i]));
5442 0 : if (sched_pressure != SCHED_PRESSURE_NONE)
5443 0 : fprintf (sched_dump, ")");
5444 : }
5445 0 : fprintf (sched_dump, "\n");
5446 : }
5447 :
5448 : /* Print the ready list. Callable from debugger. */
5449 : static void
5450 190 : debug_ready_list (struct ready_list *ready)
5451 : {
5452 0 : debug_ready_list_1 (ready, NULL);
5453 0 : }
5454 :
5455 : /* Search INSN for REG_SAVE_NOTE notes and convert them back into insn
5456 : NOTEs. This is used for NOTE_INSN_EPILOGUE_BEG, so that sched-ebb
5457 : replaces the epilogue note in the correct basic block. */
5458 : void
5459 61465333 : reemit_notes (rtx_insn *insn)
5460 : {
5461 61465333 : rtx note;
5462 61465333 : rtx_insn *last = insn;
5463 :
5464 121938389 : for (note = REG_NOTES (insn); note; note = XEXP (note, 1))
5465 : {
5466 60473056 : if (REG_NOTE_KIND (note) == REG_SAVE_NOTE)
5467 : {
5468 927461 : enum insn_note note_type = (enum insn_note) INTVAL (XEXP (note, 0));
5469 :
5470 927461 : last = emit_note_before (note_type, last);
5471 927461 : remove_note (insn, note);
5472 927461 : df_insn_create_insn_record (last);
5473 : }
5474 : }
5475 61465333 : }
5476 :
5477 : /* Move INSN. Reemit notes if needed. Update CFG, if needed. */
5478 : static void
5479 114313473 : move_insn (rtx_insn *insn, rtx_insn *last, rtx nt)
5480 : {
5481 114313473 : if (PREV_INSN (insn) != last)
5482 : {
5483 13820761 : basic_block bb;
5484 13820761 : rtx_insn *note;
5485 13820761 : int jump_p = 0;
5486 :
5487 13820761 : bb = BLOCK_FOR_INSN (insn);
5488 :
5489 : /* BB_HEAD is either LABEL or NOTE. */
5490 13820761 : gcc_assert (BB_HEAD (bb) != insn);
5491 :
5492 13820761 : if (BB_END (bb) == insn)
5493 : /* If this is last instruction in BB, move end marker one
5494 : instruction up. */
5495 : {
5496 : /* Jumps are always placed at the end of basic block. */
5497 232763 : jump_p = control_flow_insn_p (insn);
5498 :
5499 232763 : gcc_assert (!jump_p
5500 : || ((common_sched_info->sched_pass_id == SCHED_RGN_PASS)
5501 : && IS_SPECULATION_BRANCHY_CHECK_P (insn))
5502 : || (common_sched_info->sched_pass_id
5503 : == SCHED_EBB_PASS));
5504 :
5505 232763 : gcc_assert (BLOCK_FOR_INSN (PREV_INSN (insn)) == bb);
5506 :
5507 232763 : BB_END (bb) = PREV_INSN (insn);
5508 : }
5509 :
5510 13820761 : gcc_assert (BB_END (bb) != last);
5511 :
5512 13820761 : if (jump_p)
5513 : /* We move the block note along with jump. */
5514 : {
5515 1 : gcc_assert (nt);
5516 :
5517 1 : note = NEXT_INSN (insn);
5518 2 : while (NOTE_NOT_BB_P (note) && note != nt)
5519 0 : note = NEXT_INSN (note);
5520 :
5521 1 : if (note != nt
5522 1 : && (LABEL_P (note)
5523 1 : || BARRIER_P (note)))
5524 0 : note = NEXT_INSN (note);
5525 :
5526 1 : gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note));
5527 : }
5528 : else
5529 : note = insn;
5530 :
5531 13820761 : SET_NEXT_INSN (PREV_INSN (insn)) = NEXT_INSN (note);
5532 13820761 : SET_PREV_INSN (NEXT_INSN (note)) = PREV_INSN (insn);
5533 :
5534 13820761 : SET_NEXT_INSN (note) = NEXT_INSN (last);
5535 13820761 : SET_PREV_INSN (NEXT_INSN (last)) = note;
5536 :
5537 13820761 : SET_NEXT_INSN (last) = insn;
5538 13820761 : SET_PREV_INSN (insn) = last;
5539 :
5540 13820761 : bb = BLOCK_FOR_INSN (last);
5541 :
5542 13820761 : if (jump_p)
5543 : {
5544 1 : fix_jump_move (insn);
5545 :
5546 1 : if (BLOCK_FOR_INSN (insn) != bb)
5547 0 : move_block_after_check (insn);
5548 :
5549 1 : gcc_assert (BB_END (bb) == last);
5550 : }
5551 :
5552 13820761 : df_insn_change_bb (insn, bb);
5553 :
5554 : /* Update BB_END, if needed. */
5555 13820761 : if (BB_END (bb) == last)
5556 1 : BB_END (bb) = insn;
5557 : }
5558 :
5559 114313473 : SCHED_GROUP_P (insn) = 0;
5560 114313473 : }
5561 :
5562 : /* Return true if scheduling INSN will finish current clock cycle. */
5563 : static bool
5564 114986261 : insn_finishes_cycle_p (rtx_insn *insn)
5565 : {
5566 114986261 : if (SCHED_GROUP_P (insn))
5567 : /* After issuing INSN, rest of the sched_group will be forced to issue
5568 : in order. Don't make any plans for the rest of cycle. */
5569 : return true;
5570 :
5571 : /* Finishing the block will, apparently, finish the cycle. */
5572 114986033 : if (current_sched_info->insn_finishes_block_p
5573 114986033 : && current_sched_info->insn_finishes_block_p (insn))
5574 : return true;
5575 :
5576 : return false;
5577 : }
5578 :
5579 : /* Helper for autopref_multipass_init. Given a SET in PAT and whether
5580 : we're expecting a memory WRITE or not, check that the insn is relevant to
5581 : the autoprefetcher modelling code. Return true iff that is the case.
5582 : If it is relevant, record the base register of the memory op in BASE and
5583 : the offset in OFFSET. */
5584 :
5585 : static bool
5586 3 : analyze_set_insn_for_autopref (rtx pat, bool write, rtx *base, int *offset)
5587 : {
5588 3 : if (GET_CODE (pat) != SET)
5589 : return false;
5590 :
5591 3 : rtx mem = write ? SET_DEST (pat) : SET_SRC (pat);
5592 3 : if (!MEM_P (mem))
5593 : return false;
5594 :
5595 3 : struct address_info info;
5596 3 : decompose_mem_address (&info, mem);
5597 :
5598 : /* TODO: Currently only (base+const) addressing is supported. */
5599 3 : if (info.base == NULL || !REG_P (*info.base)
5600 3 : || (info.disp != NULL && !CONST_INT_P (*info.disp)))
5601 : return false;
5602 :
5603 3 : *base = *info.base;
5604 3 : *offset = info.disp ? INTVAL (*info.disp) : 0;
5605 3 : return true;
5606 : }
5607 :
5608 : /* Functions to model cache auto-prefetcher.
5609 :
5610 : Some of the CPUs have cache auto-prefetcher, which /seems/ to initiate
5611 : memory prefetches if it sees instructions with consecutive memory accesses
5612 : in the instruction stream. Details of such hardware units are not published,
5613 : so we can only guess what exactly is going on there.
5614 : In the scheduler, we model abstract auto-prefetcher. If there are memory
5615 : insns in the ready list (or the queue) that have same memory base, but
5616 : different offsets, then we delay the insns with larger offsets until insns
5617 : with smaller offsets get scheduled. If PARAM_SCHED_AUTOPREF_QUEUE_DEPTH
5618 : is "1", then we look at the ready list; if it is N>1, then we also look
5619 : through N-1 queue entries.
5620 : If the param is N>=0, then rank_for_schedule will consider auto-prefetching
5621 : among its heuristics.
5622 : Param value of "-1" disables modelling of the auto-prefetcher. */
5623 :
5624 : /* Initialize autoprefetcher model data for INSN. */
5625 : static void
5626 3 : autopref_multipass_init (const rtx_insn *insn, int write)
5627 : {
5628 3 : autopref_multipass_data_t data = &INSN_AUTOPREF_MULTIPASS_DATA (insn)[write];
5629 :
5630 3 : gcc_assert (data->status == AUTOPREF_MULTIPASS_DATA_UNINITIALIZED);
5631 3 : data->base = NULL_RTX;
5632 3 : data->offset = 0;
5633 : /* Set insn entry initialized, but not relevant for auto-prefetcher. */
5634 3 : data->status = AUTOPREF_MULTIPASS_DATA_IRRELEVANT;
5635 :
5636 3 : rtx pat = PATTERN (insn);
5637 :
5638 : /* We have a multi-set insn like a load-multiple or store-multiple.
5639 : We care about these as long as all the memory ops inside the PARALLEL
5640 : have the same base register. We care about the minimum and maximum
5641 : offsets from that base but don't check for the order of those offsets
5642 : within the PARALLEL insn itself. */
5643 3 : if (GET_CODE (pat) == PARALLEL)
5644 : {
5645 0 : int n_elems = XVECLEN (pat, 0);
5646 :
5647 0 : int i, offset;
5648 0 : rtx base, prev_base = NULL_RTX;
5649 0 : int min_offset = INT_MAX;
5650 :
5651 0 : for (i = 0; i < n_elems; i++)
5652 : {
5653 0 : rtx set = XVECEXP (pat, 0, i);
5654 0 : if (GET_CODE (set) != SET)
5655 : return;
5656 :
5657 0 : if (!analyze_set_insn_for_autopref (set, write, &base, &offset))
5658 : return;
5659 :
5660 : /* Ensure that all memory operations in the PARALLEL use the same
5661 : base register. */
5662 0 : if (i > 0 && REGNO (base) != REGNO (prev_base))
5663 : return;
5664 0 : prev_base = base;
5665 0 : min_offset = MIN (min_offset, offset);
5666 : }
5667 :
5668 : /* If we reached here then we have a valid PARALLEL of multiple memory ops
5669 : with prev_base as the base and min_offset containing the offset. */
5670 0 : gcc_assert (prev_base);
5671 0 : data->base = prev_base;
5672 0 : data->offset = min_offset;
5673 0 : data->status = AUTOPREF_MULTIPASS_DATA_NORMAL;
5674 0 : return;
5675 : }
5676 :
5677 : /* Otherwise this is a single set memory operation. */
5678 3 : rtx set = single_set (insn);
5679 3 : if (set == NULL_RTX)
5680 : return;
5681 :
5682 3 : if (!analyze_set_insn_for_autopref (set, write, &data->base,
5683 : &data->offset))
5684 : return;
5685 :
5686 : /* This insn is relevant for the auto-prefetcher.
5687 : The base and offset fields will have been filled in the
5688 : analyze_set_insn_for_autopref call above. */
5689 3 : data->status = AUTOPREF_MULTIPASS_DATA_NORMAL;
5690 : }
5691 :
5692 : /* Helper function for rank_for_schedule sorting. */
5693 : static int
5694 12 : autopref_rank_for_schedule (const rtx_insn *insn1, const rtx_insn *insn2)
5695 : {
5696 12 : int r = 0;
5697 24 : for (int write = 0; write < 2 && !r; ++write)
5698 : {
5699 12 : autopref_multipass_data_t data1
5700 12 : = &INSN_AUTOPREF_MULTIPASS_DATA (insn1)[write];
5701 12 : autopref_multipass_data_t data2
5702 12 : = &INSN_AUTOPREF_MULTIPASS_DATA (insn2)[write];
5703 :
5704 12 : if (data1->status == AUTOPREF_MULTIPASS_DATA_UNINITIALIZED)
5705 1 : autopref_multipass_init (insn1, write);
5706 :
5707 12 : if (data2->status == AUTOPREF_MULTIPASS_DATA_UNINITIALIZED)
5708 2 : autopref_multipass_init (insn2, write);
5709 :
5710 12 : int irrel1 = data1->status == AUTOPREF_MULTIPASS_DATA_IRRELEVANT;
5711 12 : int irrel2 = data2->status == AUTOPREF_MULTIPASS_DATA_IRRELEVANT;
5712 :
5713 12 : if (!irrel1 && !irrel2)
5714 : /* Sort memory references from lowest offset to the largest. */
5715 12 : r = (data1->offset > data2->offset) - (data1->offset < data2->offset);
5716 0 : else if (write)
5717 : /* Schedule "irrelevant" insns before memory stores to resolve
5718 : as many producer dependencies of stores as possible. */
5719 0 : r = irrel2 - irrel1;
5720 : else
5721 : /* Schedule "irrelevant" insns after memory reads to avoid breaking
5722 : memory read sequences. */
5723 0 : r = irrel1 - irrel2;
5724 : }
5725 :
5726 12 : return r;
5727 : }
5728 :
5729 : /* True if header of debug dump was printed. */
5730 : static bool autopref_multipass_dfa_lookahead_guard_started_dump_p;
5731 :
5732 : /* Helper for autopref_multipass_dfa_lookahead_guard.
5733 : Return "1" if INSN1 should be delayed in favor of INSN2. */
5734 : static int
5735 0 : autopref_multipass_dfa_lookahead_guard_1 (const rtx_insn *insn1,
5736 : const rtx_insn *insn2, int write)
5737 : {
5738 0 : autopref_multipass_data_t data1
5739 0 : = &INSN_AUTOPREF_MULTIPASS_DATA (insn1)[write];
5740 0 : autopref_multipass_data_t data2
5741 0 : = &INSN_AUTOPREF_MULTIPASS_DATA (insn2)[write];
5742 :
5743 0 : if (data2->status == AUTOPREF_MULTIPASS_DATA_UNINITIALIZED)
5744 0 : autopref_multipass_init (insn2, write);
5745 0 : if (data2->status == AUTOPREF_MULTIPASS_DATA_IRRELEVANT)
5746 : return 0;
5747 :
5748 0 : if (rtx_equal_p (data1->base, data2->base)
5749 0 : && data1->offset > data2->offset)
5750 : {
5751 0 : if (sched_verbose >= 2)
5752 : {
5753 0 : if (!autopref_multipass_dfa_lookahead_guard_started_dump_p)
5754 : {
5755 0 : fprintf (sched_dump,
5756 : ";;\t\tnot trying in max_issue due to autoprefetch "
5757 : "model: ");
5758 0 : autopref_multipass_dfa_lookahead_guard_started_dump_p = true;
5759 : }
5760 :
5761 0 : fprintf (sched_dump, " %d(%d)", INSN_UID (insn1), INSN_UID (insn2));
5762 : }
5763 :
5764 : return 1;
5765 : }
5766 :
5767 : return 0;
5768 : }
5769 :
5770 : /* General note:
5771 :
5772 : We could have also hooked autoprefetcher model into
5773 : first_cycle_multipass_backtrack / first_cycle_multipass_issue hooks
5774 : to enable intelligent selection of "[r1+0]=r2; [r1+4]=r3" on the same cycle
5775 : (e.g., once "[r1+0]=r2" is issued in max_issue(), "[r1+4]=r3" gets
5776 : unblocked). We don't bother about this yet because target of interest
5777 : (ARM Cortex-A15) can issue only 1 memory operation per cycle. */
5778 :
5779 : /* Implementation of first_cycle_multipass_dfa_lookahead_guard hook.
5780 : Return "1" if INSN1 should not be considered in max_issue due to
5781 : auto-prefetcher considerations. */
5782 : int
5783 0 : autopref_multipass_dfa_lookahead_guard (rtx_insn *insn1, int ready_index)
5784 : {
5785 0 : int r = 0;
5786 :
5787 : /* Exit early if the param forbids this or if we're not entering here through
5788 : normal haifa scheduling. This can happen if selective scheduling is
5789 : explicitly enabled. */
5790 0 : if (!insn_queue || param_sched_autopref_queue_depth <= 0)
5791 : return 0;
5792 :
5793 0 : if (sched_verbose >= 2 && ready_index == 0)
5794 0 : autopref_multipass_dfa_lookahead_guard_started_dump_p = false;
5795 :
5796 0 : for (int write = 0; write < 2; ++write)
5797 : {
5798 0 : autopref_multipass_data_t data1
5799 0 : = &INSN_AUTOPREF_MULTIPASS_DATA (insn1)[write];
5800 :
5801 0 : if (data1->status == AUTOPREF_MULTIPASS_DATA_UNINITIALIZED)
5802 0 : autopref_multipass_init (insn1, write);
5803 0 : if (data1->status == AUTOPREF_MULTIPASS_DATA_IRRELEVANT)
5804 0 : continue;
5805 :
5806 0 : if (ready_index == 0
5807 0 : && data1->status == AUTOPREF_MULTIPASS_DATA_DONT_DELAY)
5808 : /* We allow only a single delay on privileged instructions.
5809 : Doing otherwise would cause infinite loop. */
5810 : {
5811 0 : if (sched_verbose >= 2)
5812 : {
5813 0 : if (!autopref_multipass_dfa_lookahead_guard_started_dump_p)
5814 : {
5815 0 : fprintf (sched_dump,
5816 : ";;\t\tnot trying in max_issue due to autoprefetch "
5817 : "model: ");
5818 0 : autopref_multipass_dfa_lookahead_guard_started_dump_p = true;
5819 : }
5820 :
5821 0 : fprintf (sched_dump, " *%d*", INSN_UID (insn1));
5822 : }
5823 0 : continue;
5824 : }
5825 :
5826 0 : for (int i2 = 0; i2 < ready.n_ready; ++i2)
5827 : {
5828 0 : rtx_insn *insn2 = get_ready_element (i2);
5829 0 : if (insn1 == insn2)
5830 0 : continue;
5831 0 : r = autopref_multipass_dfa_lookahead_guard_1 (insn1, insn2, write);
5832 0 : if (r)
5833 : {
5834 0 : if (ready_index == 0)
5835 : {
5836 0 : r = -1;
5837 0 : data1->status = AUTOPREF_MULTIPASS_DATA_DONT_DELAY;
5838 : }
5839 0 : goto finish;
5840 : }
5841 : }
5842 :
5843 0 : if (param_sched_autopref_queue_depth == 1)
5844 0 : continue;
5845 :
5846 : /* Everything from the current queue slot should have been moved to
5847 : the ready list. */
5848 0 : gcc_assert (insn_queue[NEXT_Q_AFTER (q_ptr, 0)] == NULL_RTX);
5849 :
5850 0 : int n_stalls = param_sched_autopref_queue_depth - 1;
5851 0 : if (n_stalls > max_insn_queue_index)
5852 : n_stalls = max_insn_queue_index;
5853 :
5854 0 : for (int stalls = 1; stalls <= n_stalls; ++stalls)
5855 : {
5856 0 : for (rtx_insn_list *link = insn_queue[NEXT_Q_AFTER (q_ptr, stalls)];
5857 0 : link != NULL_RTX;
5858 0 : link = link->next ())
5859 : {
5860 0 : rtx_insn *insn2 = link->insn ();
5861 0 : r = autopref_multipass_dfa_lookahead_guard_1 (insn1, insn2,
5862 : write);
5863 0 : if (r)
5864 : {
5865 : /* Queue INSN1 until INSN2 can issue. */
5866 0 : r = -stalls;
5867 0 : if (ready_index == 0)
5868 0 : data1->status = AUTOPREF_MULTIPASS_DATA_DONT_DELAY;
5869 0 : goto finish;
5870 : }
5871 : }
5872 : }
5873 : }
5874 :
5875 0 : finish:
5876 0 : if (sched_verbose >= 2
5877 0 : && autopref_multipass_dfa_lookahead_guard_started_dump_p
5878 0 : && (ready_index == ready.n_ready - 1 || r < 0))
5879 : /* This does not /always/ trigger. We don't output EOL if the last
5880 : insn is not recognized (INSN_CODE < 0) and lookahead_guard is not
5881 : called. We can live with this. */
5882 0 : fprintf (sched_dump, "\n");
5883 :
5884 : return r;
5885 : }
5886 :
5887 : /* Define type for target data used in multipass scheduling. */
5888 : #ifndef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DATA_T
5889 : # define TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DATA_T int
5890 : #endif
5891 : typedef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DATA_T first_cycle_multipass_data_t;
5892 :
5893 : /* The following structure describe an entry of the stack of choices. */
5894 : struct choice_entry
5895 : {
5896 : /* Ordinal number of the issued insn in the ready queue. */
5897 : int index;
5898 : /* The number of the rest insns whose issues we should try. */
5899 : int rest;
5900 : /* The number of issued essential insns. */
5901 : int n;
5902 : /* State after issuing the insn. */
5903 : state_t state;
5904 : /* Target-specific data. */
5905 : first_cycle_multipass_data_t target_data;
5906 : };
5907 :
5908 : /* The following array is used to implement a stack of choices used in
5909 : function max_issue. */
5910 : static struct choice_entry *choice_stack;
5911 :
5912 : /* This holds the value of the target dfa_lookahead hook. */
5913 : int dfa_lookahead;
5914 :
5915 : /* The following variable value is maximal number of tries of issuing
5916 : insns for the first cycle multipass insn scheduling. We define
5917 : this value as constant*(DFA_LOOKAHEAD**ISSUE_RATE). We would not
5918 : need this constraint if all real insns (with non-negative codes)
5919 : had reservations because in this case the algorithm complexity is
5920 : O(DFA_LOOKAHEAD**ISSUE_RATE). Unfortunately, the dfa descriptions
5921 : might be incomplete and such insn might occur. For such
5922 : descriptions, the complexity of algorithm (without the constraint)
5923 : could achieve DFA_LOOKAHEAD ** N , where N is the queue length. */
5924 : static int max_lookahead_tries;
5925 :
5926 : /* The following function returns maximal (or close to maximal) number
5927 : of insns which can be issued on the same cycle and one of which
5928 : insns is insns with the best rank (the first insn in READY). To
5929 : make this function tries different samples of ready insns. READY
5930 : is current queue `ready'. Global array READY_TRY reflects what
5931 : insns are already issued in this try. The function stops immediately,
5932 : if it reached the such a solution, that all instruction can be issued.
5933 : INDEX will contain index of the best insn in READY. The following
5934 : function is used only for first cycle multipass scheduling.
5935 :
5936 : PRIVILEGED_N >= 0
5937 :
5938 : This function expects recognized insns only. All USEs,
5939 : CLOBBERs, etc must be filtered elsewhere. */
5940 : int
5941 56392543 : max_issue (struct ready_list *ready, int privileged_n, state_t state,
5942 : bool first_cycle_insn_p, int *index)
5943 : {
5944 56392543 : int n, i, all, n_ready, best, delay, tries_num;
5945 56392543 : int more_issue;
5946 56392543 : struct choice_entry *top;
5947 56392543 : rtx_insn *insn;
5948 :
5949 56392543 : if (sched_fusion)
5950 : return 0;
5951 :
5952 56392543 : n_ready = ready->n_ready;
5953 56392543 : gcc_assert (dfa_lookahead >= 1 && privileged_n >= 0
5954 : && privileged_n <= n_ready);
5955 :
5956 : /* Init MAX_LOOKAHEAD_TRIES. */
5957 56392543 : if (max_lookahead_tries == 0)
5958 : {
5959 : int64_t max_tries = 100;
5960 4869271 : for (i = 0; i < issue_rate; i++)
5961 : {
5962 3886185 : max_tries *= dfa_lookahead;
5963 3886185 : if (max_tries > INT_MAX)
5964 : break;
5965 : }
5966 983086 : max_lookahead_tries = MIN (max_tries, INT_MAX);
5967 : }
5968 :
5969 : /* Init max_points. */
5970 56392543 : more_issue = issue_rate - cycle_issued_insns;
5971 56392543 : gcc_assert (more_issue >= 0);
5972 :
5973 : /* The number of the issued insns in the best solution. */
5974 56392543 : best = 0;
5975 :
5976 56392543 : top = choice_stack;
5977 :
5978 : /* Set initial state of the search. */
5979 56392543 : memcpy (top->state, state, dfa_state_size);
5980 56392543 : top->rest = dfa_lookahead;
5981 56392543 : top->n = 0;
5982 56392543 : if (targetm.sched.first_cycle_multipass_begin)
5983 56198107 : targetm.sched.first_cycle_multipass_begin (&top->target_data,
5984 : ready_try, n_ready,
5985 : first_cycle_insn_p);
5986 :
5987 : /* Count the number of the insns to search among. */
5988 151962326 : for (all = i = 0; i < n_ready; i++)
5989 95569783 : if (!ready_try [i])
5990 87403775 : all++;
5991 :
5992 56392543 : if (sched_verbose >= 2)
5993 : {
5994 0 : fprintf (sched_dump, ";;\t\tmax_issue among %d insns:", all);
5995 0 : debug_ready_list_1 (ready, ready_try);
5996 : }
5997 :
5998 : /* I is the index of the insn to try next. */
5999 56392543 : i = 0;
6000 56392543 : tries_num = 0;
6001 1097820391 : for (;;)
6002 : {
6003 577106467 : if (/* If we've reached a dead end or searched enough of what we have
6004 : been asked... */
6005 577106467 : top->rest == 0
6006 : /* or have nothing else to try... */
6007 571405067 : || i >= n_ready
6008 : /* or should not issue more. */
6009 464095594 : || top->n >= more_issue)
6010 : {
6011 : /* ??? (... || i == n_ready). */
6012 115191586 : gcc_assert (i <= n_ready);
6013 :
6014 : /* We should not issue more than issue_rate instructions. */
6015 115191586 : gcc_assert (top->n <= more_issue);
6016 :
6017 115191586 : if (top == choice_stack)
6018 : break;
6019 :
6020 106071669 : if (best < top - choice_stack)
6021 : {
6022 54733952 : if (privileged_n)
6023 : {
6024 : n = privileged_n;
6025 : /* Try to find issued privileged insn. */
6026 109461184 : while (n && !ready_try[--n])
6027 : ;
6028 : }
6029 :
6030 54730592 : if (/* If all insns are equally good... */
6031 : privileged_n == 0
6032 : /* Or a privileged insn will be issued. */
6033 54730592 : || ready_try[n])
6034 : /* Then we have a solution. */
6035 : {
6036 54732522 : best = top - choice_stack;
6037 : /* This is the index of the insn issued first in this
6038 : solution. */
6039 54732522 : *index = choice_stack [1].index;
6040 54732522 : if (top->n == more_issue || best == all)
6041 : break;
6042 : }
6043 : }
6044 :
6045 : /* Set ready-list index to point to the last insn
6046 : ('i++' below will advance it to the next insn). */
6047 58799064 : i = top->index;
6048 :
6049 : /* Backtrack. */
6050 58799064 : ready_try [i] = 0;
6051 :
6052 58799064 : if (targetm.sched.first_cycle_multipass_backtrack)
6053 58599473 : targetm.sched.first_cycle_multipass_backtrack (&top->target_data,
6054 : ready_try, n_ready);
6055 :
6056 58799064 : top--;
6057 58799064 : memcpy (state, top->state, dfa_state_size);
6058 : }
6059 461914881 : else if (!ready_try [i])
6060 : {
6061 147073437 : tries_num++;
6062 : /* max_lookahead_tries is capped at INT_MAX, use the >= comparison
6063 : to prevent tries_num from overflowing. */
6064 147073437 : if (tries_num >= max_lookahead_tries)
6065 : break;
6066 147073416 : insn = ready_element (ready, i);
6067 147073416 : delay = state_transition (state, insn);
6068 147073416 : if (delay < 0)
6069 : {
6070 114986261 : if (state_dead_lock_p (state)
6071 114986261 : || insn_finishes_cycle_p (insn))
6072 : /* We won't issue any more instructions in the next
6073 : choice_state. */
6074 5945828 : top->rest = 0;
6075 : else
6076 109040433 : top->rest--;
6077 :
6078 114986261 : n = top->n;
6079 114986261 : if (memcmp (top->state, state, dfa_state_size) != 0)
6080 113752742 : n++;
6081 :
6082 : /* Advance to the next choice_entry. */
6083 114986261 : top++;
6084 : /* Initialize it. */
6085 114986261 : top->rest = dfa_lookahead;
6086 114986261 : top->index = i;
6087 114986261 : top->n = n;
6088 114986261 : memcpy (top->state, state, dfa_state_size);
6089 114986261 : ready_try [i] = 1;
6090 :
6091 114986261 : if (targetm.sched.first_cycle_multipass_issue)
6092 114581063 : targetm.sched.first_cycle_multipass_issue (&top->target_data,
6093 : ready_try, n_ready,
6094 : insn,
6095 114581063 : &((top - 1)
6096 : ->target_data));
6097 :
6098 : i = -1;
6099 : }
6100 : }
6101 :
6102 : /* Increase ready-list index. */
6103 520713924 : i++;
6104 : }
6105 :
6106 56392543 : if (targetm.sched.first_cycle_multipass_end)
6107 110145637 : targetm.sched.first_cycle_multipass_end (best != 0
6108 53947530 : ? &choice_stack[1].target_data
6109 : : NULL);
6110 :
6111 : /* Restore the original state of the DFA. */
6112 56392543 : memcpy (state, choice_stack->state, dfa_state_size);
6113 :
6114 56392543 : return best;
6115 : }
6116 :
6117 : /* The following function chooses insn from READY and modifies
6118 : READY. The following function is used only for first
6119 : cycle multipass scheduling.
6120 : Return:
6121 : -1 if cycle should be advanced,
6122 : 0 if INSN_PTR is set to point to the desirable insn,
6123 : 1 if choose_ready () should be restarted without advancing the cycle. */
6124 : static int
6125 61463046 : choose_ready (struct ready_list *ready, bool first_cycle_insn_p,
6126 : rtx_insn **insn_ptr)
6127 : {
6128 61463046 : if (dbg_cnt (sched_insn) == false)
6129 : {
6130 0 : if (nonscheduled_insns_begin == NULL_RTX)
6131 0 : nonscheduled_insns_begin = current_sched_info->prev_head;
6132 :
6133 0 : rtx_insn *insn = first_nonscheduled_insn ();
6134 :
6135 0 : if (QUEUE_INDEX (insn) == QUEUE_READY)
6136 : /* INSN is in the ready_list. */
6137 : {
6138 0 : ready_remove_insn (insn);
6139 0 : *insn_ptr = insn;
6140 0 : return 0;
6141 : }
6142 :
6143 : /* INSN is in the queue. Advance cycle to move it to the ready list. */
6144 0 : gcc_assert (QUEUE_INDEX (insn) >= 0);
6145 : return -1;
6146 : }
6147 :
6148 61463046 : if (SCHED_GROUP_P (ready_element (ready, 0))
6149 61463046 : || DEBUG_INSN_P (ready_element (ready, 0)))
6150 : {
6151 4212537 : *insn_ptr = ready_remove_first (ready);
6152 4212537 : return 0;
6153 : }
6154 57250509 : else if (dfa_lookahead <= 0)
6155 : {
6156 9728 : if (targetm.sched.dispatch (NULL, IS_DISPATCH_ON))
6157 16 : *insn_ptr = ready_remove_first_dispatch (ready);
6158 : else
6159 9712 : *insn_ptr = ready_remove_first (ready);
6160 :
6161 9728 : return 0;
6162 : }
6163 : else
6164 : {
6165 : /* Try to choose the best insn. */
6166 57240781 : int index = 0, i;
6167 57240781 : rtx_insn *insn;
6168 :
6169 57240781 : insn = ready_element (ready, 0);
6170 57240781 : if (INSN_CODE (insn) < 0)
6171 : {
6172 851627 : *insn_ptr = ready_remove_first (ready);
6173 851627 : return 0;
6174 : }
6175 :
6176 : /* Filter the search space. */
6177 151953444 : for (i = 0; i < ready->n_ready; i++)
6178 : {
6179 95564290 : ready_try[i] = 0;
6180 :
6181 95564290 : insn = ready_element (ready, i);
6182 :
6183 : /* If this insn is recognizable we should have already
6184 : recognized it earlier.
6185 : ??? Not very clear where this is supposed to be done.
6186 : See dep_cost_1. */
6187 95564290 : gcc_checking_assert (INSN_CODE (insn) >= 0
6188 : || recog_memoized (insn) < 0);
6189 95564290 : if (INSN_CODE (insn) < 0)
6190 : {
6191 : /* Non-recognized insns at position 0 are handled above. */
6192 244854 : gcc_assert (i > 0);
6193 244854 : ready_try[i] = 1;
6194 244854 : continue;
6195 : }
6196 :
6197 95319436 : if (targetm.sched.first_cycle_multipass_dfa_lookahead_guard)
6198 : {
6199 0 : ready_try[i]
6200 0 : = (targetm.sched.first_cycle_multipass_dfa_lookahead_guard
6201 0 : (insn, i));
6202 :
6203 0 : if (ready_try[i] < 0)
6204 : /* Queue instruction for several cycles.
6205 : We need to restart choose_ready as we have changed
6206 : the ready list. */
6207 : {
6208 0 : change_queue_index (insn, -ready_try[i]);
6209 0 : return 1;
6210 : }
6211 :
6212 : /* Make sure that we didn't end up with 0'th insn filtered out.
6213 : Don't be tempted to make life easier for backends and just
6214 : requeue 0'th insn if (ready_try[0] == 0) and restart
6215 : choose_ready. Backends should be very considerate about
6216 : requeueing instructions -- especially the highest priority
6217 : one at position 0. */
6218 0 : gcc_assert (ready_try[i] == 0 || i > 0);
6219 0 : if (ready_try[i])
6220 0 : continue;
6221 : }
6222 :
6223 95319436 : gcc_assert (ready_try[i] == 0);
6224 : /* INSN made it through the scrutiny of filters! */
6225 : }
6226 :
6227 56389154 : if (max_issue (ready, 1, curr_state, first_cycle_insn_p, &index) == 0)
6228 : {
6229 2250566 : *insn_ptr = ready_remove_first (ready);
6230 2250566 : if (sched_verbose >= 4)
6231 0 : fprintf (sched_dump, ";;\t\tChosen insn (but can't issue) : %s \n",
6232 0 : (*current_sched_info->print_insn) (*insn_ptr, 0));
6233 : return 0;
6234 : }
6235 : else
6236 : {
6237 54138588 : if (sched_verbose >= 4)
6238 0 : fprintf (sched_dump, ";;\t\tChosen insn : %s\n",
6239 0 : (*current_sched_info->print_insn)
6240 0 : (ready_element (ready, index), 0));
6241 :
6242 54138588 : *insn_ptr = ready_remove (ready, index);
6243 54138588 : return 0;
6244 : }
6245 : }
6246 : }
6247 :
6248 : /* This function is called when we have successfully scheduled a
6249 : block. It uses the schedule stored in the scheduled_insns vector
6250 : to rearrange the RTL. PREV_HEAD is used as the anchor to which we
6251 : append the scheduled insns; TAIL is the insn after the scheduled
6252 : block. TARGET_BB is the argument passed to schedule_block. */
6253 :
6254 : static void
6255 10477678 : commit_schedule (rtx_insn *prev_head, rtx_insn *tail, basic_block *target_bb)
6256 : {
6257 10477678 : unsigned int i;
6258 10477678 : rtx_insn *insn;
6259 :
6260 10477678 : last_scheduled_insn = prev_head;
6261 10477678 : for (i = 0;
6262 124791151 : scheduled_insns.iterate (i, &insn);
6263 : i++)
6264 : {
6265 114313473 : if (control_flow_insn_p (last_scheduled_insn)
6266 114313473 : || current_sched_info->advance_target_bb (*target_bb, insn))
6267 : {
6268 49 : *target_bb = current_sched_info->advance_target_bb (*target_bb, 0);
6269 :
6270 49 : if (sched_verbose)
6271 : {
6272 2 : rtx_insn *x;
6273 :
6274 2 : x = next_real_insn (last_scheduled_insn);
6275 2 : gcc_assert (x);
6276 2 : dump_new_block_header (1, *target_bb, x, tail);
6277 : }
6278 :
6279 49 : last_scheduled_insn = bb_note (*target_bb);
6280 : }
6281 :
6282 114313473 : if (current_sched_info->begin_move_insn)
6283 1259 : (*current_sched_info->begin_move_insn) (insn, last_scheduled_insn);
6284 114313473 : move_insn (insn, last_scheduled_insn,
6285 114313473 : current_sched_info->next_tail);
6286 114313473 : if (!DEBUG_INSN_P (insn))
6287 61460831 : reemit_notes (insn);
6288 114313473 : last_scheduled_insn = insn;
6289 : }
6290 :
6291 10477678 : scheduled_insns.truncate (0);
6292 10477678 : }
6293 :
6294 : /* Examine all insns on the ready list and queue those which can't be
6295 : issued in this cycle. TEMP_STATE is temporary scheduler state we
6296 : can use as scratch space. If FIRST_CYCLE_INSN_P is true, no insns
6297 : have been issued for the current cycle, which means it is valid to
6298 : issue an asm statement.
6299 :
6300 : If SHADOWS_ONLY_P is true, we eliminate all real insns and only
6301 : leave those for which SHADOW_P is true. If MODULO_EPILOGUE is true,
6302 : we only leave insns which have an INSN_EXACT_TICK. */
6303 :
6304 : static void
6305 70693714 : prune_ready_list (state_t temp_state, bool first_cycle_insn_p,
6306 : bool shadows_only_p, bool modulo_epilogue_p)
6307 : {
6308 70693714 : int i, pass;
6309 70693714 : bool sched_group_found = false;
6310 70693714 : int min_cost_group = 0;
6311 :
6312 70693714 : if (sched_fusion)
6313 : return;
6314 :
6315 193887626 : for (i = 0; i < ready.n_ready; i++)
6316 : {
6317 127427086 : rtx_insn *insn = ready_element (&ready, i);
6318 127427086 : if (SCHED_GROUP_P (insn))
6319 : {
6320 : sched_group_found = true;
6321 : break;
6322 : }
6323 : }
6324 :
6325 : /* Make two passes if there's a SCHED_GROUP_P insn; make sure to handle
6326 : such an insn first and note its cost. If at least one SCHED_GROUP_P insn
6327 : gets queued, then all other insns get queued for one cycle later. */
6328 157739164 : for (pass = sched_group_found ? 0 : 1; pass < 2; )
6329 : {
6330 87045450 : int n = ready.n_ready;
6331 207819990 : for (i = 0; i < n; i++)
6332 : {
6333 132893102 : rtx_insn *insn = ready_element (&ready, i);
6334 132893102 : int cost = 0;
6335 132893102 : const char *reason = "resource conflict";
6336 :
6337 132893102 : if (DEBUG_INSN_P (insn))
6338 6652623 : continue;
6339 :
6340 8450499 : if (sched_group_found && !SCHED_GROUP_P (insn)
6341 126240695 : && ((pass == 0) || (min_cost_group >= 1)))
6342 : {
6343 128 : if (pass == 0)
6344 108 : continue;
6345 : cost = min_cost_group;
6346 : reason = "not in sched group";
6347 : }
6348 126240351 : else if (modulo_epilogue_p
6349 126240351 : && INSN_EXACT_TICK (insn) == INVALID_TICK)
6350 : {
6351 : cost = max_insn_queue_index;
6352 : reason = "not an epilogue insn";
6353 : }
6354 126240351 : else if (shadows_only_p && !SHADOW_P (insn))
6355 : {
6356 : cost = 1;
6357 : reason = "not a shadow";
6358 : }
6359 126240351 : else if (recog_memoized (insn) < 0)
6360 : {
6361 1164431 : if (!first_cycle_insn_p
6362 1164431 : && (GET_CODE (PATTERN (insn)) == ASM_INPUT
6363 943158 : || asm_noperands (PATTERN (insn)) >= 0))
6364 : cost = 1;
6365 : reason = "asm";
6366 : }
6367 125075920 : else if (sched_pressure != SCHED_PRESSURE_NONE)
6368 : {
6369 22199 : if (sched_pressure == SCHED_PRESSURE_MODEL
6370 22199 : && INSN_TICK (insn) <= clock_var)
6371 : {
6372 0 : memcpy (temp_state, curr_state, dfa_state_size);
6373 0 : if (state_transition (temp_state, insn) >= 0)
6374 0 : INSN_TICK (insn) = clock_var + 1;
6375 : }
6376 : cost = 0;
6377 : }
6378 : else
6379 : {
6380 125053721 : int delay_cost = 0;
6381 :
6382 125053721 : if (delay_htab)
6383 : {
6384 0 : struct delay_pair *delay_entry;
6385 0 : delay_entry
6386 0 : = delay_htab->find_with_hash (insn,
6387 : htab_hash_pointer (insn));
6388 0 : while (delay_entry && delay_cost == 0)
6389 : {
6390 0 : delay_cost = estimate_shadow_tick (delay_entry);
6391 0 : if (delay_cost > max_insn_queue_index)
6392 : delay_cost = max_insn_queue_index;
6393 0 : delay_entry = delay_entry->next_same_i1;
6394 : }
6395 : }
6396 :
6397 125053721 : memcpy (temp_state, curr_state, dfa_state_size);
6398 125053721 : cost = state_transition (temp_state, insn);
6399 125053721 : if (cost < 0)
6400 : cost = 0;
6401 19855106 : else if (cost == 0)
6402 0 : cost = 1;
6403 125053721 : if (cost < delay_cost)
6404 : {
6405 0 : cost = delay_cost;
6406 0 : reason = "shadow tick";
6407 : }
6408 : }
6409 125053741 : if (cost >= 1)
6410 : {
6411 19901851 : if (SCHED_GROUP_P (insn) && cost > min_cost_group)
6412 19901851 : min_cost_group = cost;
6413 19901851 : ready_remove (&ready, i);
6414 : /* Normally we'd want to queue INSN for COST cycles. However,
6415 : if SCHED_GROUP_P is set, then we must ensure that nothing
6416 : else comes between INSN and its predecessor. If there is
6417 : some other insn ready to fire on the next cycle, then that
6418 : invariant would be broken.
6419 :
6420 : So when SCHED_GROUP_P is set, just queue this insn for a
6421 : single cycle. */
6422 19917916 : queue_insn (insn, SCHED_GROUP_P (insn) ? 1 : cost, reason);
6423 19901851 : if (i + 1 < n)
6424 : break;
6425 : }
6426 : }
6427 87045450 : if (i == n)
6428 74926888 : pass++;
6429 : }
6430 : }
6431 :
6432 : /* Called when we detect that the schedule is impossible. We examine the
6433 : backtrack queue to find the earliest insn that caused this condition. */
6434 :
6435 : static struct haifa_saved_data *
6436 0 : verify_shadows (void)
6437 : {
6438 0 : struct haifa_saved_data *save, *earliest_fail = NULL;
6439 0 : for (save = backtrack_queue; save; save = save->next)
6440 : {
6441 0 : int t;
6442 0 : struct delay_pair *pair = save->delay_pair;
6443 0 : rtx_insn *i1 = pair->i1;
6444 :
6445 0 : for (; pair; pair = pair->next_same_i1)
6446 : {
6447 0 : rtx_insn *i2 = pair->i2;
6448 :
6449 0 : if (QUEUE_INDEX (i2) == QUEUE_SCHEDULED)
6450 0 : continue;
6451 :
6452 0 : t = INSN_TICK (i1) + pair_delay (pair);
6453 0 : if (t < clock_var)
6454 : {
6455 0 : if (sched_verbose >= 2)
6456 0 : fprintf (sched_dump,
6457 : ";;\t\tfailed delay requirements for %d/%d (%d->%d)"
6458 : ", not ready\n",
6459 : INSN_UID (pair->i1), INSN_UID (pair->i2),
6460 0 : INSN_TICK (pair->i1), INSN_EXACT_TICK (pair->i2));
6461 : earliest_fail = save;
6462 : break;
6463 : }
6464 0 : if (QUEUE_INDEX (i2) >= 0)
6465 : {
6466 0 : int queued_for = INSN_TICK (i2);
6467 :
6468 0 : if (t < queued_for)
6469 : {
6470 0 : if (sched_verbose >= 2)
6471 0 : fprintf (sched_dump,
6472 : ";;\t\tfailed delay requirements for %d/%d"
6473 : " (%d->%d), queued too late\n",
6474 : INSN_UID (pair->i1), INSN_UID (pair->i2),
6475 0 : INSN_TICK (pair->i1), INSN_EXACT_TICK (pair->i2));
6476 : earliest_fail = save;
6477 : break;
6478 : }
6479 : }
6480 : }
6481 : }
6482 :
6483 0 : return earliest_fail;
6484 : }
6485 :
6486 : /* Print instructions together with useful scheduling information between
6487 : HEAD and TAIL (inclusive). */
6488 : static void
6489 0 : dump_insn_stream (rtx_insn *head, rtx_insn *tail)
6490 : {
6491 0 : fprintf (sched_dump, ";;\t| insn | prio |\n");
6492 :
6493 0 : rtx_insn *next_tail = NEXT_INSN (tail);
6494 0 : for (rtx_insn *insn = head; insn != next_tail; insn = NEXT_INSN (insn))
6495 : {
6496 0 : int priority = NOTE_P (insn) ? 0 : INSN_PRIORITY (insn);
6497 0 : const char *pattern = (NOTE_P (insn)
6498 0 : ? "note"
6499 0 : : str_pattern_slim (PATTERN (insn)));
6500 :
6501 0 : fprintf (sched_dump, ";;\t| %4d | %4d | %-30s ",
6502 0 : INSN_UID (insn), priority, pattern);
6503 :
6504 0 : if (sched_verbose >= 4)
6505 : {
6506 0 : if (NOTE_P (insn) || LABEL_P (insn) || recog_memoized (insn) < 0)
6507 0 : fprintf (sched_dump, "nothing");
6508 : else
6509 0 : print_reservation (sched_dump, insn);
6510 : }
6511 0 : fprintf (sched_dump, "\n");
6512 : }
6513 0 : }
6514 :
6515 : /* Use forward list scheduling to rearrange insns of block pointed to by
6516 : TARGET_BB, possibly bringing insns from subsequent blocks in the same
6517 : region. */
6518 :
6519 : bool
6520 10477678 : schedule_block (basic_block *target_bb, state_t init_state)
6521 : {
6522 10477678 : int i;
6523 10477678 : bool success = modulo_ii == 0;
6524 10477678 : struct sched_block_state ls;
6525 10477678 : state_t temp_state = NULL; /* It is used for multipass scheduling. */
6526 10477678 : int sort_p, advance, start_clock_var;
6527 :
6528 : /* Head/tail info for this block. */
6529 10477678 : rtx_insn *prev_head = current_sched_info->prev_head;
6530 10477678 : rtx_insn *next_tail = current_sched_info->next_tail;
6531 10477678 : rtx_insn *head = NEXT_INSN (prev_head);
6532 10477678 : rtx_insn *tail = PREV_INSN (next_tail);
6533 :
6534 10477678 : if ((current_sched_info->flags & DONT_BREAK_DEPENDENCIES) == 0
6535 10477678 : && sched_pressure != SCHED_PRESSURE_MODEL && !sched_fusion)
6536 10477678 : find_modifiable_mems (head, tail);
6537 :
6538 : /* We used to have code to avoid getting parameters moved from hard
6539 : argument registers into pseudos.
6540 :
6541 : However, it was removed when it proved to be of marginal benefit
6542 : and caused problems because schedule_block and compute_forward_dependences
6543 : had different notions of what the "head" insn was. */
6544 :
6545 10477678 : gcc_assert (head != tail || INSN_P (head));
6546 :
6547 10477678 : haifa_recovery_bb_recently_added_p = false;
6548 :
6549 10477678 : backtrack_queue = NULL;
6550 :
6551 : /* Debug info. */
6552 10477678 : if (sched_verbose)
6553 : {
6554 190 : dump_new_block_header (0, *target_bb, head, tail);
6555 :
6556 190 : if (sched_verbose >= 2)
6557 : {
6558 0 : dump_insn_stream (head, tail);
6559 0 : memset (&rank_for_schedule_stats, 0,
6560 : sizeof (rank_for_schedule_stats));
6561 : }
6562 : }
6563 :
6564 10477678 : if (init_state == NULL)
6565 173 : state_reset (curr_state);
6566 : else
6567 10477505 : memcpy (curr_state, init_state, dfa_state_size);
6568 :
6569 : /* Clear the ready list. */
6570 10477678 : ready.first = ready.veclen - 1;
6571 10477678 : ready.n_ready = 0;
6572 10477678 : ready.n_debug = 0;
6573 :
6574 : /* It is used for first cycle multipass scheduling. */
6575 10477678 : temp_state = alloca (dfa_state_size);
6576 :
6577 10477678 : if (targetm.sched.init)
6578 0 : targetm.sched.init (sched_dump, sched_verbose, ready.veclen);
6579 :
6580 : /* We start inserting insns after PREV_HEAD. */
6581 10477678 : last_scheduled_insn = prev_head;
6582 10477678 : last_nondebug_scheduled_insn = NULL;
6583 10477678 : nonscheduled_insns_begin = NULL;
6584 :
6585 10477678 : gcc_assert ((NOTE_P (last_scheduled_insn)
6586 : || DEBUG_INSN_P (last_scheduled_insn))
6587 : && BLOCK_FOR_INSN (last_scheduled_insn) == *target_bb);
6588 :
6589 : /* Initialize INSN_QUEUE. Q_SIZE is the total number of insns in the
6590 : queue. */
6591 10477678 : q_ptr = 0;
6592 10477678 : q_size = 0;
6593 :
6594 10477678 : insn_queue = XALLOCAVEC (rtx_insn_list *, max_insn_queue_index + 1);
6595 10477678 : memset (insn_queue, 0, (max_insn_queue_index + 1) * sizeof (rtx));
6596 :
6597 : /* Start just before the beginning of time. */
6598 10477678 : clock_var = -1;
6599 :
6600 : /* We need queue and ready lists and clock_var be initialized
6601 : in try_ready () (which is called through init_ready_list ()). */
6602 10477678 : (*current_sched_info->init_ready_list) ();
6603 :
6604 10477678 : if (sched_pressure)
6605 706 : sched_pressure_start_bb (*target_bb);
6606 :
6607 : /* The algorithm is O(n^2) in the number of ready insns at any given
6608 : time in the worst case. Before reload we are more likely to have
6609 : big lists so truncate them to a reasonable size. */
6610 10477678 : if (!reload_completed
6611 1201 : && ready.n_ready - ready.n_debug > param_max_sched_ready_insns)
6612 : {
6613 4 : ready_sort_debug (&ready);
6614 4 : ready_sort_real (&ready);
6615 :
6616 : /* Find first free-standing insn past param_max_sched_ready_insns.
6617 : If there are debug insns, we know they're first. */
6618 4 : for (i = param_max_sched_ready_insns + ready.n_debug; i < ready.n_ready;
6619 : i++)
6620 4 : if (!SCHED_GROUP_P (ready_element (&ready, i)))
6621 : break;
6622 :
6623 4 : if (sched_verbose >= 2)
6624 : {
6625 0 : fprintf (sched_dump,
6626 : ";;\t\tReady list on entry: %d insns: ", ready.n_ready);
6627 0 : debug_ready_list (&ready);
6628 0 : fprintf (sched_dump,
6629 : ";;\t\t before reload => truncated to %d insns\n", i);
6630 : }
6631 :
6632 : /* Delay all insns past it for 1 cycle. If debug counter is
6633 : activated make an exception for the insn right after
6634 : nonscheduled_insns_begin. */
6635 4 : {
6636 4 : rtx_insn *skip_insn;
6637 :
6638 4 : if (dbg_cnt (sched_insn) == false)
6639 0 : skip_insn = first_nonscheduled_insn ();
6640 : else
6641 4 : skip_insn = NULL;
6642 :
6643 29 : while (i < ready.n_ready)
6644 : {
6645 25 : rtx_insn *insn;
6646 :
6647 25 : insn = ready_remove (&ready, i);
6648 :
6649 25 : if (insn != skip_insn)
6650 25 : queue_insn (insn, 1, "list truncated");
6651 : }
6652 4 : if (skip_insn)
6653 0 : ready_add (&ready, skip_insn, true);
6654 : }
6655 : }
6656 :
6657 : /* Now we can restore basic block notes and maintain precise cfg. */
6658 10477678 : restore_bb_notes (*target_bb);
6659 :
6660 10477678 : last_clock_var = -1;
6661 :
6662 10477678 : advance = 0;
6663 :
6664 10477678 : gcc_assert (scheduled_insns.length () == 0);
6665 10477678 : sort_p = true;
6666 10477678 : must_backtrack = false;
6667 10477678 : modulo_insns_scheduled = 0;
6668 :
6669 10477678 : ls.modulo_epilogue = false;
6670 10477678 : ls.first_cycle_insn_p = true;
6671 :
6672 : /* Loop until all the insns in BB are scheduled. */
6673 46185778 : while ((*current_sched_info->schedule_more_p) ())
6674 : {
6675 35708100 : perform_replacements_new_cycle ();
6676 35708100 : do
6677 : {
6678 35708100 : start_clock_var = clock_var;
6679 :
6680 35708100 : clock_var++;
6681 :
6682 35708100 : advance_one_cycle ();
6683 :
6684 : /* Add to the ready list all pending insns that can be issued now.
6685 : If there are no ready insns, increment clock until one
6686 : is ready and add all pending insns at that point to the ready
6687 : list. */
6688 35708100 : queue_to_ready (&ready);
6689 :
6690 35708100 : gcc_assert (ready.n_ready);
6691 :
6692 35708100 : if (sched_verbose >= 2)
6693 : {
6694 0 : fprintf (sched_dump, ";;\t\tReady list after queue_to_ready:");
6695 0 : debug_ready_list (&ready);
6696 : }
6697 35708100 : advance -= clock_var - start_clock_var;
6698 : }
6699 35708100 : while (advance > 0);
6700 :
6701 35708100 : if (ls.modulo_epilogue)
6702 : {
6703 0 : int stage = clock_var / modulo_ii;
6704 0 : if (stage > modulo_last_stage * 2 + 2)
6705 : {
6706 0 : if (sched_verbose >= 2)
6707 0 : fprintf (sched_dump,
6708 : ";;\t\tmodulo scheduled succeeded at II %d\n",
6709 : modulo_ii);
6710 0 : success = true;
6711 0 : goto end_schedule;
6712 : }
6713 : }
6714 35708100 : else if (modulo_ii > 0)
6715 : {
6716 0 : int stage = clock_var / modulo_ii;
6717 0 : if (stage > modulo_max_stages)
6718 : {
6719 0 : if (sched_verbose >= 2)
6720 0 : fprintf (sched_dump,
6721 : ";;\t\tfailing schedule due to excessive stages\n");
6722 0 : goto end_schedule;
6723 : }
6724 0 : if (modulo_n_insns == modulo_insns_scheduled
6725 0 : && stage > modulo_last_stage)
6726 : {
6727 0 : if (sched_verbose >= 2)
6728 0 : fprintf (sched_dump,
6729 : ";;\t\tfound kernel after %d stages, II %d\n",
6730 : stage, modulo_ii);
6731 0 : ls.modulo_epilogue = true;
6732 : }
6733 : }
6734 :
6735 35708100 : prune_ready_list (temp_state, true, false, ls.modulo_epilogue);
6736 35708100 : if (ready.n_ready == 0)
6737 3780 : continue;
6738 35704320 : if (must_backtrack)
6739 0 : goto do_backtrack;
6740 :
6741 35704320 : ls.shadows_only_p = false;
6742 35704320 : cycle_issued_insns = 0;
6743 35704320 : ls.can_issue_more = issue_rate;
6744 158522732 : for (;;)
6745 : {
6746 97113526 : rtx_insn *insn;
6747 97113526 : int cost;
6748 97113526 : bool asm_p;
6749 :
6750 97113526 : if (sort_p && ready.n_ready > 0)
6751 : {
6752 : /* Sort the ready list based on priority. This must be
6753 : done every iteration through the loop, as schedule_insn
6754 : may have readied additional insns that will not be
6755 : sorted correctly. */
6756 63422994 : ready_sort (&ready);
6757 :
6758 63422994 : if (sched_verbose >= 2)
6759 : {
6760 0 : fprintf (sched_dump,
6761 : ";;\t\tReady list after ready_sort: ");
6762 0 : debug_ready_list (&ready);
6763 : }
6764 : }
6765 :
6766 : /* We don't want md sched reorder to even see debug isns, so put
6767 : them out right away. */
6768 63422994 : if (ready.n_ready && DEBUG_INSN_P (ready_element (&ready, 0))
6769 103764986 : && (*current_sched_info->schedule_more_p) ())
6770 : {
6771 59504102 : while (ready.n_ready && DEBUG_INSN_P (ready_element (&ready, 0)))
6772 : {
6773 52852642 : rtx_insn *insn = ready_remove_first (&ready);
6774 52852642 : gcc_assert (DEBUG_INSN_P (insn));
6775 52852642 : (*current_sched_info->begin_schedule_ready) (insn);
6776 52852642 : scheduled_insns.safe_push (insn);
6777 52852642 : last_scheduled_insn = insn;
6778 52852642 : advance = schedule_insn (insn);
6779 52852642 : gcc_assert (advance == 0);
6780 52852642 : if (ready.n_ready > 0)
6781 50975857 : ready_sort (&ready);
6782 : }
6783 : }
6784 :
6785 97113526 : if (ls.first_cycle_insn_p && !ready.n_ready)
6786 : break;
6787 :
6788 97112324 : resume_after_backtrack:
6789 : /* Allow the target to reorder the list, typically for
6790 : better instruction bundling. */
6791 97112324 : if (sort_p
6792 97112324 : && (ready.n_ready == 0
6793 61546209 : || !SCHED_GROUP_P (ready_element (&ready, 0))))
6794 : {
6795 92895215 : if (ls.first_cycle_insn_p && targetm.sched.reorder)
6796 35682481 : ls.can_issue_more
6797 35682481 : = targetm.sched.reorder (sched_dump, sched_verbose,
6798 : ready_lastpos (&ready),
6799 : &ready.n_ready, clock_var);
6800 57212734 : else if (!ls.first_cycle_insn_p && targetm.sched.reorder2)
6801 0 : ls.can_issue_more
6802 0 : = targetm.sched.reorder2 (sched_dump, sched_verbose,
6803 0 : ready.n_ready
6804 0 : ? ready_lastpos (&ready) : NULL,
6805 : &ready.n_ready, clock_var);
6806 : }
6807 :
6808 97112324 : restart_choose_ready:
6809 97112324 : if (sched_verbose >= 2)
6810 : {
6811 0 : fprintf (sched_dump, ";;\tReady list (t = %3d): ",
6812 : clock_var);
6813 0 : debug_ready_list (&ready);
6814 0 : if (sched_pressure == SCHED_PRESSURE_WEIGHTED)
6815 0 : print_curr_reg_pressure ();
6816 : }
6817 :
6818 97112324 : if (ready.n_ready == 0
6819 35566115 : && ls.can_issue_more
6820 33762499 : && reload_completed)
6821 : {
6822 : /* Allow scheduling insns directly from the queue in case
6823 : there's nothing better to do (ready list is empty) but
6824 : there are still vacant dispatch slots in the current cycle. */
6825 33759567 : if (sched_verbose >= 6)
6826 0 : fprintf (sched_dump,";;\t\tSecond chance\n");
6827 33759567 : memcpy (temp_state, curr_state, dfa_state_size);
6828 33759567 : if (early_queue_to_ready (temp_state, &ready))
6829 4 : ready_sort (&ready);
6830 : }
6831 :
6832 97112324 : if (ready.n_ready == 0
6833 61546213 : || !ls.can_issue_more
6834 61463046 : || state_dead_lock_p (curr_state)
6835 158575370 : || !(*current_sched_info->schedule_more_p) ())
6836 : break;
6837 :
6838 : /* Select and remove the insn from the ready list. */
6839 61463046 : if (sort_p)
6840 : {
6841 61463046 : int res;
6842 :
6843 61463046 : insn = NULL;
6844 61463046 : res = choose_ready (&ready, ls.first_cycle_insn_p, &insn);
6845 :
6846 61463046 : if (res < 0)
6847 : /* Finish cycle. */
6848 : break;
6849 61463046 : if (res > 0)
6850 0 : goto restart_choose_ready;
6851 :
6852 61463046 : gcc_assert (insn != NULL_RTX);
6853 : }
6854 : else
6855 0 : insn = ready_remove_first (&ready);
6856 :
6857 61463046 : if (sched_pressure != SCHED_PRESSURE_NONE
6858 61463046 : && INSN_TICK (insn) > clock_var)
6859 : {
6860 2215 : ready_add (&ready, insn, true);
6861 2215 : advance = 1;
6862 2215 : break;
6863 : }
6864 :
6865 61460831 : if (targetm.sched.dfa_new_cycle
6866 61460831 : && targetm.sched.dfa_new_cycle (sched_dump, sched_verbose,
6867 : insn, last_clock_var,
6868 : clock_var, &sort_p))
6869 : /* SORT_P is used by the target to override sorting
6870 : of the ready list. This is needed when the target
6871 : has modified its internal structures expecting that
6872 : the insn will be issued next. As we need the insn
6873 : to have the highest priority (so it will be returned by
6874 : the ready_remove_first call above), we invoke
6875 : ready_add (&ready, insn, true).
6876 : But, still, there is one issue: INSN can be later
6877 : discarded by scheduler's front end through
6878 : current_sched_info->can_schedule_ready_p, hence, won't
6879 : be issued next. */
6880 : {
6881 0 : ready_add (&ready, insn, true);
6882 0 : break;
6883 : }
6884 :
6885 61460831 : sort_p = true;
6886 :
6887 61460831 : if (current_sched_info->can_schedule_ready_p
6888 61460831 : && ! (*current_sched_info->can_schedule_ready_p) (insn))
6889 : /* We normally get here only if we don't want to move
6890 : insn from the split block. */
6891 : {
6892 0 : TODO_SPEC (insn) = DEP_POSTPONED;
6893 0 : goto restart_choose_ready;
6894 : }
6895 :
6896 61460831 : if (delay_htab)
6897 : {
6898 : /* If this insn is the first part of a delay-slot pair, record a
6899 : backtrack point. */
6900 0 : struct delay_pair *delay_entry;
6901 0 : delay_entry
6902 0 : = delay_htab->find_with_hash (insn, htab_hash_pointer (insn));
6903 0 : if (delay_entry)
6904 : {
6905 0 : save_backtrack_point (delay_entry, ls);
6906 0 : if (sched_verbose >= 2)
6907 0 : fprintf (sched_dump, ";;\t\tsaving backtrack point\n");
6908 : }
6909 : }
6910 :
6911 : /* DECISION is made. */
6912 :
6913 61460831 : if (modulo_ii > 0 && INSN_UID (insn) < modulo_iter0_max_uid)
6914 : {
6915 0 : modulo_insns_scheduled++;
6916 0 : modulo_last_stage = clock_var / modulo_ii;
6917 : }
6918 61460831 : if (TODO_SPEC (insn) & SPECULATIVE)
6919 0 : generate_recovery_code (insn);
6920 :
6921 61460831 : if (targetm.sched.dispatch (NULL, IS_DISPATCH_ON))
6922 14 : targetm.sched.dispatch_do (insn, ADD_TO_DISPATCH_WINDOW);
6923 :
6924 : /* Update counters, etc in the scheduler's front end. */
6925 61460831 : (*current_sched_info->begin_schedule_ready) (insn);
6926 61460831 : scheduled_insns.safe_push (insn);
6927 61460831 : gcc_assert (NONDEBUG_INSN_P (insn));
6928 61460831 : last_nondebug_scheduled_insn = last_scheduled_insn = insn;
6929 :
6930 61460831 : if (recog_memoized (insn) >= 0)
6931 : {
6932 60609016 : memcpy (temp_state, curr_state, dfa_state_size);
6933 60609016 : cost = state_transition (curr_state, insn);
6934 60609016 : if (sched_pressure != SCHED_PRESSURE_WEIGHTED && !sched_fusion)
6935 60605306 : gcc_assert (cost < 0);
6936 60609016 : if (memcmp (temp_state, curr_state, dfa_state_size) != 0)
6937 60194046 : cycle_issued_insns++;
6938 : asm_p = false;
6939 : }
6940 : else
6941 851815 : asm_p = (GET_CODE (PATTERN (insn)) == ASM_INPUT
6942 851815 : || asm_noperands (PATTERN (insn)) >= 0);
6943 :
6944 61460831 : if (targetm.sched.variable_issue)
6945 0 : ls.can_issue_more =
6946 0 : targetm.sched.variable_issue (sched_dump, sched_verbose,
6947 : insn, ls.can_issue_more);
6948 : /* A naked CLOBBER or USE generates no instruction, so do
6949 : not count them against the issue rate. */
6950 61460831 : else if (GET_CODE (PATTERN (insn)) != USE
6951 61460831 : && GET_CODE (PATTERN (insn)) != CLOBBER)
6952 60660641 : ls.can_issue_more--;
6953 61460831 : advance = schedule_insn (insn);
6954 :
6955 61460831 : if (SHADOW_P (insn))
6956 0 : ls.shadows_only_p = true;
6957 :
6958 : /* After issuing an asm insn we should start a new cycle. */
6959 61460831 : if (advance == 0 && asm_p)
6960 51625 : advance = 1;
6961 :
6962 61460831 : if (must_backtrack)
6963 : break;
6964 :
6965 61460831 : if (advance != 0)
6966 : break;
6967 :
6968 61409206 : ls.first_cycle_insn_p = false;
6969 61409206 : if (ready.n_ready > 0)
6970 34985614 : prune_ready_list (temp_state, false, ls.shadows_only_p,
6971 : ls.modulo_epilogue);
6972 61409206 : }
6973 :
6974 35704320 : do_backtrack:
6975 35704320 : if (!must_backtrack)
6976 35950348 : for (i = 0; i < ready.n_ready; i++)
6977 : {
6978 246028 : rtx_insn *insn = ready_element (&ready, i);
6979 246028 : if (INSN_EXACT_TICK (insn) == clock_var)
6980 : {
6981 0 : must_backtrack = true;
6982 0 : clock_var++;
6983 0 : break;
6984 : }
6985 : }
6986 35704320 : if (must_backtrack && modulo_ii > 0)
6987 : {
6988 0 : if (modulo_backtracks_left == 0)
6989 0 : goto end_schedule;
6990 0 : modulo_backtracks_left--;
6991 : }
6992 35704320 : while (must_backtrack)
6993 : {
6994 0 : struct haifa_saved_data *failed;
6995 0 : rtx_insn *failed_insn;
6996 :
6997 0 : must_backtrack = false;
6998 0 : failed = verify_shadows ();
6999 0 : gcc_assert (failed);
7000 :
7001 0 : failed_insn = failed->delay_pair->i1;
7002 : /* Clear these queues. */
7003 0 : perform_replacements_new_cycle ();
7004 0 : toggle_cancelled_flags (false);
7005 0 : unschedule_insns_until (failed_insn);
7006 0 : while (failed != backtrack_queue)
7007 0 : free_topmost_backtrack_point (true);
7008 0 : restore_last_backtrack_point (&ls);
7009 0 : if (sched_verbose >= 2)
7010 0 : fprintf (sched_dump, ";;\t\trewind to cycle %d\n", clock_var);
7011 : /* Delay by at least a cycle. This could cause additional
7012 : backtracking. */
7013 0 : queue_insn (failed_insn, 1, "backtracked");
7014 0 : advance = 0;
7015 0 : if (must_backtrack)
7016 0 : continue;
7017 0 : if (ready.n_ready > 0)
7018 0 : goto resume_after_backtrack;
7019 : else
7020 : {
7021 0 : if (clock_var == 0 && ls.first_cycle_insn_p)
7022 0 : goto end_schedule;
7023 : advance = 1;
7024 : break;
7025 : }
7026 : }
7027 35704320 : ls.first_cycle_insn_p = true;
7028 : }
7029 10477678 : if (ls.modulo_epilogue)
7030 0 : success = true;
7031 10477678 : end_schedule:
7032 10477678 : if (!ls.first_cycle_insn_p || advance)
7033 942 : advance_one_cycle ();
7034 10477678 : perform_replacements_new_cycle ();
7035 10477678 : if (modulo_ii > 0)
7036 : {
7037 : /* Once again, debug insn suckiness: they can be on the ready list
7038 : even if they have unresolved dependencies. To make our view
7039 : of the world consistent, remove such "ready" insns. */
7040 0 : restart_debug_insn_loop:
7041 0 : for (i = ready.n_ready - 1; i >= 0; i--)
7042 : {
7043 0 : rtx_insn *x;
7044 :
7045 0 : x = ready_element (&ready, i);
7046 0 : if (DEPS_LIST_FIRST (INSN_HARD_BACK_DEPS (x)) != NULL
7047 0 : || DEPS_LIST_FIRST (INSN_SPEC_BACK_DEPS (x)) != NULL)
7048 : {
7049 0 : ready_remove (&ready, i);
7050 0 : goto restart_debug_insn_loop;
7051 : }
7052 : }
7053 0 : for (i = ready.n_ready - 1; i >= 0; i--)
7054 : {
7055 0 : rtx_insn *x;
7056 :
7057 0 : x = ready_element (&ready, i);
7058 0 : resolve_dependencies (x);
7059 : }
7060 0 : for (i = 0; i <= max_insn_queue_index; i++)
7061 : {
7062 : rtx_insn_list *link;
7063 0 : while ((link = insn_queue[i]) != NULL)
7064 : {
7065 0 : rtx_insn *x = link->insn ();
7066 0 : insn_queue[i] = link->next ();
7067 0 : QUEUE_INDEX (x) = QUEUE_NOWHERE;
7068 0 : free_INSN_LIST_node (link);
7069 0 : resolve_dependencies (x);
7070 : }
7071 : }
7072 : }
7073 :
7074 10477678 : if (!success)
7075 0 : undo_all_replacements ();
7076 :
7077 : /* Debug info. */
7078 10477678 : if (sched_verbose)
7079 : {
7080 190 : fprintf (sched_dump, ";;\tReady list (final): ");
7081 190 : debug_ready_list (&ready);
7082 : }
7083 :
7084 10477678 : if (modulo_ii == 0 && current_sched_info->queue_must_finish_empty)
7085 : /* Sanity check -- queue must be empty now. Meaningless if region has
7086 : multiple bbs. */
7087 10477606 : gcc_assert (!q_size && !ready.n_ready && !ready.n_debug);
7088 72 : else if (modulo_ii == 0)
7089 : {
7090 : /* We must maintain QUEUE_INDEX between blocks in region. */
7091 84 : for (i = ready.n_ready - 1; i >= 0; i--)
7092 : {
7093 12 : rtx_insn *x;
7094 :
7095 12 : x = ready_element (&ready, i);
7096 12 : QUEUE_INDEX (x) = QUEUE_NOWHERE;
7097 12 : TODO_SPEC (x) = HARD_DEP;
7098 : }
7099 :
7100 72 : if (q_size)
7101 0 : for (i = 0; i <= max_insn_queue_index; i++)
7102 : {
7103 0 : rtx_insn_list *link;
7104 0 : for (link = insn_queue[i]; link; link = link->next ())
7105 : {
7106 0 : rtx_insn *x;
7107 :
7108 0 : x = link->insn ();
7109 0 : QUEUE_INDEX (x) = QUEUE_NOWHERE;
7110 0 : TODO_SPEC (x) = HARD_DEP;
7111 : }
7112 0 : free_INSN_LIST_list (&insn_queue[i]);
7113 : }
7114 : }
7115 :
7116 10477678 : if (sched_pressure == SCHED_PRESSURE_MODEL)
7117 0 : model_end_schedule ();
7118 :
7119 10477678 : if (success)
7120 : {
7121 10477678 : commit_schedule (prev_head, tail, target_bb);
7122 10477678 : if (sched_verbose)
7123 190 : fprintf (sched_dump, ";; total time = %d\n", clock_var);
7124 : }
7125 : else
7126 0 : last_scheduled_insn = tail;
7127 :
7128 10477678 : scheduled_insns.truncate (0);
7129 :
7130 10477678 : if (!current_sched_info->queue_must_finish_empty
7131 10477606 : || haifa_recovery_bb_recently_added_p)
7132 : {
7133 : /* INSN_TICK (minimum clock tick at which the insn becomes
7134 : ready) may be not correct for the insn in the subsequent
7135 : blocks of the region. We should use a correct value of
7136 : `clock_var' or modify INSN_TICK. It is better to keep
7137 : clock_var value equal to 0 at the start of a basic block.
7138 : Therefore we modify INSN_TICK here. */
7139 72 : fix_inter_tick (NEXT_INSN (prev_head), last_scheduled_insn);
7140 : }
7141 :
7142 10477678 : if (targetm.sched.finish)
7143 : {
7144 0 : targetm.sched.finish (sched_dump, sched_verbose);
7145 : /* Target might have added some instructions to the scheduled block
7146 : in its md_finish () hook. These new insns don't have any data
7147 : initialized and to identify them we extend h_i_d so that they'll
7148 : get zero luids. */
7149 0 : sched_extend_luids ();
7150 : }
7151 :
7152 : /* Update head/tail boundaries. */
7153 10477678 : head = NEXT_INSN (prev_head);
7154 10477678 : tail = last_scheduled_insn;
7155 :
7156 10477678 : if (sched_verbose)
7157 : {
7158 190 : fprintf (sched_dump, ";; new head = %d\n;; new tail = %d\n",
7159 190 : INSN_UID (head), INSN_UID (tail));
7160 :
7161 190 : if (sched_verbose >= 2)
7162 : {
7163 0 : dump_insn_stream (head, tail);
7164 0 : print_rank_for_schedule_stats (";; TOTAL ", &rank_for_schedule_stats,
7165 : NULL);
7166 : }
7167 :
7168 190 : fprintf (sched_dump, "\n");
7169 : }
7170 :
7171 10477678 : head = restore_other_notes (head, NULL);
7172 :
7173 10477678 : current_sched_info->head = head;
7174 10477678 : current_sched_info->tail = tail;
7175 :
7176 10477678 : free_backtrack_queue ();
7177 :
7178 10477678 : return success;
7179 : }
7180 :
7181 : /* Set_priorities: compute priority of each insn in the block. */
7182 :
7183 : int
7184 10478647 : set_priorities (rtx_insn *head, rtx_insn *tail)
7185 : {
7186 10478647 : rtx_insn *insn;
7187 10478647 : int n_insn;
7188 10478647 : int sched_max_insns_priority =
7189 10478647 : current_sched_info->sched_max_insns_priority;
7190 10478647 : rtx_insn *prev_head;
7191 :
7192 10478647 : if (head == tail && ! INSN_P (head))
7193 0 : gcc_unreachable ();
7194 :
7195 10478647 : n_insn = 0;
7196 :
7197 10478647 : prev_head = PREV_INSN (head);
7198 141194172 : for (insn = tail; insn != prev_head; insn = PREV_INSN (insn))
7199 : {
7200 120236878 : if (!INSN_P (insn))
7201 5919048 : continue;
7202 :
7203 114317830 : n_insn++;
7204 114317830 : (void) priority (insn);
7205 :
7206 114317830 : gcc_assert (INSN_PRIORITY_KNOWN (insn));
7207 :
7208 114317830 : sched_max_insns_priority = MAX (sched_max_insns_priority,
7209 : INSN_PRIORITY (insn));
7210 : }
7211 :
7212 10478647 : current_sched_info->sched_max_insns_priority = sched_max_insns_priority;
7213 :
7214 10478647 : return n_insn;
7215 : }
7216 :
7217 : /* Set sched_dump and sched_verbose for the desired debugging output. */
7218 : void
7219 983586 : setup_sched_dump (void)
7220 : {
7221 983586 : sched_verbose = sched_verbose_param;
7222 983586 : sched_dump = dump_file;
7223 983586 : if (!dump_file)
7224 983549 : sched_verbose = 0;
7225 983586 : }
7226 :
7227 : /* Allocate data for register pressure sensitive scheduling. */
7228 : static void
7229 983586 : alloc_global_sched_pressure_data (void)
7230 : {
7231 983586 : if (sched_pressure != SCHED_PRESSURE_NONE)
7232 : {
7233 57 : int i, max_regno = max_reg_num ();
7234 :
7235 57 : if (sched_dump != NULL)
7236 : /* We need info about pseudos for rtl dumps about pseudo
7237 : classes and costs. */
7238 0 : regstat_init_n_sets_and_refs ();
7239 57 : ira_set_pseudo_classes (true, sched_verbose ? sched_dump : NULL);
7240 57 : sched_regno_pressure_class
7241 57 : = (enum reg_class *) xmalloc (max_regno * sizeof (enum reg_class));
7242 8732 : for (i = 0; i < max_regno; i++)
7243 17350 : sched_regno_pressure_class[i]
7244 8675 : = (i < FIRST_PSEUDO_REGISTER
7245 8675 : ? ira_pressure_class_translate[REGNO_REG_CLASS (i)]
7246 3317 : : ira_pressure_class_translate[reg_allocno_class (i)]);
7247 57 : curr_reg_live = BITMAP_ALLOC (NULL);
7248 57 : if (sched_pressure == SCHED_PRESSURE_WEIGHTED)
7249 : {
7250 57 : saved_reg_live = BITMAP_ALLOC (NULL);
7251 57 : region_ref_regs = BITMAP_ALLOC (NULL);
7252 : }
7253 57 : if (sched_pressure == SCHED_PRESSURE_MODEL)
7254 0 : tmp_bitmap = BITMAP_ALLOC (NULL);
7255 :
7256 : /* Calculate number of CALL_SAVED_REGS and FIXED_REGS in register classes
7257 : that we calculate register pressure for. */
7258 287 : for (int c = 0; c < ira_pressure_classes_num; ++c)
7259 : {
7260 230 : enum reg_class cl = ira_pressure_classes[c];
7261 :
7262 230 : call_saved_regs_num[cl] = 0;
7263 230 : fixed_regs_num[cl] = 0;
7264 :
7265 2905 : for (int i = 0; i < ira_class_hard_regs_num[cl]; ++i)
7266 : {
7267 2675 : unsigned int regno = ira_class_hard_regs[cl][i];
7268 2675 : if (fixed_regs[regno])
7269 0 : ++fixed_regs_num[cl];
7270 2675 : else if (!crtl->abi->clobbers_full_reg_p (regno))
7271 332 : ++call_saved_regs_num[cl];
7272 : }
7273 : }
7274 : }
7275 983586 : }
7276 :
7277 : /* Free data for register pressure sensitive scheduling. Also called
7278 : from schedule_region when stopping sched-pressure early. */
7279 : void
7280 983586 : free_global_sched_pressure_data (void)
7281 : {
7282 983586 : if (sched_pressure != SCHED_PRESSURE_NONE)
7283 : {
7284 57 : if (regstat_n_sets_and_refs != NULL)
7285 0 : regstat_free_n_sets_and_refs ();
7286 57 : if (sched_pressure == SCHED_PRESSURE_WEIGHTED)
7287 : {
7288 57 : BITMAP_FREE (region_ref_regs);
7289 57 : BITMAP_FREE (saved_reg_live);
7290 : }
7291 57 : if (sched_pressure == SCHED_PRESSURE_MODEL)
7292 0 : BITMAP_FREE (tmp_bitmap);
7293 57 : BITMAP_FREE (curr_reg_live);
7294 57 : free (sched_regno_pressure_class);
7295 : }
7296 983586 : }
7297 :
7298 : /* Initialize some global state for the scheduler. This function works
7299 : with the common data shared between all the schedulers. It is called
7300 : from the scheduler specific initialization routine. */
7301 :
7302 : void
7303 983586 : sched_init (void)
7304 : {
7305 983586 : if (targetm.sched.dispatch (NULL, IS_DISPATCH_ON))
7306 3 : targetm.sched.dispatch_do (NULL, DISPATCH_INIT);
7307 :
7308 983586 : if (live_range_shrinkage_p)
7309 35 : sched_pressure = SCHED_PRESSURE_WEIGHTED;
7310 983551 : else if (flag_sched_pressure
7311 50 : && !reload_completed
7312 22 : && common_sched_info->sched_pass_id == SCHED_RGN_PASS)
7313 22 : sched_pressure = ((enum sched_pressure_algorithm)
7314 22 : param_sched_pressure_algorithm);
7315 : else
7316 983529 : sched_pressure = SCHED_PRESSURE_NONE;
7317 :
7318 983586 : if (sched_pressure != SCHED_PRESSURE_NONE)
7319 57 : ira_setup_eliminable_regset ();
7320 :
7321 : /* Initialize SPEC_INFO. */
7322 983586 : if (targetm.sched.set_sched_flags)
7323 : {
7324 0 : spec_info = &spec_info_var;
7325 0 : targetm.sched.set_sched_flags (spec_info);
7326 :
7327 0 : if (spec_info->mask != 0)
7328 : {
7329 0 : spec_info->data_weakness_cutoff
7330 0 : = (param_sched_spec_prob_cutoff * MAX_DEP_WEAK) / 100;
7331 0 : spec_info->control_weakness_cutoff
7332 0 : = (param_sched_spec_prob_cutoff * REG_BR_PROB_BASE) / 100;
7333 : }
7334 : else
7335 : /* So we won't read anything accidentally. */
7336 0 : spec_info = NULL;
7337 :
7338 : }
7339 : else
7340 : /* So we won't read anything accidentally. */
7341 983586 : spec_info = 0;
7342 :
7343 : /* Initialize issue_rate. */
7344 983586 : if (targetm.sched.issue_rate)
7345 983586 : issue_rate = targetm.sched.issue_rate ();
7346 : else
7347 0 : issue_rate = 1;
7348 :
7349 983586 : if (targetm.sched.first_cycle_multipass_dfa_lookahead
7350 : /* Don't use max_issue with reg_pressure scheduling. Multipass
7351 : scheduling and reg_pressure scheduling undo each other's decisions. */
7352 983586 : && sched_pressure == SCHED_PRESSURE_NONE)
7353 983529 : dfa_lookahead = targetm.sched.first_cycle_multipass_dfa_lookahead ();
7354 : else
7355 57 : dfa_lookahead = 0;
7356 :
7357 : /* Set to "0" so that we recalculate. */
7358 983586 : max_lookahead_tries = 0;
7359 :
7360 983586 : if (targetm.sched.init_dfa_pre_cycle_insn)
7361 0 : targetm.sched.init_dfa_pre_cycle_insn ();
7362 :
7363 983586 : if (targetm.sched.init_dfa_post_cycle_insn)
7364 0 : targetm.sched.init_dfa_post_cycle_insn ();
7365 :
7366 983586 : dfa_start ();
7367 983586 : dfa_state_size = state_size ();
7368 :
7369 983586 : init_alias_analysis ();
7370 :
7371 983586 : if (!sched_no_dce)
7372 983586 : df_set_flags (DF_LR_RUN_DCE);
7373 983586 : df_note_add_problem ();
7374 :
7375 : /* More problems needed for interloop dep calculation in SMS. */
7376 983586 : if (common_sched_info->sched_pass_id == SCHED_SMS_PASS)
7377 : {
7378 267 : df_rd_add_problem ();
7379 267 : df_chain_add_problem (DF_DU_CHAIN + DF_UD_CHAIN);
7380 : }
7381 :
7382 983586 : df_analyze ();
7383 :
7384 : /* Do not run DCE after reload, as this can kill nops inserted
7385 : by bundling. */
7386 983586 : if (reload_completed)
7387 983088 : df_clear_flags (DF_LR_RUN_DCE);
7388 :
7389 983586 : regstat_compute_calls_crossed ();
7390 :
7391 983586 : if (targetm.sched.init_global)
7392 983586 : targetm.sched.init_global (sched_dump, sched_verbose, get_max_uid () + 1);
7393 :
7394 983586 : alloc_global_sched_pressure_data ();
7395 :
7396 983586 : curr_state = xmalloc (dfa_state_size);
7397 983586 : }
7398 :
7399 : static void haifa_init_only_bb (basic_block, basic_block);
7400 :
7401 : /* Initialize data structures specific to the Haifa scheduler. */
7402 : void
7403 983455 : haifa_sched_init (void)
7404 : {
7405 983455 : setup_sched_dump ();
7406 983455 : sched_init ();
7407 :
7408 983455 : scheduled_insns.create (0);
7409 :
7410 983455 : if (spec_info != NULL)
7411 : {
7412 0 : sched_deps_info->use_deps_list = 1;
7413 0 : sched_deps_info->generate_spec_deps = 1;
7414 : }
7415 :
7416 : /* Initialize luids, dependency caches, target and h_i_d for the
7417 : whole function. */
7418 983455 : {
7419 983455 : sched_init_bbs ();
7420 :
7421 983455 : auto_vec<basic_block> bbs (n_basic_blocks_for_fn (cfun));
7422 983455 : basic_block bb;
7423 11481822 : FOR_EACH_BB_FN (bb, cfun)
7424 10498367 : bbs.quick_push (bb);
7425 983455 : sched_init_luids (bbs);
7426 983455 : sched_deps_init (true);
7427 983455 : sched_extend_target ();
7428 983455 : haifa_init_h_i_d (bbs);
7429 983455 : }
7430 :
7431 983455 : sched_init_only_bb = haifa_init_only_bb;
7432 983455 : sched_split_block = sched_split_block_1;
7433 983455 : sched_create_empty_bb = sched_create_empty_bb_1;
7434 983455 : haifa_recovery_bb_ever_added_p = false;
7435 :
7436 983455 : nr_begin_data = nr_begin_control = nr_be_in_data = nr_be_in_control = 0;
7437 983455 : before_recovery = 0;
7438 983455 : after_recovery = 0;
7439 :
7440 983455 : modulo_ii = 0;
7441 983455 : }
7442 :
7443 : /* Finish work with the data specific to the Haifa scheduler. */
7444 : void
7445 983455 : haifa_sched_finish (void)
7446 : {
7447 983455 : sched_create_empty_bb = NULL;
7448 983455 : sched_split_block = NULL;
7449 983455 : sched_init_only_bb = NULL;
7450 :
7451 983455 : if (spec_info && spec_info->dump)
7452 : {
7453 0 : char c = reload_completed ? 'a' : 'b';
7454 :
7455 0 : fprintf (spec_info->dump,
7456 : ";; %s:\n", current_function_name ());
7457 :
7458 0 : fprintf (spec_info->dump,
7459 : ";; Procedure %cr-begin-data-spec motions == %d\n",
7460 : c, nr_begin_data);
7461 0 : fprintf (spec_info->dump,
7462 : ";; Procedure %cr-be-in-data-spec motions == %d\n",
7463 : c, nr_be_in_data);
7464 0 : fprintf (spec_info->dump,
7465 : ";; Procedure %cr-begin-control-spec motions == %d\n",
7466 : c, nr_begin_control);
7467 0 : fprintf (spec_info->dump,
7468 : ";; Procedure %cr-be-in-control-spec motions == %d\n",
7469 : c, nr_be_in_control);
7470 : }
7471 :
7472 983455 : scheduled_insns.release ();
7473 :
7474 : /* Finalize h_i_d, dependency caches, and luids for the whole
7475 : function. Target will be finalized in md_global_finish (). */
7476 983455 : sched_deps_finish ();
7477 983455 : sched_finish_luids ();
7478 983455 : current_sched_info = NULL;
7479 983455 : insn_queue = NULL;
7480 983455 : sched_finish ();
7481 983455 : }
7482 :
7483 : /* Free global data used during insn scheduling. This function works with
7484 : the common data shared between the schedulers. */
7485 :
7486 : void
7487 983586 : sched_finish (void)
7488 : {
7489 983586 : haifa_finish_h_i_d ();
7490 983586 : free_global_sched_pressure_data ();
7491 983586 : free (curr_state);
7492 :
7493 983586 : if (targetm.sched.finish_global)
7494 0 : targetm.sched.finish_global (sched_dump, sched_verbose);
7495 :
7496 983586 : end_alias_analysis ();
7497 :
7498 983586 : regstat_free_calls_crossed ();
7499 :
7500 983586 : dfa_finish ();
7501 983586 : }
7502 :
7503 : /* Free all delay_pair structures that were recorded. */
7504 : void
7505 0 : free_delay_pairs (void)
7506 : {
7507 0 : if (delay_htab)
7508 : {
7509 0 : delay_htab->empty ();
7510 0 : delay_htab_i2->empty ();
7511 : }
7512 0 : }
7513 :
7514 : /* Fix INSN_TICKs of the instructions in the current block as well as
7515 : INSN_TICKs of their dependents.
7516 : HEAD and TAIL are the begin and the end of the current scheduled block. */
7517 : static void
7518 72 : fix_inter_tick (rtx_insn *head, rtx_insn *tail)
7519 : {
7520 : /* Set of instructions with corrected INSN_TICK. */
7521 72 : auto_bitmap processed;
7522 : /* ??? It is doubtful if we should assume that cycle advance happens on
7523 : basic block boundaries. Basically insns that are unconditionally ready
7524 : on the start of the block are more preferable then those which have
7525 : a one cycle dependency over insn from the previous block. */
7526 72 : int next_clock = clock_var + 1;
7527 :
7528 : /* Iterates over scheduled instructions and fix their INSN_TICKs and
7529 : INSN_TICKs of dependent instructions, so that INSN_TICKs are consistent
7530 : across different blocks. */
7531 1287 : for (tail = NEXT_INSN (tail); head != tail; head = NEXT_INSN (head))
7532 : {
7533 1143 : if (INSN_P (head))
7534 : {
7535 1143 : int tick;
7536 1143 : sd_iterator_def sd_it;
7537 1143 : dep_t dep;
7538 :
7539 1143 : tick = INSN_TICK (head);
7540 1143 : gcc_assert (tick >= MIN_TICK);
7541 :
7542 : /* Fix INSN_TICK of instruction from just scheduled block. */
7543 1143 : if (bitmap_set_bit (processed, INSN_LUID (head)))
7544 : {
7545 392 : tick -= next_clock;
7546 :
7547 392 : if (tick < MIN_TICK)
7548 : tick = MIN_TICK;
7549 :
7550 392 : INSN_TICK (head) = tick;
7551 : }
7552 :
7553 1143 : if (DEBUG_INSN_P (head))
7554 428 : continue;
7555 :
7556 2995 : FOR_EACH_DEP (head, SD_LIST_RES_FORW, sd_it, dep)
7557 : {
7558 2280 : rtx_insn *next;
7559 :
7560 2280 : next = DEP_CON (dep);
7561 2280 : tick = INSN_TICK (next);
7562 :
7563 2280 : if (tick != INVALID_TICK
7564 : /* If NEXT has its INSN_TICK calculated, fix it.
7565 : If not - it will be properly calculated from
7566 : scratch later in fix_tick_ready. */
7567 2280 : && bitmap_set_bit (processed, INSN_LUID (next)))
7568 : {
7569 763 : tick -= next_clock;
7570 :
7571 763 : if (tick < MIN_TICK)
7572 : tick = MIN_TICK;
7573 :
7574 763 : if (tick > INTER_TICK (next))
7575 763 : INTER_TICK (next) = tick;
7576 : else
7577 : tick = INTER_TICK (next);
7578 :
7579 763 : INSN_TICK (next) = tick;
7580 : }
7581 : }
7582 : }
7583 : }
7584 72 : }
7585 :
7586 : /* Check if NEXT is ready to be added to the ready or queue list.
7587 : If "yes", add it to the proper list.
7588 : Returns:
7589 : -1 - is not ready yet,
7590 : 0 - added to the ready list,
7591 : 0 < N - queued for N cycles. */
7592 : int
7593 325168464 : try_ready (rtx_insn *next)
7594 : {
7595 325168464 : ds_t old_ts, new_ts;
7596 :
7597 325168464 : old_ts = TODO_SPEC (next);
7598 :
7599 325168464 : gcc_assert (!(old_ts & ~(SPECULATIVE | HARD_DEP | DEP_CONTROL | DEP_POSTPONED))
7600 : && (old_ts == HARD_DEP
7601 : || old_ts == DEP_POSTPONED
7602 : || (old_ts & SPECULATIVE)
7603 : || old_ts == DEP_CONTROL));
7604 :
7605 325168464 : new_ts = recompute_todo_spec (next, false);
7606 :
7607 325168464 : if (new_ts & (HARD_DEP | DEP_POSTPONED))
7608 210854943 : gcc_assert (new_ts == old_ts
7609 : && QUEUE_INDEX (next) == QUEUE_NOWHERE);
7610 114313521 : else if (current_sched_info->new_ready)
7611 114312262 : new_ts = current_sched_info->new_ready (next, new_ts);
7612 :
7613 : /* * if !(old_ts & SPECULATIVE) (e.g. HARD_DEP or 0), then insn might
7614 : have its original pattern or changed (speculative) one. This is due
7615 : to changing ebb in region scheduling.
7616 : * But if (old_ts & SPECULATIVE), then we are pretty sure that insn
7617 : has speculative pattern.
7618 :
7619 : We can't assert (!(new_ts & HARD_DEP) || new_ts == old_ts) here because
7620 : control-speculative NEXT could have been discarded by sched-rgn.cc
7621 : (the same case as when discarded by can_schedule_ready_p ()). */
7622 :
7623 325168464 : if ((new_ts & SPECULATIVE)
7624 : /* If (old_ts == new_ts), then (old_ts & SPECULATIVE) and we don't
7625 : need to change anything. */
7626 0 : && new_ts != old_ts)
7627 : {
7628 0 : int res;
7629 0 : rtx new_pat;
7630 :
7631 0 : gcc_assert ((new_ts & SPECULATIVE) && !(new_ts & ~SPECULATIVE));
7632 :
7633 0 : res = haifa_speculate_insn (next, new_ts, &new_pat);
7634 :
7635 0 : switch (res)
7636 : {
7637 : case -1:
7638 : /* It would be nice to change DEP_STATUS of all dependences,
7639 : which have ((DEP_STATUS & SPECULATIVE) == new_ts) to HARD_DEP,
7640 : so we won't reanalyze anything. */
7641 : new_ts = HARD_DEP;
7642 : break;
7643 :
7644 0 : case 0:
7645 : /* We follow the rule, that every speculative insn
7646 : has non-null ORIG_PAT. */
7647 0 : if (!ORIG_PAT (next))
7648 0 : ORIG_PAT (next) = PATTERN (next);
7649 : break;
7650 :
7651 0 : case 1:
7652 0 : if (!ORIG_PAT (next))
7653 : /* If we gonna to overwrite the original pattern of insn,
7654 : save it. */
7655 0 : ORIG_PAT (next) = PATTERN (next);
7656 :
7657 0 : res = haifa_change_pattern (next, new_pat);
7658 0 : gcc_assert (res);
7659 : break;
7660 :
7661 0 : default:
7662 0 : gcc_unreachable ();
7663 : }
7664 : }
7665 :
7666 : /* We need to restore pattern only if (new_ts == 0), because otherwise it is
7667 : either correct (new_ts & SPECULATIVE),
7668 : or we simply don't care (new_ts & HARD_DEP). */
7669 :
7670 325168464 : gcc_assert (!ORIG_PAT (next)
7671 : || !IS_SPECULATION_BRANCHY_CHECK_P (next));
7672 :
7673 325168464 : TODO_SPEC (next) = new_ts;
7674 :
7675 325168464 : if (new_ts & (HARD_DEP | DEP_POSTPONED))
7676 : {
7677 : /* We can't assert (QUEUE_INDEX (next) == QUEUE_NOWHERE) here because
7678 : control-speculative NEXT could have been discarded by sched-rgn.cc
7679 : (the same case as when discarded by can_schedule_ready_p ()). */
7680 : /*gcc_assert (QUEUE_INDEX (next) == QUEUE_NOWHERE);*/
7681 :
7682 210854979 : change_queue_index (next, QUEUE_NOWHERE);
7683 :
7684 210854979 : return -1;
7685 : }
7686 114313485 : else if (!(new_ts & BEGIN_SPEC)
7687 114313485 : && ORIG_PAT (next) && PREDICATED_PAT (next) == NULL_RTX
7688 114313485 : && !IS_SPECULATION_CHECK_P (next))
7689 : /* We should change pattern of every previously speculative
7690 : instruction - and we determine if NEXT was speculative by using
7691 : ORIG_PAT field. Except one case - speculation checks have ORIG_PAT
7692 : pat too, so skip them. */
7693 : {
7694 0 : bool success = haifa_change_pattern (next, ORIG_PAT (next));
7695 0 : gcc_assert (success);
7696 0 : ORIG_PAT (next) = 0;
7697 : }
7698 :
7699 114313485 : if (sched_verbose >= 2)
7700 : {
7701 0 : fprintf (sched_dump, ";;\t\tdependencies resolved: insn %s",
7702 0 : (*current_sched_info->print_insn) (next, 0));
7703 :
7704 0 : if (spec_info && spec_info->dump)
7705 : {
7706 0 : if (new_ts & BEGIN_DATA)
7707 0 : fprintf (spec_info->dump, "; data-spec;");
7708 0 : if (new_ts & BEGIN_CONTROL)
7709 0 : fprintf (spec_info->dump, "; control-spec;");
7710 0 : if (new_ts & BE_IN_CONTROL)
7711 0 : fprintf (spec_info->dump, "; in-control-spec;");
7712 : }
7713 0 : if (TODO_SPEC (next) & DEP_CONTROL)
7714 0 : fprintf (sched_dump, " predicated");
7715 0 : fprintf (sched_dump, "\n");
7716 : }
7717 :
7718 114313485 : adjust_priority (next);
7719 :
7720 114313485 : return fix_tick_ready (next);
7721 : }
7722 :
7723 : /* Calculate INSN_TICK of NEXT and add it to either ready or queue list. */
7724 : static int
7725 114631299 : fix_tick_ready (rtx_insn *next)
7726 : {
7727 114631299 : int tick, delay;
7728 :
7729 114631299 : if (!DEBUG_INSN_P (next) && !sd_lists_empty_p (next, SD_LIST_RES_BACK))
7730 : {
7731 42999959 : int full_p;
7732 42999959 : sd_iterator_def sd_it;
7733 42999959 : dep_t dep;
7734 :
7735 42999959 : tick = INSN_TICK (next);
7736 : /* if tick is not equal to INVALID_TICK, then update
7737 : INSN_TICK of NEXT with the most recent resolved dependence
7738 : cost. Otherwise, recalculate from scratch. */
7739 42999959 : full_p = (tick == INVALID_TICK);
7740 :
7741 193954196 : FOR_EACH_DEP (next, SD_LIST_RES_BACK, sd_it, dep)
7742 : {
7743 150954249 : rtx_insn *pro = DEP_PRO (dep);
7744 150954249 : int tick1;
7745 :
7746 150954249 : gcc_assert (INSN_TICK (pro) >= MIN_TICK);
7747 :
7748 150954249 : tick1 = INSN_TICK (pro) + dep_cost (dep);
7749 150954249 : if (tick1 > tick)
7750 : tick = tick1;
7751 :
7752 150954249 : if (!full_p)
7753 : break;
7754 : }
7755 : }
7756 : else
7757 : tick = -1;
7758 :
7759 114631299 : INSN_TICK (next) = tick;
7760 :
7761 114631299 : delay = tick - clock_var;
7762 114631299 : if (delay <= 0 || sched_pressure != SCHED_PRESSURE_NONE || sched_fusion)
7763 94636682 : delay = QUEUE_READY;
7764 :
7765 114631299 : change_queue_index (next, delay);
7766 :
7767 114631299 : return delay;
7768 : }
7769 :
7770 : /* Move NEXT to the proper queue list with (DELAY >= 1),
7771 : or add it to the ready list (DELAY == QUEUE_READY),
7772 : or remove it from ready and queue lists at all (DELAY == QUEUE_NOWHERE). */
7773 : static void
7774 325486278 : change_queue_index (rtx_insn *next, int delay)
7775 : {
7776 325486278 : int i = QUEUE_INDEX (next);
7777 :
7778 325486278 : gcc_assert (QUEUE_NOWHERE <= delay && delay <= max_insn_queue_index
7779 : && delay != 0);
7780 325486278 : gcc_assert (i != QUEUE_SCHEDULED);
7781 :
7782 325486278 : if ((delay > 0 && NEXT_Q_AFTER (q_ptr, delay) == i)
7783 325360592 : || (delay < 0 && delay == i))
7784 : /* We have nothing to do. */
7785 : return;
7786 :
7787 : /* Remove NEXT from wherever it is now. */
7788 114453095 : if (i == QUEUE_READY)
7789 0 : ready_remove_insn (next);
7790 114453095 : else if (i >= 0)
7791 139610 : queue_remove (next);
7792 :
7793 : /* Add it to the proper place. */
7794 114453095 : if (delay == QUEUE_READY)
7795 94584164 : ready_add (readyp, next, false);
7796 19868931 : else if (delay >= 1)
7797 19868931 : queue_insn (next, delay, "change queue index");
7798 :
7799 114453095 : if (sched_verbose >= 2)
7800 : {
7801 0 : fprintf (sched_dump, ";;\t\ttick updated: insn %s",
7802 0 : (*current_sched_info->print_insn) (next, 0));
7803 :
7804 0 : if (delay == QUEUE_READY)
7805 0 : fprintf (sched_dump, " into ready\n");
7806 0 : else if (delay >= 1)
7807 0 : fprintf (sched_dump, " into queue with cost=%d\n", delay);
7808 : else
7809 0 : fprintf (sched_dump, " removed from ready or queue lists\n");
7810 : }
7811 : }
7812 :
7813 : static int sched_ready_n_insns = -1;
7814 :
7815 : /* Initialize per region data structures. */
7816 : void
7817 10497212 : sched_extend_ready_list (int new_sched_ready_n_insns)
7818 : {
7819 10497212 : int i;
7820 :
7821 10497212 : if (sched_ready_n_insns == -1)
7822 : /* At the first call we need to initialize one more choice_stack
7823 : entry. */
7824 : {
7825 10497106 : i = 0;
7826 10497106 : sched_ready_n_insns = 0;
7827 10497106 : scheduled_insns.reserve (new_sched_ready_n_insns);
7828 : }
7829 : else
7830 106 : i = sched_ready_n_insns + 1;
7831 :
7832 10497212 : ready.veclen = new_sched_ready_n_insns + issue_rate;
7833 10497212 : ready.vec = XRESIZEVEC (rtx_insn *, ready.vec, ready.veclen);
7834 :
7835 10497212 : gcc_assert (new_sched_ready_n_insns >= sched_ready_n_insns);
7836 :
7837 10497212 : ready_try = (signed char *) xrecalloc (ready_try, new_sched_ready_n_insns,
7838 : sched_ready_n_insns,
7839 : sizeof (*ready_try));
7840 :
7841 : /* We allocate +1 element to save initial state in the choice_stack[0]
7842 : entry. */
7843 10497212 : choice_stack = XRESIZEVEC (struct choice_entry, choice_stack,
7844 : new_sched_ready_n_insns + 1);
7845 :
7846 135309109 : for (; i <= new_sched_ready_n_insns; i++)
7847 : {
7848 124811897 : choice_stack[i].state = xmalloc (dfa_state_size);
7849 :
7850 124811897 : if (targetm.sched.first_cycle_multipass_init)
7851 124501813 : targetm.sched.first_cycle_multipass_init (&(choice_stack[i]
7852 : .target_data));
7853 : }
7854 :
7855 10497212 : sched_ready_n_insns = new_sched_ready_n_insns;
7856 10497212 : }
7857 :
7858 : /* Free per region data structures. */
7859 : void
7860 10497106 : sched_finish_ready_list (void)
7861 : {
7862 10497106 : int i;
7863 :
7864 10497106 : free (ready.vec);
7865 10497106 : ready.vec = NULL;
7866 10497106 : ready.veclen = 0;
7867 :
7868 10497106 : free (ready_try);
7869 10497106 : ready_try = NULL;
7870 :
7871 135309003 : for (i = 0; i <= sched_ready_n_insns; i++)
7872 : {
7873 124811897 : if (targetm.sched.first_cycle_multipass_fini)
7874 124501813 : targetm.sched.first_cycle_multipass_fini (&(choice_stack[i]
7875 : .target_data));
7876 :
7877 124811897 : free (choice_stack [i].state);
7878 : }
7879 10497106 : free (choice_stack);
7880 10497106 : choice_stack = NULL;
7881 :
7882 10497106 : sched_ready_n_insns = -1;
7883 10497106 : }
7884 :
7885 : static int
7886 24076540 : haifa_luid_for_non_insn (rtx x)
7887 : {
7888 24076540 : gcc_assert (NOTE_P (x) || LABEL_P (x));
7889 :
7890 24076540 : return 0;
7891 : }
7892 :
7893 : /* Generates recovery code for INSN. */
7894 : static void
7895 0 : generate_recovery_code (rtx_insn *insn)
7896 : {
7897 0 : if (TODO_SPEC (insn) & BEGIN_SPEC)
7898 0 : begin_speculative_block (insn);
7899 :
7900 : /* Here we have insn with no dependencies to
7901 : instructions other then CHECK_SPEC ones. */
7902 :
7903 0 : if (TODO_SPEC (insn) & BE_IN_SPEC)
7904 0 : add_to_speculative_block (insn);
7905 0 : }
7906 :
7907 : /* Helper function.
7908 : Tries to add speculative dependencies of type FS between instructions
7909 : in deps_list L and TWIN. */
7910 : static void
7911 0 : process_insn_forw_deps_be_in_spec (rtx_insn *insn, rtx_insn *twin, ds_t fs)
7912 : {
7913 0 : sd_iterator_def sd_it;
7914 0 : dep_t dep;
7915 :
7916 0 : FOR_EACH_DEP (insn, SD_LIST_FORW, sd_it, dep)
7917 : {
7918 0 : ds_t ds;
7919 0 : rtx_insn *consumer;
7920 :
7921 0 : consumer = DEP_CON (dep);
7922 :
7923 0 : ds = DEP_STATUS (dep);
7924 :
7925 0 : if (/* If we want to create speculative dep. */
7926 : fs
7927 : /* And we can do that because this is a true dep. */
7928 0 : && (ds & DEP_TYPES) == DEP_TRUE)
7929 : {
7930 0 : gcc_assert (!(ds & BE_IN_SPEC));
7931 :
7932 0 : if (/* If this dep can be overcome with 'begin speculation'. */
7933 0 : ds & BEGIN_SPEC)
7934 : /* Then we have a choice: keep the dep 'begin speculative'
7935 : or transform it into 'be in speculative'. */
7936 : {
7937 0 : if (/* In try_ready we assert that if insn once became ready
7938 : it can be removed from the ready (or queue) list only
7939 : due to backend decision. Hence we can't let the
7940 : probability of the speculative dep to decrease. */
7941 0 : ds_weak (ds) <= ds_weak (fs))
7942 : {
7943 0 : ds_t new_ds;
7944 :
7945 0 : new_ds = (ds & ~BEGIN_SPEC) | fs;
7946 :
7947 0 : if (/* consumer can 'be in speculative'. */
7948 0 : sched_insn_is_legitimate_for_speculation_p (consumer,
7949 : new_ds))
7950 : /* Transform it to be in speculative. */
7951 0 : ds = new_ds;
7952 : }
7953 : }
7954 : else
7955 : /* Mark the dep as 'be in speculative'. */
7956 0 : ds |= fs;
7957 : }
7958 :
7959 0 : {
7960 0 : dep_def _new_dep, *new_dep = &_new_dep;
7961 :
7962 0 : init_dep_1 (new_dep, twin, consumer, DEP_TYPE (dep), ds);
7963 0 : sd_add_dep (new_dep, false);
7964 : }
7965 : }
7966 0 : }
7967 :
7968 : /* Generates recovery code for BEGIN speculative INSN. */
7969 : static void
7970 0 : begin_speculative_block (rtx_insn *insn)
7971 : {
7972 0 : if (TODO_SPEC (insn) & BEGIN_DATA)
7973 0 : nr_begin_data++;
7974 0 : if (TODO_SPEC (insn) & BEGIN_CONTROL)
7975 0 : nr_begin_control++;
7976 :
7977 0 : create_check_block_twin (insn, false);
7978 :
7979 0 : TODO_SPEC (insn) &= ~BEGIN_SPEC;
7980 0 : }
7981 :
7982 : static void haifa_init_insn (rtx_insn *);
7983 :
7984 : /* Generates recovery code for BE_IN speculative INSN. */
7985 : static void
7986 0 : add_to_speculative_block (rtx_insn *insn)
7987 : {
7988 0 : ds_t ts;
7989 0 : sd_iterator_def sd_it;
7990 0 : dep_t dep;
7991 0 : auto_vec<rtx_insn *, 10> twins;
7992 :
7993 0 : ts = TODO_SPEC (insn);
7994 0 : gcc_assert (!(ts & ~BE_IN_SPEC));
7995 :
7996 0 : if (ts & BE_IN_DATA)
7997 0 : nr_be_in_data++;
7998 0 : if (ts & BE_IN_CONTROL)
7999 0 : nr_be_in_control++;
8000 :
8001 0 : TODO_SPEC (insn) &= ~BE_IN_SPEC;
8002 0 : gcc_assert (!TODO_SPEC (insn));
8003 :
8004 0 : DONE_SPEC (insn) |= ts;
8005 :
8006 : /* First we convert all simple checks to branchy. */
8007 0 : for (sd_it = sd_iterator_start (insn, SD_LIST_SPEC_BACK);
8008 0 : sd_iterator_cond (&sd_it, &dep);)
8009 : {
8010 0 : rtx_insn *check = DEP_PRO (dep);
8011 :
8012 0 : if (IS_SPECULATION_SIMPLE_CHECK_P (check))
8013 : {
8014 0 : create_check_block_twin (check, true);
8015 :
8016 : /* Restart search. */
8017 0 : sd_it = sd_iterator_start (insn, SD_LIST_SPEC_BACK);
8018 : }
8019 : else
8020 : /* Continue search. */
8021 0 : sd_iterator_next (&sd_it);
8022 : }
8023 :
8024 0 : auto_vec<rtx_insn *> priorities_roots;
8025 0 : clear_priorities (insn, &priorities_roots);
8026 :
8027 0 : while (1)
8028 : {
8029 0 : rtx_insn *check, *twin;
8030 0 : basic_block rec;
8031 :
8032 : /* Get the first backward dependency of INSN. */
8033 0 : sd_it = sd_iterator_start (insn, SD_LIST_SPEC_BACK);
8034 0 : if (!sd_iterator_cond (&sd_it, &dep))
8035 : /* INSN has no backward dependencies left. */
8036 : break;
8037 :
8038 0 : gcc_assert ((DEP_STATUS (dep) & BEGIN_SPEC) == 0
8039 : && (DEP_STATUS (dep) & BE_IN_SPEC) != 0
8040 : && (DEP_STATUS (dep) & DEP_TYPES) == DEP_TRUE);
8041 :
8042 0 : check = DEP_PRO (dep);
8043 :
8044 0 : gcc_assert (!IS_SPECULATION_CHECK_P (check) && !ORIG_PAT (check)
8045 : && QUEUE_INDEX (check) == QUEUE_NOWHERE);
8046 :
8047 0 : rec = BLOCK_FOR_INSN (check);
8048 :
8049 0 : twin = emit_insn_before (copy_insn (PATTERN (insn)), BB_END (rec));
8050 0 : haifa_init_insn (twin);
8051 :
8052 0 : sd_copy_back_deps (twin, insn, true);
8053 :
8054 0 : if (sched_verbose && spec_info->dump)
8055 : /* INSN_BB (insn) isn't determined for twin insns yet.
8056 : So we can't use current_sched_info->print_insn. */
8057 0 : fprintf (spec_info->dump, ";;\t\tGenerated twin insn : %d/rec%d\n",
8058 0 : INSN_UID (twin), rec->index);
8059 :
8060 0 : twins.safe_push (twin);
8061 :
8062 : /* Add dependences between TWIN and all appropriate
8063 : instructions from REC. */
8064 0 : FOR_EACH_DEP (insn, SD_LIST_SPEC_BACK, sd_it, dep)
8065 : {
8066 0 : rtx_insn *pro = DEP_PRO (dep);
8067 :
8068 0 : gcc_assert (DEP_TYPE (dep) == REG_DEP_TRUE);
8069 :
8070 : /* INSN might have dependencies from the instructions from
8071 : several recovery blocks. At this iteration we process those
8072 : producers that reside in REC. */
8073 0 : if (BLOCK_FOR_INSN (pro) == rec)
8074 : {
8075 0 : dep_def _new_dep, *new_dep = &_new_dep;
8076 :
8077 0 : init_dep (new_dep, pro, twin, REG_DEP_TRUE);
8078 0 : sd_add_dep (new_dep, false);
8079 : }
8080 : }
8081 :
8082 0 : process_insn_forw_deps_be_in_spec (insn, twin, ts);
8083 :
8084 : /* Remove all dependencies between INSN and insns in REC. */
8085 0 : for (sd_it = sd_iterator_start (insn, SD_LIST_SPEC_BACK);
8086 0 : sd_iterator_cond (&sd_it, &dep);)
8087 : {
8088 0 : rtx_insn *pro = DEP_PRO (dep);
8089 :
8090 0 : if (BLOCK_FOR_INSN (pro) == rec)
8091 0 : sd_delete_dep (sd_it);
8092 : else
8093 0 : sd_iterator_next (&sd_it);
8094 : }
8095 0 : }
8096 :
8097 : /* We couldn't have added the dependencies between INSN and TWINS earlier
8098 : because that would make TWINS appear in the INSN_BACK_DEPS (INSN). */
8099 0 : unsigned int i;
8100 0 : rtx_insn *twin;
8101 0 : FOR_EACH_VEC_ELT_REVERSE (twins, i, twin)
8102 : {
8103 0 : dep_def _new_dep, *new_dep = &_new_dep;
8104 :
8105 0 : init_dep (new_dep, insn, twin, REG_DEP_OUTPUT);
8106 0 : sd_add_dep (new_dep, false);
8107 : }
8108 :
8109 0 : calc_priorities (priorities_roots);
8110 0 : }
8111 :
8112 : /* Extends and fills with zeros (only the new part) array pointed to by P. */
8113 : void *
8114 10497314 : xrecalloc (void *p, size_t new_nmemb, size_t old_nmemb, size_t size)
8115 : {
8116 10497314 : gcc_assert (new_nmemb >= old_nmemb);
8117 10497314 : p = XRESIZEVAR (void, p, new_nmemb * size);
8118 10497314 : memset (((char *) p) + old_nmemb * size, 0, (new_nmemb - old_nmemb) * size);
8119 10497314 : return p;
8120 : }
8121 :
8122 : /* Helper function.
8123 : Find fallthru edge from PRED. */
8124 : edge
8125 124 : find_fallthru_edge_from (basic_block pred)
8126 : {
8127 124 : edge e;
8128 124 : basic_block succ;
8129 :
8130 124 : succ = pred->next_bb;
8131 124 : gcc_assert (succ->prev_bb == pred);
8132 :
8133 372 : if (EDGE_COUNT (pred->succs) <= EDGE_COUNT (succ->preds))
8134 : {
8135 42 : e = find_fallthru_edge (pred->succs);
8136 :
8137 42 : if (e)
8138 : {
8139 24 : gcc_assert (e->dest == succ || e->dest->index == EXIT_BLOCK);
8140 : return e;
8141 : }
8142 : }
8143 : else
8144 : {
8145 82 : e = find_fallthru_edge (succ->preds);
8146 :
8147 82 : if (e)
8148 : {
8149 82 : gcc_assert (e->src == pred);
8150 : return e;
8151 : }
8152 : }
8153 :
8154 : return NULL;
8155 : }
8156 :
8157 : /* Extend per basic block data structures. */
8158 : static void
8159 983663 : sched_extend_bb (void)
8160 : {
8161 : /* The following is done to keep current_sched_info->next_tail non null. */
8162 983663 : rtx_insn *end = BB_END (EXIT_BLOCK_PTR_FOR_FN (cfun)->prev_bb);
8163 983663 : rtx_insn *insn = DEBUG_INSN_P (end) ? prev_nondebug_insn (end) : end;
8164 983663 : if (NEXT_INSN (end) == 0
8165 983663 : || (!NOTE_P (insn)
8166 983201 : && !LABEL_P (insn)
8167 : /* Don't emit a NOTE if it would end up before a BARRIER. */
8168 983182 : && !BARRIER_P (next_nondebug_insn (end))))
8169 : {
8170 559 : rtx_note *note = emit_note_after (NOTE_INSN_DELETED, end);
8171 : /* Make note appear outside BB. */
8172 559 : set_block_for_insn (note, NULL);
8173 559 : BB_END (EXIT_BLOCK_PTR_FOR_FN (cfun)->prev_bb) = end;
8174 : }
8175 983663 : }
8176 :
8177 : /* Init per basic block data structures. */
8178 : void
8179 983663 : sched_init_bbs (void)
8180 : {
8181 983663 : sched_extend_bb ();
8182 983663 : }
8183 :
8184 : /* Initialize BEFORE_RECOVERY variable. */
8185 : static void
8186 0 : init_before_recovery (basic_block *before_recovery_ptr)
8187 : {
8188 0 : basic_block last;
8189 0 : edge e;
8190 :
8191 0 : last = EXIT_BLOCK_PTR_FOR_FN (cfun)->prev_bb;
8192 0 : e = find_fallthru_edge_from (last);
8193 :
8194 0 : if (e)
8195 : {
8196 : /* We create two basic blocks:
8197 : 1. Single instruction block is inserted right after E->SRC
8198 : and has jump to
8199 : 2. Empty block right before EXIT_BLOCK.
8200 : Between these two blocks recovery blocks will be emitted. */
8201 :
8202 0 : basic_block single, empty;
8203 :
8204 : /* If the fallthrough edge to exit we've found is from the block we've
8205 : created before, don't do anything more. */
8206 0 : if (last == after_recovery)
8207 : return;
8208 :
8209 0 : adding_bb_to_current_region_p = false;
8210 :
8211 0 : single = sched_create_empty_bb (last);
8212 0 : empty = sched_create_empty_bb (single);
8213 :
8214 : /* Add new blocks to the root loop. */
8215 0 : if (current_loops != NULL)
8216 : {
8217 0 : add_bb_to_loop (single, (*current_loops->larray)[0]);
8218 0 : add_bb_to_loop (empty, (*current_loops->larray)[0]);
8219 : }
8220 :
8221 0 : single->count = last->count;
8222 0 : empty->count = last->count;
8223 0 : BB_COPY_PARTITION (single, last);
8224 0 : BB_COPY_PARTITION (empty, last);
8225 :
8226 0 : redirect_edge_succ (e, single);
8227 0 : make_single_succ_edge (single, empty, 0);
8228 0 : make_single_succ_edge (empty, EXIT_BLOCK_PTR_FOR_FN (cfun),
8229 : EDGE_FALLTHRU);
8230 :
8231 0 : rtx_code_label *label = block_label (empty);
8232 0 : rtx_jump_insn *x = emit_jump_insn_after (targetm.gen_jump (label),
8233 0 : BB_END (single));
8234 0 : JUMP_LABEL (x) = label;
8235 0 : LABEL_NUSES (label)++;
8236 0 : haifa_init_insn (x);
8237 :
8238 0 : emit_barrier_after (x);
8239 :
8240 0 : sched_init_only_bb (empty, NULL);
8241 0 : sched_init_only_bb (single, NULL);
8242 0 : sched_extend_bb ();
8243 :
8244 0 : adding_bb_to_current_region_p = true;
8245 0 : before_recovery = single;
8246 0 : after_recovery = empty;
8247 :
8248 0 : if (before_recovery_ptr)
8249 0 : *before_recovery_ptr = before_recovery;
8250 :
8251 0 : if (sched_verbose >= 2 && spec_info->dump)
8252 0 : fprintf (spec_info->dump,
8253 : ";;\t\tFixed fallthru to EXIT : %d->>%d->%d->>EXIT\n",
8254 : last->index, single->index, empty->index);
8255 : }
8256 : else
8257 0 : before_recovery = last;
8258 : }
8259 :
8260 : /* Returns new recovery block. */
8261 : basic_block
8262 0 : sched_create_recovery_block (basic_block *before_recovery_ptr)
8263 : {
8264 0 : rtx_insn *barrier;
8265 0 : basic_block rec;
8266 :
8267 0 : haifa_recovery_bb_recently_added_p = true;
8268 0 : haifa_recovery_bb_ever_added_p = true;
8269 :
8270 0 : init_before_recovery (before_recovery_ptr);
8271 :
8272 0 : barrier = get_last_bb_insn (before_recovery);
8273 0 : gcc_assert (BARRIER_P (barrier));
8274 :
8275 0 : rtx_insn *label = emit_label_after (gen_label_rtx (), barrier);
8276 :
8277 0 : rec = create_basic_block (label, label, before_recovery);
8278 :
8279 : /* A recovery block always ends with an unconditional jump. */
8280 0 : emit_barrier_after (BB_END (rec));
8281 :
8282 0 : if (BB_PARTITION (before_recovery) != BB_UNPARTITIONED)
8283 0 : BB_SET_PARTITION (rec, BB_COLD_PARTITION);
8284 :
8285 0 : if (sched_verbose && spec_info->dump)
8286 0 : fprintf (spec_info->dump, ";;\t\tGenerated recovery block rec%d\n",
8287 : rec->index);
8288 :
8289 0 : return rec;
8290 : }
8291 :
8292 : /* Create edges: FIRST_BB -> REC; FIRST_BB -> SECOND_BB; REC -> SECOND_BB
8293 : and emit necessary jumps. */
8294 : void
8295 0 : sched_create_recovery_edges (basic_block first_bb, basic_block rec,
8296 : basic_block second_bb)
8297 : {
8298 0 : int edge_flags;
8299 :
8300 : /* This is fixing of incoming edge. */
8301 : /* ??? Which other flags should be specified? */
8302 0 : if (BB_PARTITION (first_bb) != BB_PARTITION (rec))
8303 : /* Partition type is the same, if it is "unpartitioned". */
8304 : edge_flags = EDGE_CROSSING;
8305 : else
8306 0 : edge_flags = 0;
8307 :
8308 0 : edge e2 = single_succ_edge (first_bb);
8309 0 : edge e = make_edge (first_bb, rec, edge_flags);
8310 :
8311 : /* TODO: The actual probability can be determined and is computed as
8312 : 'todo_spec' variable in create_check_block_twin and
8313 : in sel-sched.cc `check_ds' in create_speculation_check. */
8314 0 : e->probability = profile_probability::very_unlikely ();
8315 0 : rec->count = e->count ();
8316 0 : e2->probability = e->probability.invert ();
8317 :
8318 0 : rtx_code_label *label = block_label (second_bb);
8319 0 : rtx_jump_insn *jump = emit_jump_insn_after (targetm.gen_jump (label),
8320 0 : BB_END (rec));
8321 0 : JUMP_LABEL (jump) = label;
8322 0 : LABEL_NUSES (label)++;
8323 :
8324 0 : if (BB_PARTITION (second_bb) != BB_PARTITION (rec))
8325 : /* Partition type is the same, if it is "unpartitioned". */
8326 : {
8327 : /* Rewritten from cfgrtl.cc. */
8328 0 : if (crtl->has_bb_partition && targetm_common.have_named_sections)
8329 : {
8330 : /* We don't need the same note for the check because
8331 : any_condjump_p (check) == true. */
8332 0 : CROSSING_JUMP_P (jump) = 1;
8333 : }
8334 : edge_flags = EDGE_CROSSING;
8335 : }
8336 : else
8337 : edge_flags = 0;
8338 :
8339 0 : make_single_succ_edge (rec, second_bb, edge_flags);
8340 0 : if (dom_info_available_p (CDI_DOMINATORS))
8341 0 : set_immediate_dominator (CDI_DOMINATORS, rec, first_bb);
8342 0 : }
8343 :
8344 : /* This function creates recovery code for INSN. If MUTATE_P is nonzero,
8345 : INSN is a simple check, that should be converted to branchy one. */
8346 : static void
8347 0 : create_check_block_twin (rtx_insn *insn, bool mutate_p)
8348 : {
8349 0 : basic_block rec;
8350 0 : rtx_insn *label, *check, *twin;
8351 0 : rtx check_pat;
8352 0 : ds_t fs;
8353 0 : sd_iterator_def sd_it;
8354 0 : dep_t dep;
8355 0 : dep_def _new_dep, *new_dep = &_new_dep;
8356 0 : ds_t todo_spec;
8357 :
8358 0 : gcc_assert (ORIG_PAT (insn) != NULL_RTX);
8359 :
8360 0 : if (!mutate_p)
8361 0 : todo_spec = TODO_SPEC (insn);
8362 : else
8363 : {
8364 0 : gcc_assert (IS_SPECULATION_SIMPLE_CHECK_P (insn)
8365 : && (TODO_SPEC (insn) & SPECULATIVE) == 0);
8366 :
8367 0 : todo_spec = CHECK_SPEC (insn);
8368 : }
8369 :
8370 0 : todo_spec &= SPECULATIVE;
8371 :
8372 : /* Create recovery block. */
8373 0 : if (mutate_p || targetm.sched.needs_block_p (todo_spec))
8374 : {
8375 0 : rec = sched_create_recovery_block (NULL);
8376 0 : label = BB_HEAD (rec);
8377 : }
8378 : else
8379 : {
8380 0 : rec = EXIT_BLOCK_PTR_FOR_FN (cfun);
8381 0 : label = NULL;
8382 : }
8383 :
8384 : /* Emit CHECK. */
8385 0 : check_pat = targetm.sched.gen_spec_check (insn, label, todo_spec);
8386 :
8387 0 : if (rec != EXIT_BLOCK_PTR_FOR_FN (cfun))
8388 : {
8389 : /* To have mem_reg alive at the beginning of second_bb,
8390 : we emit check BEFORE insn, so insn after splitting
8391 : insn will be at the beginning of second_bb, which will
8392 : provide us with the correct life information. */
8393 0 : check = emit_jump_insn_before (check_pat, insn);
8394 0 : JUMP_LABEL (check) = label;
8395 0 : LABEL_NUSES (label)++;
8396 : }
8397 : else
8398 0 : check = emit_insn_before (check_pat, insn);
8399 :
8400 : /* Extend data structures. */
8401 0 : haifa_init_insn (check);
8402 :
8403 : /* CHECK is being added to current region. Extend ready list. */
8404 0 : gcc_assert (sched_ready_n_insns != -1);
8405 0 : sched_extend_ready_list (sched_ready_n_insns + 1);
8406 :
8407 0 : if (current_sched_info->add_remove_insn)
8408 0 : current_sched_info->add_remove_insn (insn, 0);
8409 :
8410 0 : RECOVERY_BLOCK (check) = rec;
8411 :
8412 0 : if (sched_verbose && spec_info->dump)
8413 0 : fprintf (spec_info->dump, ";;\t\tGenerated check insn : %s\n",
8414 0 : (*current_sched_info->print_insn) (check, 0));
8415 :
8416 0 : gcc_assert (ORIG_PAT (insn));
8417 :
8418 : /* Initialize TWIN (twin is a duplicate of original instruction
8419 : in the recovery block). */
8420 0 : if (rec != EXIT_BLOCK_PTR_FOR_FN (cfun))
8421 : {
8422 0 : sd_iterator_def sd_it;
8423 0 : dep_t dep;
8424 :
8425 0 : FOR_EACH_DEP (insn, SD_LIST_RES_BACK, sd_it, dep)
8426 0 : if ((DEP_STATUS (dep) & DEP_OUTPUT) != 0)
8427 : {
8428 0 : struct _dep _dep2, *dep2 = &_dep2;
8429 :
8430 0 : init_dep (dep2, DEP_PRO (dep), check, REG_DEP_TRUE);
8431 :
8432 0 : sd_add_dep (dep2, true);
8433 : }
8434 :
8435 0 : twin = emit_insn_after (ORIG_PAT (insn), BB_END (rec));
8436 0 : haifa_init_insn (twin);
8437 :
8438 0 : if (sched_verbose && spec_info->dump)
8439 : /* INSN_BB (insn) isn't determined for twin insns yet.
8440 : So we can't use current_sched_info->print_insn. */
8441 0 : fprintf (spec_info->dump, ";;\t\tGenerated twin insn : %d/rec%d\n",
8442 0 : INSN_UID (twin), rec->index);
8443 : }
8444 : else
8445 : {
8446 0 : ORIG_PAT (check) = ORIG_PAT (insn);
8447 0 : HAS_INTERNAL_DEP (check) = 1;
8448 0 : twin = check;
8449 : /* ??? We probably should change all OUTPUT dependencies to
8450 : (TRUE | OUTPUT). */
8451 : }
8452 :
8453 : /* Copy all resolved back dependencies of INSN to TWIN. This will
8454 : provide correct value for INSN_TICK (TWIN). */
8455 0 : sd_copy_back_deps (twin, insn, true);
8456 :
8457 0 : if (rec != EXIT_BLOCK_PTR_FOR_FN (cfun))
8458 : /* In case of branchy check, fix CFG. */
8459 : {
8460 0 : basic_block first_bb, second_bb;
8461 0 : rtx_insn *jump;
8462 :
8463 0 : first_bb = BLOCK_FOR_INSN (check);
8464 0 : second_bb = sched_split_block (first_bb, check);
8465 :
8466 0 : sched_create_recovery_edges (first_bb, rec, second_bb);
8467 :
8468 0 : sched_init_only_bb (second_bb, first_bb);
8469 0 : sched_init_only_bb (rec, EXIT_BLOCK_PTR_FOR_FN (cfun));
8470 :
8471 0 : jump = BB_END (rec);
8472 0 : haifa_init_insn (jump);
8473 : }
8474 :
8475 : /* Move backward dependences from INSN to CHECK and
8476 : move forward dependences from INSN to TWIN. */
8477 :
8478 : /* First, create dependencies between INSN's producers and CHECK & TWIN. */
8479 0 : FOR_EACH_DEP (insn, SD_LIST_BACK, sd_it, dep)
8480 : {
8481 0 : rtx_insn *pro = DEP_PRO (dep);
8482 0 : ds_t ds;
8483 :
8484 : /* If BEGIN_DATA: [insn ~~TRUE~~> producer]:
8485 : check --TRUE--> producer ??? or ANTI ???
8486 : twin --TRUE--> producer
8487 : twin --ANTI--> check
8488 :
8489 : If BEGIN_CONTROL: [insn ~~ANTI~~> producer]:
8490 : check --ANTI--> producer
8491 : twin --ANTI--> producer
8492 : twin --ANTI--> check
8493 :
8494 : If BE_IN_SPEC: [insn ~~TRUE~~> producer]:
8495 : check ~~TRUE~~> producer
8496 : twin ~~TRUE~~> producer
8497 : twin --ANTI--> check */
8498 :
8499 0 : ds = DEP_STATUS (dep);
8500 :
8501 0 : if (ds & BEGIN_SPEC)
8502 : {
8503 0 : gcc_assert (!mutate_p);
8504 0 : ds &= ~BEGIN_SPEC;
8505 : }
8506 :
8507 0 : init_dep_1 (new_dep, pro, check, DEP_TYPE (dep), ds);
8508 0 : sd_add_dep (new_dep, false);
8509 :
8510 0 : if (rec != EXIT_BLOCK_PTR_FOR_FN (cfun))
8511 : {
8512 0 : DEP_CON (new_dep) = twin;
8513 0 : sd_add_dep (new_dep, false);
8514 : }
8515 : }
8516 :
8517 : /* Second, remove backward dependencies of INSN. */
8518 0 : for (sd_it = sd_iterator_start (insn, SD_LIST_SPEC_BACK);
8519 0 : sd_iterator_cond (&sd_it, &dep);)
8520 : {
8521 0 : if ((DEP_STATUS (dep) & BEGIN_SPEC)
8522 0 : || mutate_p)
8523 : /* We can delete this dep because we overcome it with
8524 : BEGIN_SPECULATION. */
8525 0 : sd_delete_dep (sd_it);
8526 : else
8527 0 : sd_iterator_next (&sd_it);
8528 : }
8529 :
8530 : /* Future Speculations. Determine what BE_IN speculations will be like. */
8531 0 : fs = 0;
8532 :
8533 : /* Fields (DONE_SPEC (x) & BEGIN_SPEC) and CHECK_SPEC (x) are set only
8534 : here. */
8535 :
8536 0 : gcc_assert (!DONE_SPEC (insn));
8537 :
8538 0 : if (!mutate_p)
8539 : {
8540 0 : ds_t ts = TODO_SPEC (insn);
8541 :
8542 0 : DONE_SPEC (insn) = ts & BEGIN_SPEC;
8543 0 : CHECK_SPEC (check) = ts & BEGIN_SPEC;
8544 :
8545 : /* Luckiness of future speculations solely depends upon initial
8546 : BEGIN speculation. */
8547 0 : if (ts & BEGIN_DATA)
8548 0 : fs = set_dep_weak (fs, BE_IN_DATA, get_dep_weak (ts, BEGIN_DATA));
8549 0 : if (ts & BEGIN_CONTROL)
8550 0 : fs = set_dep_weak (fs, BE_IN_CONTROL,
8551 : get_dep_weak (ts, BEGIN_CONTROL));
8552 : }
8553 : else
8554 0 : CHECK_SPEC (check) = CHECK_SPEC (insn);
8555 :
8556 : /* Future speculations: call the helper. */
8557 0 : process_insn_forw_deps_be_in_spec (insn, twin, fs);
8558 :
8559 0 : if (rec != EXIT_BLOCK_PTR_FOR_FN (cfun))
8560 : {
8561 : /* Which types of dependencies should we use here is,
8562 : generally, machine-dependent question... But, for now,
8563 : it is not. */
8564 :
8565 0 : if (!mutate_p)
8566 : {
8567 0 : init_dep (new_dep, insn, check, REG_DEP_TRUE);
8568 0 : sd_add_dep (new_dep, false);
8569 :
8570 0 : init_dep (new_dep, insn, twin, REG_DEP_OUTPUT);
8571 0 : sd_add_dep (new_dep, false);
8572 : }
8573 : else
8574 : {
8575 0 : if (spec_info->dump)
8576 0 : fprintf (spec_info->dump, ";;\t\tRemoved simple check : %s\n",
8577 0 : (*current_sched_info->print_insn) (insn, 0));
8578 :
8579 : /* Remove all dependencies of the INSN. */
8580 0 : {
8581 0 : sd_it = sd_iterator_start (insn, (SD_LIST_FORW
8582 : | SD_LIST_BACK
8583 : | SD_LIST_RES_BACK));
8584 0 : while (sd_iterator_cond (&sd_it, &dep))
8585 0 : sd_delete_dep (sd_it);
8586 : }
8587 :
8588 : /* If former check (INSN) already was moved to the ready (or queue)
8589 : list, add new check (CHECK) there too. */
8590 0 : if (QUEUE_INDEX (insn) != QUEUE_NOWHERE)
8591 0 : try_ready (check);
8592 :
8593 : /* Remove old check from instruction stream and free its
8594 : data. */
8595 0 : sched_remove_insn (insn);
8596 : }
8597 :
8598 0 : init_dep (new_dep, check, twin, REG_DEP_ANTI);
8599 0 : sd_add_dep (new_dep, false);
8600 : }
8601 : else
8602 : {
8603 0 : init_dep_1 (new_dep, insn, check, REG_DEP_TRUE, DEP_TRUE | DEP_OUTPUT);
8604 0 : sd_add_dep (new_dep, false);
8605 : }
8606 :
8607 0 : if (!mutate_p)
8608 : /* Fix priorities. If MUTATE_P is nonzero, this is not necessary,
8609 : because it'll be done later in add_to_speculative_block. */
8610 : {
8611 0 : auto_vec<rtx_insn *> priorities_roots;
8612 :
8613 0 : clear_priorities (twin, &priorities_roots);
8614 0 : calc_priorities (priorities_roots);
8615 0 : }
8616 0 : }
8617 :
8618 : /* Removes dependency between instructions in the recovery block REC
8619 : and usual region instructions. It keeps inner dependences so it
8620 : won't be necessary to recompute them. */
8621 : static void
8622 0 : fix_recovery_deps (basic_block rec)
8623 : {
8624 0 : rtx_insn *note, *insn, *jump;
8625 0 : auto_vec<rtx_insn *, 10> ready_list;
8626 0 : auto_bitmap in_ready;
8627 :
8628 : /* NOTE - a basic block note. */
8629 0 : note = NEXT_INSN (BB_HEAD (rec));
8630 0 : gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note));
8631 0 : insn = BB_END (rec);
8632 0 : gcc_assert (JUMP_P (insn));
8633 0 : insn = PREV_INSN (insn);
8634 :
8635 0 : do
8636 : {
8637 0 : sd_iterator_def sd_it;
8638 0 : dep_t dep;
8639 :
8640 0 : for (sd_it = sd_iterator_start (insn, SD_LIST_FORW);
8641 0 : sd_iterator_cond (&sd_it, &dep);)
8642 : {
8643 0 : rtx_insn *consumer = DEP_CON (dep);
8644 :
8645 0 : if (BLOCK_FOR_INSN (consumer) != rec)
8646 : {
8647 0 : sd_delete_dep (sd_it);
8648 :
8649 0 : if (bitmap_set_bit (in_ready, INSN_LUID (consumer)))
8650 0 : ready_list.safe_push (consumer);
8651 : }
8652 : else
8653 : {
8654 0 : gcc_assert ((DEP_STATUS (dep) & DEP_TYPES) == DEP_TRUE);
8655 :
8656 0 : sd_iterator_next (&sd_it);
8657 : }
8658 : }
8659 :
8660 0 : insn = PREV_INSN (insn);
8661 : }
8662 0 : while (insn != note);
8663 :
8664 : /* Try to add instructions to the ready or queue list. */
8665 0 : unsigned int i;
8666 0 : rtx_insn *temp;
8667 0 : FOR_EACH_VEC_ELT_REVERSE (ready_list, i, temp)
8668 0 : try_ready (temp);
8669 :
8670 : /* Fixing jump's dependences. */
8671 0 : insn = BB_HEAD (rec);
8672 0 : jump = BB_END (rec);
8673 :
8674 0 : gcc_assert (LABEL_P (insn));
8675 0 : insn = NEXT_INSN (insn);
8676 :
8677 0 : gcc_assert (NOTE_INSN_BASIC_BLOCK_P (insn));
8678 0 : add_jump_dependencies (insn, jump);
8679 0 : }
8680 :
8681 : /* Change pattern of INSN to NEW_PAT. Invalidate cached haifa
8682 : instruction data. */
8683 : static bool
8684 0 : haifa_change_pattern (rtx_insn *insn, rtx new_pat)
8685 : {
8686 0 : int t;
8687 :
8688 0 : t = validate_change (insn, &PATTERN (insn), new_pat, 0);
8689 0 : if (!t)
8690 : return false;
8691 :
8692 0 : update_insn_after_change (insn);
8693 0 : return true;
8694 : }
8695 :
8696 : /* -1 - can't speculate,
8697 : 0 - for speculation with REQUEST mode it is OK to use
8698 : current instruction pattern,
8699 : 1 - need to change pattern for *NEW_PAT to be speculative. */
8700 : int
8701 0 : sched_speculate_insn (rtx_insn *insn, ds_t request, rtx *new_pat)
8702 : {
8703 0 : gcc_assert (current_sched_info->flags & DO_SPECULATION
8704 : && (request & SPECULATIVE)
8705 : && sched_insn_is_legitimate_for_speculation_p (insn, request));
8706 :
8707 0 : if ((request & spec_info->mask) != request)
8708 : return -1;
8709 :
8710 0 : if (request & BE_IN_SPEC
8711 0 : && !(request & BEGIN_SPEC))
8712 : return 0;
8713 :
8714 0 : return targetm.sched.speculate_insn (insn, request, new_pat);
8715 : }
8716 :
8717 : static int
8718 0 : haifa_speculate_insn (rtx_insn *insn, ds_t request, rtx *new_pat)
8719 : {
8720 0 : gcc_assert (sched_deps_info->generate_spec_deps
8721 : && !IS_SPECULATION_CHECK_P (insn));
8722 :
8723 0 : if (HAS_INTERNAL_DEP (insn)
8724 0 : || SCHED_GROUP_P (insn))
8725 : return -1;
8726 :
8727 0 : return sched_speculate_insn (insn, request, new_pat);
8728 : }
8729 :
8730 : /* Print some information about block BB, which starts with HEAD and
8731 : ends with TAIL, before scheduling it.
8732 : I is zero, if scheduler is about to start with the fresh ebb. */
8733 : static void
8734 192 : dump_new_block_header (int i, basic_block bb, rtx_insn *head, rtx_insn *tail)
8735 : {
8736 192 : if (!i)
8737 190 : fprintf (sched_dump,
8738 : ";; ======================================================\n");
8739 : else
8740 2 : fprintf (sched_dump,
8741 : ";; =====================ADVANCING TO=====================\n");
8742 192 : fprintf (sched_dump,
8743 : ";; -- basic block %d from %d to %d -- %s reload\n",
8744 192 : bb->index, INSN_UID (head), INSN_UID (tail),
8745 192 : (reload_completed ? "after" : "before"));
8746 192 : fprintf (sched_dump,
8747 : ";; ======================================================\n");
8748 192 : fprintf (sched_dump, "\n");
8749 192 : }
8750 :
8751 : /* Unlink basic block notes and labels and saves them, so they
8752 : can be easily restored. We unlink basic block notes in EBB to
8753 : provide back-compatibility with the previous code, as target backends
8754 : assume, that there'll be only instructions between
8755 : current_sched_info->{head and tail}. We restore these notes as soon
8756 : as we can.
8757 : FIRST (LAST) is the first (last) basic block in the ebb.
8758 : NB: In usual case (FIRST == LAST) nothing is really done. */
8759 : void
8760 10477678 : unlink_bb_notes (basic_block first, basic_block last)
8761 : {
8762 : /* We DON'T unlink basic block notes of the first block in the ebb. */
8763 10477678 : if (first == last)
8764 : return;
8765 :
8766 14 : bb_header = XNEWVEC (rtx_insn *, last_basic_block_for_fn (cfun));
8767 :
8768 : /* Make a sentinel. */
8769 14 : if (last->next_bb != EXIT_BLOCK_PTR_FOR_FN (cfun))
8770 14 : bb_header[last->next_bb->index] = 0;
8771 :
8772 14 : first = first->next_bb;
8773 82 : do
8774 : {
8775 48 : rtx_insn *prev, *label, *note, *next;
8776 :
8777 48 : label = BB_HEAD (last);
8778 48 : if (LABEL_P (label))
8779 0 : note = NEXT_INSN (label);
8780 : else
8781 : note = label;
8782 48 : gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note));
8783 :
8784 48 : prev = PREV_INSN (label);
8785 48 : next = NEXT_INSN (note);
8786 48 : gcc_assert (prev && next);
8787 :
8788 48 : SET_NEXT_INSN (prev) = next;
8789 48 : SET_PREV_INSN (next) = prev;
8790 :
8791 48 : bb_header[last->index] = label;
8792 :
8793 48 : if (last == first)
8794 : break;
8795 :
8796 34 : last = last->prev_bb;
8797 34 : }
8798 : while (1);
8799 : }
8800 :
8801 : /* Restore basic block notes.
8802 : FIRST is the first basic block in the ebb. */
8803 : static void
8804 10477678 : restore_bb_notes (basic_block first)
8805 : {
8806 10477678 : if (!bb_header)
8807 : return;
8808 :
8809 : /* We DON'T unlink basic block notes of the first block in the ebb. */
8810 14 : first = first->next_bb;
8811 : /* Remember: FIRST is actually a second basic block in the ebb. */
8812 :
8813 14 : while (first != EXIT_BLOCK_PTR_FOR_FN (cfun)
8814 62 : && bb_header[first->index])
8815 : {
8816 48 : rtx_insn *prev, *label, *note, *next;
8817 :
8818 48 : label = bb_header[first->index];
8819 48 : prev = PREV_INSN (label);
8820 48 : next = NEXT_INSN (prev);
8821 :
8822 48 : if (LABEL_P (label))
8823 0 : note = NEXT_INSN (label);
8824 : else
8825 : note = label;
8826 48 : gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note));
8827 :
8828 48 : bb_header[first->index] = 0;
8829 :
8830 48 : SET_NEXT_INSN (prev) = label;
8831 48 : SET_NEXT_INSN (note) = next;
8832 48 : SET_PREV_INSN (next) = note;
8833 :
8834 48 : first = first->next_bb;
8835 : }
8836 :
8837 14 : free (bb_header);
8838 14 : bb_header = 0;
8839 : }
8840 :
8841 : /* Helper function.
8842 : Fix CFG after both in- and inter-block movement of
8843 : control_flow_insn_p JUMP. */
8844 : static void
8845 1 : fix_jump_move (rtx_insn *jump)
8846 : {
8847 1 : basic_block bb, jump_bb, jump_bb_next;
8848 :
8849 1 : bb = BLOCK_FOR_INSN (PREV_INSN (jump));
8850 1 : jump_bb = BLOCK_FOR_INSN (jump);
8851 1 : jump_bb_next = jump_bb->next_bb;
8852 :
8853 1 : gcc_assert (common_sched_info->sched_pass_id == SCHED_EBB_PASS
8854 : || IS_SPECULATION_BRANCHY_CHECK_P (jump));
8855 :
8856 1 : if (!NOTE_INSN_BASIC_BLOCK_P (BB_END (jump_bb_next)))
8857 : /* if jump_bb_next is not empty. */
8858 0 : BB_END (jump_bb) = BB_END (jump_bb_next);
8859 :
8860 1 : if (BB_END (bb) != PREV_INSN (jump))
8861 : /* Then there are instruction after jump that should be placed
8862 : to jump_bb_next. */
8863 : BB_END (jump_bb_next) = BB_END (bb);
8864 : else
8865 : /* Otherwise jump_bb_next is empty. */
8866 0 : BB_END (jump_bb_next) = NEXT_INSN (BB_HEAD (jump_bb_next));
8867 :
8868 : /* To make assertion in move_insn happy. */
8869 1 : BB_END (bb) = PREV_INSN (jump);
8870 :
8871 1 : update_bb_for_insn (jump_bb_next);
8872 1 : }
8873 :
8874 : /* Fix CFG after interblock movement of control_flow_insn_p JUMP. */
8875 : static void
8876 0 : move_block_after_check (rtx_insn *jump)
8877 : {
8878 0 : basic_block bb, jump_bb, jump_bb_next;
8879 0 : vec<edge, va_gc> *t;
8880 :
8881 0 : bb = BLOCK_FOR_INSN (PREV_INSN (jump));
8882 0 : jump_bb = BLOCK_FOR_INSN (jump);
8883 0 : jump_bb_next = jump_bb->next_bb;
8884 :
8885 0 : update_bb_for_insn (jump_bb);
8886 :
8887 0 : gcc_assert (IS_SPECULATION_CHECK_P (jump)
8888 : || IS_SPECULATION_CHECK_P (BB_END (jump_bb_next)));
8889 :
8890 0 : unlink_block (jump_bb_next);
8891 0 : link_block (jump_bb_next, bb);
8892 :
8893 0 : t = bb->succs;
8894 0 : bb->succs = 0;
8895 0 : move_succs (&(jump_bb->succs), bb);
8896 0 : move_succs (&(jump_bb_next->succs), jump_bb);
8897 0 : move_succs (&t, jump_bb_next);
8898 :
8899 0 : df_mark_solutions_dirty ();
8900 :
8901 0 : common_sched_info->fix_recovery_cfg
8902 0 : (bb->index, jump_bb->index, jump_bb_next->index);
8903 0 : }
8904 :
8905 : /* Helper function for move_block_after_check.
8906 : This functions attaches edge vector pointed to by SUCCSP to
8907 : block TO. */
8908 : static void
8909 0 : move_succs (vec<edge, va_gc> **succsp, basic_block to)
8910 : {
8911 0 : edge e;
8912 0 : edge_iterator ei;
8913 :
8914 0 : gcc_assert (to->succs == 0);
8915 :
8916 0 : to->succs = *succsp;
8917 :
8918 0 : FOR_EACH_EDGE (e, ei, to->succs)
8919 0 : e->src = to;
8920 :
8921 0 : *succsp = 0;
8922 0 : }
8923 :
8924 : /* Remove INSN from the instruction stream.
8925 : INSN should have any dependencies. */
8926 : static void
8927 0 : sched_remove_insn (rtx_insn *insn)
8928 : {
8929 0 : sd_finish_insn (insn);
8930 :
8931 0 : change_queue_index (insn, QUEUE_NOWHERE);
8932 0 : current_sched_info->add_remove_insn (insn, 1);
8933 0 : delete_insn (insn);
8934 0 : }
8935 :
8936 : /* Clear priorities of all instructions, that are forward dependent on INSN.
8937 : Store in vector pointed to by ROOTS_PTR insns on which priority () should
8938 : be invoked to initialize all cleared priorities. */
8939 : static void
8940 0 : clear_priorities (rtx_insn *insn, rtx_vec_t *roots_ptr)
8941 : {
8942 0 : sd_iterator_def sd_it;
8943 0 : dep_t dep;
8944 0 : bool insn_is_root_p = true;
8945 :
8946 0 : gcc_assert (QUEUE_INDEX (insn) != QUEUE_SCHEDULED);
8947 :
8948 0 : FOR_EACH_DEP (insn, SD_LIST_BACK, sd_it, dep)
8949 : {
8950 0 : rtx_insn *pro = DEP_PRO (dep);
8951 :
8952 0 : if (INSN_PRIORITY_STATUS (pro) >= 0
8953 0 : && QUEUE_INDEX (insn) != QUEUE_SCHEDULED)
8954 : {
8955 : /* If DEP doesn't contribute to priority then INSN itself should
8956 : be added to priority roots. */
8957 0 : if (contributes_to_priority_p (dep))
8958 0 : insn_is_root_p = false;
8959 :
8960 0 : INSN_PRIORITY_STATUS (pro) = -1;
8961 0 : clear_priorities (pro, roots_ptr);
8962 : }
8963 : }
8964 :
8965 0 : if (insn_is_root_p)
8966 0 : roots_ptr->safe_push (insn);
8967 0 : }
8968 :
8969 : /* Recompute priorities of instructions, whose priorities might have been
8970 : changed. ROOTS is a vector of instructions whose priority computation will
8971 : trigger initialization of all cleared priorities. */
8972 : static void
8973 0 : calc_priorities (const rtx_vec_t &roots)
8974 : {
8975 0 : int i;
8976 0 : rtx_insn *insn;
8977 :
8978 0 : FOR_EACH_VEC_ELT (roots, i, insn)
8979 0 : priority (insn);
8980 0 : }
8981 :
8982 :
8983 : /* Add dependences between JUMP and other instructions in the recovery
8984 : block. INSN is the first insn the recovery block. */
8985 : static void
8986 0 : add_jump_dependencies (rtx_insn *insn, rtx_insn *jump)
8987 : {
8988 0 : do
8989 : {
8990 0 : insn = NEXT_INSN (insn);
8991 0 : if (insn == jump)
8992 : break;
8993 :
8994 0 : if (dep_list_size (insn, SD_LIST_FORW) == 0)
8995 : {
8996 0 : dep_def _new_dep, *new_dep = &_new_dep;
8997 :
8998 0 : init_dep (new_dep, insn, jump, REG_DEP_ANTI);
8999 0 : sd_add_dep (new_dep, false);
9000 : }
9001 : }
9002 : while (1);
9003 :
9004 0 : gcc_assert (!sd_lists_empty_p (jump, SD_LIST_BACK));
9005 0 : }
9006 :
9007 : /* Extend data structures for logical insn UID. */
9008 : void
9009 986306 : sched_extend_luids (void)
9010 : {
9011 986306 : int new_luids_max_uid = get_max_uid () + 1;
9012 :
9013 986306 : sched_luids.safe_grow_cleared (new_luids_max_uid, true);
9014 986306 : }
9015 :
9016 : /* Initialize LUID for INSN. */
9017 : void
9018 138408970 : sched_init_insn_luid (rtx_insn *insn)
9019 : {
9020 138408970 : int i = INSN_P (insn) ? 1 : common_sched_info->luid_for_non_insn (insn);
9021 24078026 : int luid;
9022 :
9023 24078026 : if (i >= 0)
9024 : {
9025 138407484 : luid = sched_max_luid;
9026 138407484 : sched_max_luid += i;
9027 : }
9028 : else
9029 : luid = -1;
9030 :
9031 138408970 : SET_INSN_LUID (insn, luid);
9032 138408970 : }
9033 :
9034 : /* Initialize luids for BBS.
9035 : The hook common_sched_info->luid_for_non_insn () is used to determine
9036 : if notes, labels, etc. need luids. */
9037 : void
9038 984164 : sched_init_luids (const bb_vec_t &bbs)
9039 : {
9040 984164 : int i;
9041 984164 : basic_block bb;
9042 :
9043 984164 : sched_extend_luids ();
9044 12467695 : FOR_EACH_VEC_ELT (bbs, i, bb)
9045 : {
9046 10499367 : rtx_insn *insn;
9047 :
9048 148907031 : FOR_BB_INSNS (bb, insn)
9049 138407664 : sched_init_insn_luid (insn);
9050 : }
9051 984164 : }
9052 :
9053 : /* Free LUIDs. */
9054 : void
9055 984164 : sched_finish_luids (void)
9056 : {
9057 984164 : sched_luids.release ();
9058 984164 : sched_max_luid = 1;
9059 984164 : }
9060 :
9061 : /* Return logical uid of INSN. Helpful while debugging. */
9062 : int
9063 0 : insn_luid (rtx_insn *insn)
9064 : {
9065 0 : return INSN_LUID (insn);
9066 : }
9067 :
9068 : /* Extend per insn data in the target. */
9069 : void
9070 1971149 : sched_extend_target (void)
9071 : {
9072 1971149 : if (targetm.sched.h_i_d_extended)
9073 0 : targetm.sched.h_i_d_extended ();
9074 1971149 : }
9075 :
9076 : /* Extend global scheduler structures (those, that live across calls to
9077 : schedule_block) to include information about just emitted INSN. */
9078 : static void
9079 984164 : extend_h_i_d (void)
9080 : {
9081 984164 : int reserve = (get_max_uid () + 1 - h_i_d.length ());
9082 984164 : if (reserve > 0
9083 1967751 : && ! h_i_d.space (reserve))
9084 : {
9085 983587 : h_i_d.safe_grow_cleared (3U * get_max_uid () / 2, true);
9086 983587 : sched_extend_target ();
9087 : }
9088 984164 : }
9089 :
9090 : /* Initialize h_i_d entry of the INSN with default values.
9091 : Values, that are not explicitly initialized here, hold zero. */
9092 : static void
9093 138407664 : init_h_i_d (rtx_insn *insn)
9094 : {
9095 138407664 : if (INSN_LUID (insn) > 0)
9096 : {
9097 138406178 : INSN_COST (insn) = -1;
9098 138406178 : QUEUE_INDEX (insn) = QUEUE_NOWHERE;
9099 138406178 : INSN_TICK (insn) = INVALID_TICK;
9100 138406178 : INSN_EXACT_TICK (insn) = INVALID_TICK;
9101 138406178 : INTER_TICK (insn) = INVALID_TICK;
9102 138406178 : TODO_SPEC (insn) = HARD_DEP;
9103 138406178 : INSN_AUTOPREF_MULTIPASS_DATA (insn)[0].status
9104 138406178 : = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED;
9105 138406178 : INSN_AUTOPREF_MULTIPASS_DATA (insn)[1].status
9106 138406178 : = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED;
9107 : }
9108 138407664 : }
9109 :
9110 : /* Initialize haifa_insn_data for BBS. */
9111 : void
9112 984164 : haifa_init_h_i_d (const bb_vec_t &bbs)
9113 : {
9114 984164 : int i;
9115 984164 : basic_block bb;
9116 :
9117 984164 : extend_h_i_d ();
9118 12467695 : FOR_EACH_VEC_ELT (bbs, i, bb)
9119 : {
9120 10499367 : rtx_insn *insn;
9121 :
9122 148907031 : FOR_BB_INSNS (bb, insn)
9123 138407664 : init_h_i_d (insn);
9124 : }
9125 984164 : }
9126 :
9127 : /* Finalize haifa_insn_data. */
9128 : void
9129 983586 : haifa_finish_h_i_d (void)
9130 : {
9131 983586 : int i;
9132 983586 : haifa_insn_data_t data;
9133 983586 : reg_use_data *use, *next_use;
9134 983586 : reg_set_data *set, *next_set;
9135 :
9136 321565426 : FOR_EACH_VEC_ELT (h_i_d, i, data)
9137 : {
9138 320581840 : free (data->max_reg_pressure);
9139 320581840 : free (data->reg_pressure);
9140 320584330 : for (use = data->reg_use_list; use != NULL; use = next_use)
9141 : {
9142 2490 : next_use = use->next_insn_use;
9143 2490 : free (use);
9144 : }
9145 320584164 : for (set = data->reg_set_list; set != NULL; set = next_set)
9146 : {
9147 2324 : next_set = set->next_insn_set;
9148 2324 : free (set);
9149 : }
9150 :
9151 : }
9152 983586 : h_i_d.release ();
9153 983586 : }
9154 :
9155 : /* Init data for the new insn INSN. */
9156 : static void
9157 0 : haifa_init_insn (rtx_insn *insn)
9158 : {
9159 0 : gcc_assert (insn != NULL);
9160 :
9161 0 : sched_extend_luids ();
9162 0 : sched_init_insn_luid (insn);
9163 0 : sched_extend_target ();
9164 0 : sched_deps_init (false);
9165 0 : extend_h_i_d ();
9166 0 : init_h_i_d (insn);
9167 :
9168 0 : if (adding_bb_to_current_region_p)
9169 : {
9170 0 : sd_init_insn (insn);
9171 :
9172 : /* Extend dependency caches by one element. */
9173 0 : extend_dependency_caches (1, false);
9174 : }
9175 0 : if (sched_pressure != SCHED_PRESSURE_NONE)
9176 0 : init_insn_reg_pressure_info (insn);
9177 0 : }
9178 :
9179 : /* Init data for the new basic block BB which comes after AFTER. */
9180 : static void
9181 1 : haifa_init_only_bb (basic_block bb, basic_block after)
9182 : {
9183 1 : gcc_assert (bb != NULL);
9184 :
9185 1 : sched_init_bbs ();
9186 :
9187 1 : if (common_sched_info->add_block)
9188 : /* This changes only data structures of the front-end. */
9189 1 : common_sched_info->add_block (bb, after);
9190 1 : }
9191 :
9192 : /* A generic version of sched_split_block (). */
9193 : basic_block
9194 68 : sched_split_block_1 (basic_block first_bb, rtx after)
9195 : {
9196 68 : edge e;
9197 :
9198 68 : e = split_block (first_bb, after);
9199 68 : gcc_assert (e->src == first_bb);
9200 :
9201 : /* sched_split_block emits note if *check == BB_END. Probably it
9202 : is better to rip that note off. */
9203 :
9204 68 : return e->dest;
9205 : }
9206 :
9207 : /* A generic version of sched_create_empty_bb (). */
9208 : basic_block
9209 0 : sched_create_empty_bb_1 (basic_block after)
9210 : {
9211 0 : return create_empty_bb (after);
9212 : }
9213 :
9214 : /* Insert PAT as an INSN into the schedule and update the necessary data
9215 : structures to account for it. */
9216 : rtx_insn *
9217 0 : sched_emit_insn (rtx pat)
9218 : {
9219 0 : rtx_insn *insn = emit_insn_before (pat, first_nonscheduled_insn ());
9220 0 : haifa_init_insn (insn);
9221 :
9222 0 : if (current_sched_info->add_remove_insn)
9223 0 : current_sched_info->add_remove_insn (insn, 0);
9224 :
9225 0 : (*current_sched_info->begin_schedule_ready) (insn);
9226 0 : scheduled_insns.safe_push (insn);
9227 :
9228 0 : last_scheduled_insn = insn;
9229 0 : return insn;
9230 : }
9231 :
9232 : /* This function returns a candidate satisfying dispatch constraints from
9233 : the ready list. */
9234 :
9235 : static rtx_insn *
9236 16 : ready_remove_first_dispatch (struct ready_list *ready)
9237 : {
9238 16 : int i;
9239 16 : rtx_insn *insn = ready_element (ready, 0);
9240 :
9241 16 : if (ready->n_ready == 1
9242 0 : || !INSN_P (insn)
9243 0 : || INSN_CODE (insn) < 0
9244 0 : || !active_insn_p (insn)
9245 16 : || targetm.sched.dispatch (insn, FITS_DISPATCH_WINDOW))
9246 16 : return ready_remove_first (ready);
9247 :
9248 0 : for (i = 1; i < ready->n_ready; i++)
9249 : {
9250 0 : insn = ready_element (ready, i);
9251 :
9252 0 : if (!INSN_P (insn)
9253 0 : || INSN_CODE (insn) < 0
9254 0 : || !active_insn_p (insn))
9255 0 : continue;
9256 :
9257 0 : if (targetm.sched.dispatch (insn, FITS_DISPATCH_WINDOW))
9258 : {
9259 : /* Return ith element of ready. */
9260 0 : insn = ready_remove (ready, i);
9261 0 : return insn;
9262 : }
9263 : }
9264 :
9265 0 : if (targetm.sched.dispatch (NULL, DISPATCH_VIOLATION))
9266 0 : return ready_remove_first (ready);
9267 :
9268 0 : for (i = 1; i < ready->n_ready; i++)
9269 : {
9270 0 : insn = ready_element (ready, i);
9271 :
9272 0 : if (!INSN_P (insn)
9273 0 : || INSN_CODE (insn) < 0
9274 0 : || !active_insn_p (insn))
9275 0 : continue;
9276 :
9277 : /* Return i-th element of ready. */
9278 0 : if (targetm.sched.dispatch (insn, IS_CMP))
9279 0 : return ready_remove (ready, i);
9280 : }
9281 :
9282 0 : return ready_remove_first (ready);
9283 : }
9284 :
9285 : /* Get number of ready insn in the ready list. */
9286 :
9287 : int
9288 0 : number_in_ready (void)
9289 : {
9290 0 : return ready.n_ready;
9291 : }
9292 :
9293 : /* Get number of ready's in the ready list. */
9294 :
9295 : rtx_insn *
9296 278300592 : get_ready_element (int i)
9297 : {
9298 278300592 : return ready_element (&ready, i);
9299 : }
9300 :
9301 : #endif /* INSN_SCHEDULING */
|
