daemons/controld/controld

/* */
This source file includes following definitions.
init_dotfile
do_fsa_action
do_log
controld_init_fsa_trigger
controld_destroy_fsa_trigger
controld_trigger_fsa_as
s_crmd_fsa
s_crmd_fsa_actions
log_fsa_input
check_join_counts
do_state_transition
   1 /*
   2  * Copyright 2004-2022 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU General Public License version 2
   7  * or later (GPLv2+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <sys/param.h>
  13 #include <stdio.h>
  14 #include <stdint.h>                 // uint64_t
  15 #include <string.h>
  16 #include <time.h>
  17 
  18 #include <crm/crm.h>
  19 #include <crm/lrmd.h>
  20 #include <crm/cib.h>
  21 #include <crm/msg_xml.h>
  22 #include <crm/common/xml.h>
  23 #include <crm/cluster/election_internal.h>
  24 #include <crm/cluster.h>
  25 
  26 #include <pacemaker-controld.h>
  27 
  28 //! Triggers an FSA invocation
  29 static crm_trigger_t *fsa_trigger = NULL;
  30 
  31 #define DOT_PREFIX "actions:trace: "
  32 #define do_dot_log(fmt, args...)     crm_trace( fmt, ##args)
  33 
  34 static void do_state_transition(enum crmd_fsa_state cur_state,
  35                                 enum crmd_fsa_state next_state,
  36                                 fsa_data_t *msg_data);
  37 
  38 void s_crmd_fsa_actions(fsa_data_t * fsa_data);
  39 void log_fsa_input(fsa_data_t * stored_msg);
  40 void init_dotfile(void);
  41 
  42 void
  43 init_dotfile(void)
     /*  */
  44 {
  45     do_dot_log(DOT_PREFIX "digraph \"g\" {");
  46     do_dot_log(DOT_PREFIX "     size = \"30,30\"");
  47     do_dot_log(DOT_PREFIX "     graph [");
  48     do_dot_log(DOT_PREFIX "             fontsize = \"12\"");
  49     do_dot_log(DOT_PREFIX "             fontname = \"Times-Roman\"");
  50     do_dot_log(DOT_PREFIX "             fontcolor = \"black\"");
  51     do_dot_log(DOT_PREFIX "             bb = \"0,0,398.922306,478.927856\"");
  52     do_dot_log(DOT_PREFIX "             color = \"black\"");
  53     do_dot_log(DOT_PREFIX "     ]");
  54     do_dot_log(DOT_PREFIX "     node [");
  55     do_dot_log(DOT_PREFIX "             fontsize = \"12\"");
  56     do_dot_log(DOT_PREFIX "             fontname = \"Times-Roman\"");
  57     do_dot_log(DOT_PREFIX "             fontcolor = \"black\"");
  58     do_dot_log(DOT_PREFIX "             shape = \"ellipse\"");
  59     do_dot_log(DOT_PREFIX "             color = \"black\"");
  60     do_dot_log(DOT_PREFIX "     ]");
  61     do_dot_log(DOT_PREFIX "     edge [");
  62     do_dot_log(DOT_PREFIX "             fontsize = \"12\"");
  63     do_dot_log(DOT_PREFIX "             fontname = \"Times-Roman\"");
  64     do_dot_log(DOT_PREFIX "             fontcolor = \"black\"");
  65     do_dot_log(DOT_PREFIX "             color = \"black\"");
  66     do_dot_log(DOT_PREFIX "     ]");
  67     do_dot_log(DOT_PREFIX "// special nodes");
  68     do_dot_log(DOT_PREFIX "     \"S_PENDING\" ");
  69     do_dot_log(DOT_PREFIX "     [");
  70     do_dot_log(DOT_PREFIX "      color = \"blue\"");
  71     do_dot_log(DOT_PREFIX "      fontcolor = \"blue\"");
  72     do_dot_log(DOT_PREFIX "      ]");
  73     do_dot_log(DOT_PREFIX "     \"S_TERMINATE\" ");
  74     do_dot_log(DOT_PREFIX "     [");
  75     do_dot_log(DOT_PREFIX "      color = \"red\"");
  76     do_dot_log(DOT_PREFIX "      fontcolor = \"red\"");
  77     do_dot_log(DOT_PREFIX "      ]");
  78     do_dot_log(DOT_PREFIX "// DC only nodes");
  79     do_dot_log(DOT_PREFIX "     \"S_INTEGRATION\" [ fontcolor = \"green\" ]");
  80     do_dot_log(DOT_PREFIX "     \"S_POLICY_ENGINE\" [ fontcolor = \"green\" ]");
  81     do_dot_log(DOT_PREFIX "     \"S_TRANSITION_ENGINE\" [ fontcolor = \"green\" ]");
  82     do_dot_log(DOT_PREFIX "     \"S_RELEASE_DC\" [ fontcolor = \"green\" ]");
  83     do_dot_log(DOT_PREFIX "     \"S_IDLE\" [ fontcolor = \"green\" ]");
  84 }
  85 
  86 static void
  87 do_fsa_action(fsa_data_t * fsa_data, long long an_action,
     /*  */
  88               void (*function) (long long action,
  89                                 enum crmd_fsa_cause cause,
  90                                 enum crmd_fsa_state cur_state,
  91                                 enum crmd_fsa_input cur_input, fsa_data_t * msg_data))
  92 {
  93     controld_clear_fsa_action_flags(an_action);
  94     crm_trace(DOT_PREFIX "\t// %s", fsa_action2string(an_action));
  95     function(an_action, fsa_data->fsa_cause, controld_globals.fsa_state,
  96              fsa_data->fsa_input, fsa_data);
  97 }
  98 
  99 static const uint64_t startup_actions =
 100     A_STARTUP | A_CIB_START | A_LRM_CONNECT | A_HA_CONNECT | A_READCONFIG |
 101     A_STARTED | A_CL_JOIN_QUERY;
 102 
 103 // A_LOG, A_WARN, A_ERROR
 104 void
 105 do_log(long long action, enum crmd_fsa_cause cause,
     /*  */
 106        enum crmd_fsa_state cur_state,
 107        enum crmd_fsa_input current_input, fsa_data_t *msg_data)
 108 {
 109     unsigned log_type = LOG_TRACE;
 110 
 111     if (action & A_LOG) {
 112         log_type = LOG_INFO;
 113     } else if (action & A_WARN) {
 114         log_type = LOG_WARNING;
 115     } else if (action & A_ERROR) {
 116         log_type = LOG_ERR;
 117     }
 118 
 119     do_crm_log(log_type, "Input %s received in state %s from %s",
 120                fsa_input2string(msg_data->fsa_input),
 121                fsa_state2string(cur_state), msg_data->origin);
 122 
 123     if (msg_data->data_type == fsa_dt_ha_msg) {
 124         ha_msg_input_t *input = fsa_typed_data(msg_data->data_type);
 125 
 126         crm_log_xml_debug(input->msg, __func__);
 127 
 128     } else if (msg_data->data_type == fsa_dt_xml) {
 129         xmlNode *input = fsa_typed_data(msg_data->data_type);
 130 
 131         crm_log_xml_debug(input, __func__);
 132 
 133     } else if (msg_data->data_type == fsa_dt_lrm) {
 134         lrmd_event_data_t *input = fsa_typed_data(msg_data->data_type);
 135 
 136         do_crm_log(log_type,
 137                    "Resource %s: Call ID %d returned %d (%d)."
 138                    "  New status if rc=0: %s",
 139                    input->rsc_id, input->call_id, input->rc,
 140                    input->op_status, (char *)input->user_data);
 141     }
 142 }
 143 
 144 /*!
 145  * \internal
 146  * \brief Initialize the FSA trigger
 147  */
 148 void
 149 controld_init_fsa_trigger(void)
     /*  */
 150 {
 151     fsa_trigger = mainloop_add_trigger(G_PRIORITY_HIGH, crm_fsa_trigger, NULL);
 152 }
 153 
 154 /*!
 155  * \internal
 156  * \brief Destroy the FSA trigger
 157  */
 158 void
 159 controld_destroy_fsa_trigger(void)
     /*  */
 160 {
 161     // This basically will not work, since mainloop has a reference to it
 162     mainloop_destroy_trigger(fsa_trigger);
 163     fsa_trigger = NULL;
 164 }
 165 
 166 /*!
 167  * \internal
 168  * \brief Trigger an FSA invocation
 169  *
 170  * \param[in] fn    Calling function name
 171  * \param[in] line  Line number where call occurred
 172  */
 173 void
 174 controld_trigger_fsa_as(const char *fn, int line)
     /*  */
 175 {
 176     if (fsa_trigger != NULL) {
 177         crm_trace("%s:%d - Triggered FSA invocation", fn, line);
 178         mainloop_set_trigger(fsa_trigger);
 179     }
 180 }
 181 
 182 enum crmd_fsa_state
 183 s_crmd_fsa(enum crmd_fsa_cause cause)
     /*  */
 184 {
 185     controld_globals_t *globals = &controld_globals;
 186     fsa_data_t *fsa_data = NULL;
 187     uint64_t register_copy = controld_globals.fsa_input_register;
 188     uint64_t new_actions = A_NOTHING;
 189     enum crmd_fsa_state last_state;
 190 
 191     crm_trace("FSA invoked with Cause: %s\tState: %s",
 192               fsa_cause2string(cause),
 193               fsa_state2string(globals->fsa_state));
 194 
 195     fsa_dump_actions(controld_globals.fsa_actions, "Initial");
 196 
 197     controld_clear_global_flags(controld_fsa_is_stalled);
 198     if ((controld_globals.fsa_message_queue == NULL)
 199         && (controld_globals.fsa_actions != A_NOTHING)) {
 200         /* fake the first message so we can get into the loop */
 201         fsa_data = calloc(1, sizeof(fsa_data_t));
 202         fsa_data->fsa_input = I_NULL;
 203         fsa_data->fsa_cause = C_FSA_INTERNAL;
 204         fsa_data->origin = __func__;
 205         fsa_data->data_type = fsa_dt_none;
 206         controld_globals.fsa_message_queue
 207             = g_list_append(controld_globals.fsa_message_queue, fsa_data);
 208         fsa_data = NULL;
 209     }
 210     while ((controld_globals.fsa_message_queue != NULL)
 211            && !pcmk_is_set(controld_globals.flags, controld_fsa_is_stalled)) {
 212         crm_trace("Checking messages (%d remaining)",
 213                   g_list_length(controld_globals.fsa_message_queue));
 214 
 215         fsa_data = get_message();
 216         if(fsa_data == NULL) {
 217             continue;
 218         }
 219 
 220         log_fsa_input(fsa_data);
 221 
 222         /* add any actions back to the queue */
 223         controld_set_fsa_action_flags(fsa_data->actions);
 224         fsa_dump_actions(fsa_data->actions, "Restored actions");
 225 
 226         /* get the next batch of actions */
 227         new_actions = controld_fsa_get_action(fsa_data->fsa_input);
 228         controld_set_fsa_action_flags(new_actions);
 229         fsa_dump_actions(new_actions, "New actions");
 230 
 231         if (fsa_data->fsa_input != I_NULL && fsa_data->fsa_input != I_ROUTER) {
 232             crm_debug("Processing %s: [ state=%s cause=%s origin=%s ]",
 233                       fsa_input2string(fsa_data->fsa_input),
 234                       fsa_state2string(globals->fsa_state),
 235                       fsa_cause2string(fsa_data->fsa_cause), fsa_data->origin);
 236         }
 237 
 238         /* logging : *before* the state is changed */
 239         if (pcmk_is_set(controld_globals.fsa_actions, A_ERROR)) {
 240             do_fsa_action(fsa_data, A_ERROR, do_log);
 241         }
 242         if (pcmk_is_set(controld_globals.fsa_actions, A_WARN)) {
 243             do_fsa_action(fsa_data, A_WARN, do_log);
 244         }
 245         if (pcmk_is_set(controld_globals.fsa_actions, A_LOG)) {
 246             do_fsa_action(fsa_data, A_LOG, do_log);
 247         }
 248 
 249         /* update state variables */
 250         last_state = globals->fsa_state;
 251         globals->fsa_state = controld_fsa_get_next_state(fsa_data->fsa_input);
 252 
 253         /*
 254          * Remove certain actions during shutdown
 255          */
 256         if ((globals->fsa_state == S_STOPPING)
 257             || pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) {
 258             controld_clear_fsa_action_flags(startup_actions);
 259         }
 260 
 261         /*
 262          * Hook for change of state.
 263          * Allows actions to be added or removed when entering a state
 264          */
 265         if (last_state != globals->fsa_state) {
 266             do_state_transition(last_state, globals->fsa_state, fsa_data);
 267         } else {
 268             do_dot_log(DOT_PREFIX "\t// FSA input: State=%s \tCause=%s"
 269                        " \tInput=%s \tOrigin=%s() \tid=%d",
 270                        fsa_state2string(globals->fsa_state),
 271                        fsa_cause2string(fsa_data->fsa_cause),
 272                        fsa_input2string(fsa_data->fsa_input), fsa_data->origin, fsa_data->id);
 273         }
 274 
 275         /* start doing things... */
 276         s_crmd_fsa_actions(fsa_data);
 277         delete_fsa_input(fsa_data);
 278         fsa_data = NULL;
 279     }
 280 
 281     if ((controld_globals.fsa_message_queue != NULL)
 282         || (controld_globals.fsa_actions != A_NOTHING)
 283         || pcmk_is_set(controld_globals.flags, controld_fsa_is_stalled)) {
 284 
 285         crm_debug("Exiting the FSA: queue=%d, fsa_actions=%#llx, stalled=%s",
 286                   g_list_length(controld_globals.fsa_message_queue),
 287                   (unsigned long long) controld_globals.fsa_actions,
 288                   pcmk__btoa(pcmk_is_set(controld_globals.flags,
 289                                          controld_fsa_is_stalled)));
 290     } else {
 291         crm_trace("Exiting the FSA");
 292     }
 293 
 294     /* cleanup inputs? */
 295     if (register_copy != controld_globals.fsa_input_register) {
 296         uint64_t same = register_copy & controld_globals.fsa_input_register;
 297 
 298         fsa_dump_inputs(LOG_DEBUG, "Added",
 299                         controld_globals.fsa_input_register ^ same);
 300         fsa_dump_inputs(LOG_DEBUG, "Removed", register_copy ^ same);
 301     }
 302 
 303     fsa_dump_actions(controld_globals.fsa_actions, "Remaining");
 304     fsa_dump_queue(LOG_DEBUG);
 305 
 306     return globals->fsa_state;
 307 }
 308 
 309 void
 310 s_crmd_fsa_actions(fsa_data_t * fsa_data)
     /*  */
 311 {
 312     /*
 313      * Process actions in order of priority but do only one
 314      * action at a time to avoid complicating the ordering.
 315      */
 316     CRM_CHECK(fsa_data != NULL, return);
 317     while ((controld_globals.fsa_actions != A_NOTHING)
 318            && !pcmk_is_set(controld_globals.flags, controld_fsa_is_stalled)) {
 319 
 320         /* regular action processing in order of action priority
 321          *
 322          * Make sure all actions that connect to required systems
 323          * are performed first
 324          */
 325         if (pcmk_is_set(controld_globals.fsa_actions, A_ERROR)) {
 326             do_fsa_action(fsa_data, A_ERROR, do_log);
 327         } else if (pcmk_is_set(controld_globals.fsa_actions, A_WARN)) {
 328             do_fsa_action(fsa_data, A_WARN, do_log);
 329         } else if (pcmk_is_set(controld_globals.fsa_actions, A_LOG)) {
 330             do_fsa_action(fsa_data, A_LOG, do_log);
 331 
 332             /* get out of here NOW! before anything worse happens */
 333         } else if (pcmk_is_set(controld_globals.fsa_actions, A_EXIT_1)) {
 334             do_fsa_action(fsa_data, A_EXIT_1, do_exit);
 335 
 336             /* sub-system restart */
 337         } else if (pcmk_all_flags_set(controld_globals.fsa_actions,
 338                                       O_LRM_RECONNECT)) {
 339             do_fsa_action(fsa_data, O_LRM_RECONNECT, do_lrm_control);
 340 
 341         } else if (pcmk_all_flags_set(controld_globals.fsa_actions,
 342                                       O_CIB_RESTART)) {
 343             do_fsa_action(fsa_data, O_CIB_RESTART, do_cib_control);
 344 
 345         } else if (pcmk_all_flags_set(controld_globals.fsa_actions,
 346                                       O_PE_RESTART)) {
 347             do_fsa_action(fsa_data, O_PE_RESTART, do_pe_control);
 348 
 349         } else if (pcmk_all_flags_set(controld_globals.fsa_actions,
 350                                       O_TE_RESTART)) {
 351             do_fsa_action(fsa_data, O_TE_RESTART, do_te_control);
 352 
 353             /* essential start tasks */
 354         } else if (pcmk_is_set(controld_globals.fsa_actions, A_STARTUP)) {
 355             do_fsa_action(fsa_data, A_STARTUP, do_startup);
 356         } else if (pcmk_is_set(controld_globals.fsa_actions, A_CIB_START)) {
 357             do_fsa_action(fsa_data, A_CIB_START, do_cib_control);
 358         } else if (pcmk_is_set(controld_globals.fsa_actions, A_HA_CONNECT)) {
 359             do_fsa_action(fsa_data, A_HA_CONNECT, do_ha_control);
 360         } else if (pcmk_is_set(controld_globals.fsa_actions, A_READCONFIG)) {
 361             do_fsa_action(fsa_data, A_READCONFIG, do_read_config);
 362 
 363             /* sub-system start/connect */
 364         } else if (pcmk_is_set(controld_globals.fsa_actions, A_LRM_CONNECT)) {
 365             do_fsa_action(fsa_data, A_LRM_CONNECT, do_lrm_control);
 366         } else if (pcmk_is_set(controld_globals.fsa_actions, A_TE_START)) {
 367             do_fsa_action(fsa_data, A_TE_START, do_te_control);
 368         } else if (pcmk_is_set(controld_globals.fsa_actions, A_PE_START)) {
 369             do_fsa_action(fsa_data, A_PE_START, do_pe_control);
 370 
 371             /* Timers */
 372         } else if (pcmk_is_set(controld_globals.fsa_actions, A_DC_TIMER_STOP)) {
 373             do_fsa_action(fsa_data, A_DC_TIMER_STOP, do_timer_control);
 374         } else if (pcmk_is_set(controld_globals.fsa_actions,
 375                                A_INTEGRATE_TIMER_STOP)) {
 376             do_fsa_action(fsa_data, A_INTEGRATE_TIMER_STOP, do_timer_control);
 377         } else if (pcmk_is_set(controld_globals.fsa_actions,
 378                                A_INTEGRATE_TIMER_START)) {
 379             do_fsa_action(fsa_data, A_INTEGRATE_TIMER_START, do_timer_control);
 380         } else if (pcmk_is_set(controld_globals.fsa_actions,
 381                                A_FINALIZE_TIMER_STOP)) {
 382             do_fsa_action(fsa_data, A_FINALIZE_TIMER_STOP, do_timer_control);
 383         } else if (pcmk_is_set(controld_globals.fsa_actions,
 384                                A_FINALIZE_TIMER_START)) {
 385             do_fsa_action(fsa_data, A_FINALIZE_TIMER_START, do_timer_control);
 386 
 387             /*
 388              * Highest priority actions
 389              */
 390         } else if (pcmk_is_set(controld_globals.fsa_actions, A_MSG_ROUTE)) {
 391             do_fsa_action(fsa_data, A_MSG_ROUTE, do_msg_route);
 392         } else if (pcmk_is_set(controld_globals.fsa_actions, A_RECOVER)) {
 393             do_fsa_action(fsa_data, A_RECOVER, do_recover);
 394         } else if (pcmk_is_set(controld_globals.fsa_actions,
 395                                A_CL_JOIN_RESULT)) {
 396             do_fsa_action(fsa_data, A_CL_JOIN_RESULT,
 397                           do_cl_join_finalize_respond);
 398 
 399         } else if (pcmk_is_set(controld_globals.fsa_actions,
 400                                A_CL_JOIN_REQUEST)) {
 401             do_fsa_action(fsa_data, A_CL_JOIN_REQUEST,
 402                           do_cl_join_offer_respond);
 403 
 404         } else if (pcmk_is_set(controld_globals.fsa_actions, A_SHUTDOWN_REQ)) {
 405             do_fsa_action(fsa_data, A_SHUTDOWN_REQ, do_shutdown_req);
 406         } else if (pcmk_is_set(controld_globals.fsa_actions, A_ELECTION_VOTE)) {
 407             do_fsa_action(fsa_data, A_ELECTION_VOTE, do_election_vote);
 408         } else if (pcmk_is_set(controld_globals.fsa_actions,
 409                                A_ELECTION_COUNT)) {
 410             do_fsa_action(fsa_data, A_ELECTION_COUNT, do_election_count_vote);
 411 
 412         } else if (pcmk_is_set(controld_globals.fsa_actions, A_LRM_EVENT)) {
 413             do_fsa_action(fsa_data, A_LRM_EVENT, do_lrm_event);
 414 
 415             /*
 416              * High priority actions
 417              */
 418         } else if (pcmk_is_set(controld_globals.fsa_actions, A_STARTED)) {
 419             do_fsa_action(fsa_data, A_STARTED, do_started);
 420         } else if (pcmk_is_set(controld_globals.fsa_actions, A_CL_JOIN_QUERY)) {
 421             do_fsa_action(fsa_data, A_CL_JOIN_QUERY, do_cl_join_query);
 422         } else if (pcmk_is_set(controld_globals.fsa_actions,
 423                                A_DC_TIMER_START)) {
 424             do_fsa_action(fsa_data, A_DC_TIMER_START, do_timer_control);
 425 
 426             /*
 427              * Medium priority actions
 428              * - Membership
 429              */
 430         } else if (pcmk_is_set(controld_globals.fsa_actions, A_DC_TAKEOVER)) {
 431             do_fsa_action(fsa_data, A_DC_TAKEOVER, do_dc_takeover);
 432         } else if (pcmk_is_set(controld_globals.fsa_actions, A_DC_RELEASE)) {
 433             do_fsa_action(fsa_data, A_DC_RELEASE, do_dc_release);
 434         } else if (pcmk_is_set(controld_globals.fsa_actions, A_DC_JOIN_FINAL)) {
 435             do_fsa_action(fsa_data, A_DC_JOIN_FINAL, do_dc_join_final);
 436         } else if (pcmk_is_set(controld_globals.fsa_actions,
 437                                A_ELECTION_CHECK)) {
 438             do_fsa_action(fsa_data, A_ELECTION_CHECK, do_election_check);
 439 
 440         } else if (pcmk_is_set(controld_globals.fsa_actions,
 441                                A_ELECTION_START)) {
 442             do_fsa_action(fsa_data, A_ELECTION_START, do_election_vote);
 443 
 444         } else if (pcmk_is_set(controld_globals.fsa_actions,
 445                                A_DC_JOIN_OFFER_ALL)) {
 446             do_fsa_action(fsa_data, A_DC_JOIN_OFFER_ALL, do_dc_join_offer_all);
 447 
 448         } else if (pcmk_is_set(controld_globals.fsa_actions,
 449                                A_DC_JOIN_OFFER_ONE)) {
 450             do_fsa_action(fsa_data, A_DC_JOIN_OFFER_ONE, do_dc_join_offer_one);
 451 
 452         } else if (pcmk_is_set(controld_globals.fsa_actions,
 453                                A_DC_JOIN_PROCESS_REQ)) {
 454             do_fsa_action(fsa_data, A_DC_JOIN_PROCESS_REQ,
 455                           do_dc_join_filter_offer);
 456 
 457         } else if (pcmk_is_set(controld_globals.fsa_actions,
 458                                A_DC_JOIN_PROCESS_ACK)) {
 459             do_fsa_action(fsa_data, A_DC_JOIN_PROCESS_ACK, do_dc_join_ack);
 460 
 461         } else if (pcmk_is_set(controld_globals.fsa_actions,
 462                                A_DC_JOIN_FINALIZE)) {
 463             do_fsa_action(fsa_data, A_DC_JOIN_FINALIZE, do_dc_join_finalize);
 464 
 465         } else if (pcmk_is_set(controld_globals.fsa_actions,
 466                                A_CL_JOIN_ANNOUNCE)) {
 467             do_fsa_action(fsa_data, A_CL_JOIN_ANNOUNCE, do_cl_join_announce);
 468 
 469             /*
 470              * Low(er) priority actions
 471              * Make sure the CIB is always updated before invoking the
 472              * scheduler, and the scheduler before the transition engine.
 473              */
 474         } else if (pcmk_is_set(controld_globals.fsa_actions, A_TE_HALT)) {
 475             do_fsa_action(fsa_data, A_TE_HALT, do_te_invoke);
 476         } else if (pcmk_is_set(controld_globals.fsa_actions, A_TE_CANCEL)) {
 477             do_fsa_action(fsa_data, A_TE_CANCEL, do_te_invoke);
 478         } else if (pcmk_is_set(controld_globals.fsa_actions, A_LRM_INVOKE)) {
 479             do_fsa_action(fsa_data, A_LRM_INVOKE, do_lrm_invoke);
 480         } else if (pcmk_is_set(controld_globals.fsa_actions, A_PE_INVOKE)) {
 481             do_fsa_action(fsa_data, A_PE_INVOKE, do_pe_invoke);
 482         } else if (pcmk_is_set(controld_globals.fsa_actions, A_TE_INVOKE)) {
 483             do_fsa_action(fsa_data, A_TE_INVOKE, do_te_invoke);
 484 
 485             /* Shutdown actions */
 486         } else if (pcmk_is_set(controld_globals.fsa_actions, A_DC_RELEASED)) {
 487             do_fsa_action(fsa_data, A_DC_RELEASED, do_dc_release);
 488         } else if (pcmk_is_set(controld_globals.fsa_actions, A_PE_STOP)) {
 489             do_fsa_action(fsa_data, A_PE_STOP, do_pe_control);
 490         } else if (pcmk_is_set(controld_globals.fsa_actions, A_TE_STOP)) {
 491             do_fsa_action(fsa_data, A_TE_STOP, do_te_control);
 492         } else if (pcmk_is_set(controld_globals.fsa_actions, A_SHUTDOWN)) {
 493             do_fsa_action(fsa_data, A_SHUTDOWN, do_shutdown);
 494         } else if (pcmk_is_set(controld_globals.fsa_actions,
 495                                A_LRM_DISCONNECT)) {
 496             do_fsa_action(fsa_data, A_LRM_DISCONNECT, do_lrm_control);
 497 
 498         } else if (pcmk_is_set(controld_globals.fsa_actions, A_HA_DISCONNECT)) {
 499             do_fsa_action(fsa_data, A_HA_DISCONNECT, do_ha_control);
 500         } else if (pcmk_is_set(controld_globals.fsa_actions, A_CIB_STOP)) {
 501             do_fsa_action(fsa_data, A_CIB_STOP, do_cib_control);
 502         } else if (pcmk_is_set(controld_globals.fsa_actions, A_STOP)) {
 503             do_fsa_action(fsa_data, A_STOP, do_stop);
 504 
 505             /* exit gracefully */
 506         } else if (pcmk_is_set(controld_globals.fsa_actions, A_EXIT_0)) {
 507             do_fsa_action(fsa_data, A_EXIT_0, do_exit);
 508 
 509             /* Error checking and reporting */
 510         } else {
 511             crm_err("Action %s not supported "CRM_XS" %#llx",
 512                     fsa_action2string(controld_globals.fsa_actions),
 513                     (unsigned long long) controld_globals.fsa_actions);
 514             register_fsa_error_adv(C_FSA_INTERNAL, I_ERROR, fsa_data, NULL,
 515                                    __func__);
 516         }
 517     }
 518 }
 519 
 520 void
 521 log_fsa_input(fsa_data_t * stored_msg)
     /*  */
 522 {
 523     CRM_ASSERT(stored_msg);
 524     crm_trace("Processing queued input %d", stored_msg->id);
 525     if (stored_msg->fsa_cause == C_LRM_OP_CALLBACK) {
 526         crm_trace("FSA processing LRM callback from %s", stored_msg->origin);
 527 
 528     } else if (stored_msg->data == NULL) {
 529         crm_trace("FSA processing input from %s", stored_msg->origin);
 530 
 531     } else {
 532         ha_msg_input_t *ha_input = fsa_typed_data_adv(stored_msg, fsa_dt_ha_msg,
 533                                                       __func__);
 534 
 535         crm_trace("FSA processing XML message from %s", stored_msg->origin);
 536         crm_log_xml_trace(ha_input->xml, "FSA message data");
 537     }
 538 }
 539 
 540 static void
 541 check_join_counts(fsa_data_t *msg_data)
     /*  */
 542 {
 543     int count;
 544     guint npeers;
 545 
 546     count = crmd_join_phase_count(crm_join_finalized);
 547     if (count > 0) {
 548         crm_err("%d cluster node%s failed to confirm join",
 549                 count, pcmk__plural_s(count));
 550         crmd_join_phase_log(LOG_NOTICE);
 551         return;
 552     }
 553 
 554     npeers = crm_active_peers();
 555     count = crmd_join_phase_count(crm_join_confirmed);
 556     if (count == npeers) {
 557         if (npeers == 1) {
 558             crm_debug("Sole active cluster node is fully joined");
 559         } else {
 560             crm_debug("All %d active cluster nodes are fully joined", count);
 561         }
 562 
 563     } else if (count > npeers) {
 564         crm_err("New election needed because more nodes confirmed join "
 565                 "than are in membership (%d > %u)", count, npeers);
 566         register_fsa_input(C_FSA_INTERNAL, I_ELECTION, NULL);
 567 
 568     } else if (controld_globals.membership_id != crm_peer_seq) {
 569         crm_info("New join needed because membership changed (%llu -> %llu)",
 570                  controld_globals.membership_id, crm_peer_seq);
 571         register_fsa_input_before(C_FSA_INTERNAL, I_NODE_JOIN, NULL);
 572 
 573     } else {
 574         crm_warn("Only %d of %u active cluster nodes fully joined "
 575                  "(%d did not respond to offer)",
 576                  count, npeers, crmd_join_phase_count(crm_join_welcomed));
 577     }
 578 }
 579 
 580 static void
 581 do_state_transition(enum crmd_fsa_state cur_state,
     /*  */
 582                     enum crmd_fsa_state next_state, fsa_data_t *msg_data)
 583 {
 584     int level = LOG_INFO;
 585     int count = 0;
 586     gboolean clear_recovery_bit = TRUE;
 587 #if 0
 588     uint64_t original_fsa_actions = controld_globals.fsa_actions;
 589 #endif
 590 
 591     enum crmd_fsa_cause cause = msg_data->fsa_cause;
 592     enum crmd_fsa_input current_input = msg_data->fsa_input;
 593 
 594     const char *state_from = fsa_state2string(cur_state);
 595     const char *state_to = fsa_state2string(next_state);
 596     const char *input = fsa_input2string(current_input);
 597 
 598     CRM_LOG_ASSERT(cur_state != next_state);
 599 
 600     do_dot_log(DOT_PREFIX "\t%s -> %s [ label=%s cause=%s origin=%s ]",
 601                state_from, state_to, input, fsa_cause2string(cause), msg_data->origin);
 602 
 603     if (cur_state == S_IDLE || next_state == S_IDLE) {
 604         level = LOG_NOTICE;
 605     } else if (cur_state == S_NOT_DC || next_state == S_NOT_DC) {
 606         level = LOG_NOTICE;
 607     } else if (cur_state == S_ELECTION) {
 608         level = LOG_NOTICE;
 609     } else if (cur_state == S_STARTING) {
 610         level = LOG_NOTICE;
 611     } else if (next_state == S_RECOVERY) {
 612         level = LOG_WARNING;
 613     }
 614 
 615     do_crm_log(level, "State transition %s -> %s "
 616                CRM_XS " input=%s cause=%s origin=%s",
 617                state_from, state_to, input, fsa_cause2string(cause),
 618                msg_data->origin);
 619 
 620     if (next_state != S_ELECTION && cur_state != S_RELEASE_DC) {
 621         controld_stop_current_election_timeout();
 622     }
 623 #if 0
 624     if ((controld_globals.fsa_input_register & R_SHUTDOWN)) {
 625         controld_set_fsa_action_flags(A_DC_TIMER_STOP);
 626     }
 627 #endif
 628     if (next_state == S_INTEGRATION) {
 629         controld_set_fsa_action_flags(A_INTEGRATE_TIMER_START);
 630     } else {
 631         controld_set_fsa_action_flags(A_INTEGRATE_TIMER_STOP);
 632     }
 633 
 634     if (next_state == S_FINALIZE_JOIN) {
 635         controld_set_fsa_action_flags(A_FINALIZE_TIMER_START);
 636     } else {
 637         controld_set_fsa_action_flags(A_FINALIZE_TIMER_STOP);
 638     }
 639 
 640     if (next_state != S_PENDING) {
 641         controld_set_fsa_action_flags(A_DC_TIMER_STOP);
 642     }
 643     if (next_state != S_IDLE) {
 644         controld_stop_recheck_timer();
 645     }
 646 
 647     if (cur_state == S_FINALIZE_JOIN && next_state == S_POLICY_ENGINE) {
 648         populate_cib_nodes(node_update_quick|node_update_all, __func__);
 649     }
 650 
 651     switch (next_state) {
 652         case S_PENDING:
 653             {
 654                 cib_t *cib_conn = controld_globals.cib_conn;
 655                 cib_conn->cmds->set_secondary(cib_conn, cib_scope_local);
 656             }
 657             update_dc(NULL);
 658             break;
 659 
 660         case S_ELECTION:
 661             update_dc(NULL);
 662             break;
 663 
 664         case S_NOT_DC:
 665             controld_reset_counter_election_timer();
 666             purge_stonith_cleanup();
 667 
 668             if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) {
 669                 crm_info("(Re)Issuing shutdown request now" " that we have a new DC");
 670                 controld_set_fsa_action_flags(A_SHUTDOWN_REQ);
 671             }
 672             CRM_LOG_ASSERT(controld_globals.dc_name != NULL);
 673             if (controld_globals.dc_name == NULL) {
 674                 crm_err("Reached S_NOT_DC without a DC" " being recorded");
 675             }
 676             break;
 677 
 678         case S_RECOVERY:
 679             clear_recovery_bit = FALSE;
 680             break;
 681 
 682         case S_FINALIZE_JOIN:
 683             CRM_LOG_ASSERT(AM_I_DC);
 684             if (cause == C_TIMER_POPPED) {
 685                 crm_warn("Progressed to state %s after %s",
 686                          fsa_state2string(next_state), fsa_cause2string(cause));
 687             }
 688             count = crmd_join_phase_count(crm_join_welcomed);
 689             if (count > 0) {
 690                 crm_warn("%d cluster node%s failed to respond to join offer",
 691                          count, pcmk__plural_s(count));
 692                 crmd_join_phase_log(LOG_NOTICE);
 693 
 694             } else {
 695                 crm_debug("All cluster nodes (%d) responded to join offer",
 696                           crmd_join_phase_count(crm_join_integrated));
 697             }
 698             break;
 699 
 700         case S_POLICY_ENGINE:
 701             controld_reset_counter_election_timer();
 702             CRM_LOG_ASSERT(AM_I_DC);
 703             if (cause == C_TIMER_POPPED) {
 704                 crm_info("Progressed to state %s after %s",
 705                          fsa_state2string(next_state), fsa_cause2string(cause));
 706             }
 707             check_join_counts(msg_data);
 708             break;
 709 
 710         case S_STOPPING:
 711         case S_TERMINATE:
 712             /* possibly redundant */
 713             controld_set_fsa_input_flags(R_SHUTDOWN);
 714             break;
 715 
 716         case S_IDLE:
 717             CRM_LOG_ASSERT(AM_I_DC);
 718             if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) {
 719                 crm_info("(Re)Issuing shutdown request now" " that we are the DC");
 720                 controld_set_fsa_action_flags(A_SHUTDOWN_REQ);
 721             }
 722             controld_start_recheck_timer();
 723             break;
 724 
 725         default:
 726             break;
 727     }
 728 
 729     if (clear_recovery_bit && next_state != S_PENDING) {
 730         controld_clear_fsa_action_flags(A_RECOVER);
 731     } else if (clear_recovery_bit == FALSE) {
 732         controld_set_fsa_action_flags(A_RECOVER);
 733     }
 734 
 735 #if 0
 736     if (original_fsa_actions != controld_globals.fsa_actions) {
 737         fsa_dump_actions(original_fsa_actions ^ controld_globals.fsa_actions,
 738                          "New actions");
 739     }
 740 #endif
 741 }
/* */
root/daemons/controld/controld_fsa.c

DEFINITIONS
