This source file includes following definitions.
- is_action_required
- get_action_delay_max
- get_action_delay_base
- get_action_timeout
- cmd_device
- fenced_device_reboot_action
- fenced_device_supports_on
- free_async_command
- create_async_command
- get_action_limit
- get_active_cmds
- fork_cb
- get_agent_metadata_cb
- report_internal_result
- stonith_device_execute
- stonith_device_dispatch
- start_delay_helper
- schedule_stonith_command
- free_device
- free_device_list
- init_device_list
- build_port_aliases
- free_metadata_cache
- init_metadata_cache
- get_agent_metadata
- is_nodeid_required
- read_action_metadata
- map_action
- xml2device_params
- target_list_type
- build_device_from_xml
- schedule_internal_command
- status_search_cb
- dynamic_list_search_cb
- device_params_diff
- device_has_duplicate
- stonith_device_register
- stonith_device_remove
- count_active_levels
- free_topology_entry
- free_topology_list
- init_topology_list
- stonith_level_key
- unpack_level_kind
- parse_device_list
- unpack_level_request
- fenced_register_level
- fenced_unregister_level
- list_to_string
- execute_agent_action
- search_devices_record_result
- localhost_is_eligible
- localhost_is_eligible_with_remap
- can_fence_host_with_device
- search_devices
- get_capable_devices
- add_action_specific_attributes
- add_disallowed
- add_action_reply
- stonith_send_reply
- stonith_query_capable_device_cb
- log_async_result
- send_async_reply
- cancel_stonith_command
- reply_to_duplicates
- next_required_device
- st_child_done
- sort_device_priority
- stonith_fence_get_devices_cb
- fence_locally
- fenced_construct_reply
- construct_async_reply
- fencing_peer_active
- set_fencing_completed
- check_alternate_host
- remove_relay_op
- is_privileged
- handle_register_request
- handle_agent_request
- handle_update_timeout_request
- handle_query_request
- handle_notify_request
- handle_relay_request
- handle_fence_request
- handle_history_request
- handle_device_add_request
- handle_device_delete_request
- handle_level_add_request
- handle_level_delete_request
- handle_cache_request
- handle_unknown_request
- fenced_register_handlers
- fenced_unregister_handlers
- handle_request
- handle_reply
- stonith_command
1
2
3
4
5
6
7
8
9
10 #include <crm_internal.h>
11
12 #include <sys/param.h>
13 #include <stdio.h>
14 #include <sys/types.h>
15 #include <sys/wait.h>
16 #include <sys/stat.h>
17 #include <unistd.h>
18 #include <sys/utsname.h>
19
20 #include <stdlib.h>
21 #include <errno.h>
22 #include <fcntl.h>
23 #include <ctype.h>
24
25 #include <crm/crm.h>
26 #include <crm/msg_xml.h>
27 #include <crm/common/ipc.h>
28 #include <crm/common/ipc_internal.h>
29 #include <crm/cluster/internal.h>
30 #include <crm/common/mainloop.h>
31
32 #include <crm/stonith-ng.h>
33 #include <crm/fencing/internal.h>
34 #include <crm/common/xml.h>
35
36 #include <pacemaker-fenced.h>
37
38 GHashTable *device_list = NULL;
39 GHashTable *topology = NULL;
40 static GList *cmd_list = NULL;
41
42 static GHashTable *fenced_handlers = NULL;
43
44 struct device_search_s {
45
46 char *host;
47
48 char *action;
49
50 int per_device_timeout;
51
52 int replies_needed;
53
54 int replies_received;
55
56 bool allow_suicide;
57
58
59 void *user_data;
60
61 void (*callback) (GList * devices, void *user_data);
62
63 GList *capable;
64
65 uint32_t support_action_only;
66 };
67
68 static gboolean stonith_device_dispatch(gpointer user_data);
69 static void st_child_done(int pid, const pcmk__action_result_t *result,
70 void *user_data);
71
72 static void search_devices_record_result(struct device_search_s *search, const char *device,
73 gboolean can_fence);
74
75 static int get_agent_metadata(const char *agent, xmlNode **metadata);
76 static void read_action_metadata(stonith_device_t *device);
77 static enum fenced_target_by unpack_level_kind(const xmlNode *level);
78
79 typedef struct async_command_s {
80
81 int id;
82 int pid;
83 int fd_stdout;
84 int options;
85 int default_timeout;
86 int timeout;
87
88 int start_delay;
89 int delay_id;
90
91 char *op;
92 char *origin;
93 char *client;
94 char *client_name;
95 char *remote_op_id;
96
97 char *target;
98 uint32_t target_nodeid;
99 char *action;
100 char *device;
101
102 GList *device_list;
103 GList *next_device_iter;
104
105 void *internal_user_data;
106 void (*done_cb) (int pid, const pcmk__action_result_t *result,
107 void *user_data);
108 guint timer_sigterm;
109 guint timer_sigkill;
110
111
112 int last_timeout_signo;
113
114 stonith_device_t *active_on;
115 stonith_device_t *activating_on;
116 } async_command_t;
117
118 static xmlNode *construct_async_reply(const async_command_t *cmd,
119 const pcmk__action_result_t *result);
120
121 static gboolean
122 is_action_required(const char *action, const stonith_device_t *device)
123 {
124 return (device != NULL) && device->automatic_unfencing
125 && pcmk__str_eq(action, PCMK_ACTION_ON, pcmk__str_none);
126 }
127
128 static int
129 get_action_delay_max(const stonith_device_t *device, const char *action)
130 {
131 const char *value = NULL;
132 int delay_max = 0;
133
134 if (!pcmk__is_fencing_action(action)) {
135 return 0;
136 }
137
138 value = g_hash_table_lookup(device->params, PCMK_STONITH_DELAY_MAX);
139 if (value) {
140 delay_max = crm_parse_interval_spec(value) / 1000;
141 }
142
143 return delay_max;
144 }
145
146 static int
147 get_action_delay_base(const stonith_device_t *device, const char *action,
148 const char *target)
149 {
150 char *hash_value = NULL;
151 int delay_base = 0;
152
153 if (!pcmk__is_fencing_action(action)) {
154 return 0;
155 }
156
157 hash_value = g_hash_table_lookup(device->params, PCMK_STONITH_DELAY_BASE);
158
159 if (hash_value) {
160 char *value = strdup(hash_value);
161 char *valptr = value;
162
163 CRM_ASSERT(value != NULL);
164
165 if (target != NULL) {
166 for (char *val = strtok(value, "; \t"); val != NULL; val = strtok(NULL, "; \t")) {
167 char *mapval = strchr(val, ':');
168
169 if (mapval == NULL || mapval[1] == 0) {
170 crm_err("pcmk_delay_base: empty value in mapping", val);
171 continue;
172 }
173
174 if (mapval != val && strncasecmp(target, val, (size_t)(mapval - val)) == 0) {
175 value = mapval + 1;
176 crm_debug("pcmk_delay_base mapped to %s for %s",
177 value, target);
178 break;
179 }
180 }
181 }
182
183 if (strchr(value, ':') == 0) {
184 delay_base = crm_parse_interval_spec(value) / 1000;
185 }
186
187 free(valptr);
188 }
189
190 return delay_base;
191 }
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213 static int
214 get_action_timeout(const stonith_device_t *device, const char *action,
215 int default_timeout)
216 {
217 if (action && device && device->params) {
218 char buffer[64] = { 0, };
219 const char *value = NULL;
220
221
222
223
224 if (pcmk__str_eq(action, PCMK_ACTION_REBOOT, pcmk__str_none)
225 && !pcmk_is_set(device->flags, st_device_supports_reboot)) {
226 crm_trace("%s doesn't support reboot, using timeout for off instead",
227 device->id);
228 action = PCMK_ACTION_OFF;
229 }
230
231
232 snprintf(buffer, sizeof(buffer), "pcmk_%s_timeout", action);
233 value = g_hash_table_lookup(device->params, buffer);
234 if (value) {
235 return atoi(value);
236 }
237 }
238 return default_timeout;
239 }
240
241
242
243
244
245
246
247
248
249 static stonith_device_t *
250 cmd_device(const async_command_t *cmd)
251 {
252 if ((cmd == NULL) || (cmd->device == NULL) || (device_list == NULL)) {
253 return NULL;
254 }
255 return g_hash_table_lookup(device_list, cmd->device);
256 }
257
258
259
260
261
262
263
264
265
266 const char *
267 fenced_device_reboot_action(const char *device_id)
268 {
269 const char *action = NULL;
270
271 if ((device_list != NULL) && (device_id != NULL)) {
272 stonith_device_t *device = g_hash_table_lookup(device_list, device_id);
273
274 if ((device != NULL) && (device->params != NULL)) {
275 action = g_hash_table_lookup(device->params, "pcmk_reboot_action");
276 }
277 }
278 return pcmk__s(action, PCMK_ACTION_REBOOT);
279 }
280
281
282
283
284
285
286
287
288
289 bool
290 fenced_device_supports_on(const char *device_id)
291 {
292 if ((device_list != NULL) && (device_id != NULL)) {
293 stonith_device_t *device = g_hash_table_lookup(device_list, device_id);
294
295 if (device != NULL) {
296 return pcmk_is_set(device->flags, st_device_supports_on);
297 }
298 }
299 return false;
300 }
301
302 static void
303 free_async_command(async_command_t * cmd)
304 {
305 if (!cmd) {
306 return;
307 }
308
309 if (cmd->delay_id) {
310 g_source_remove(cmd->delay_id);
311 }
312
313 cmd_list = g_list_remove(cmd_list, cmd);
314
315 g_list_free_full(cmd->device_list, free);
316 free(cmd->device);
317 free(cmd->action);
318 free(cmd->target);
319 free(cmd->remote_op_id);
320 free(cmd->client);
321 free(cmd->client_name);
322 free(cmd->origin);
323 free(cmd->op);
324 free(cmd);
325 }
326
327
328
329
330
331
332
333
334
335
336
337
338 static async_command_t *
339 create_async_command(xmlNode *msg)
340 {
341 xmlNode *op = NULL;
342 async_command_t *cmd = NULL;
343
344 if (msg == NULL) {
345 return NULL;
346 }
347
348 op = get_xpath_object("//@" F_STONITH_ACTION, msg, LOG_ERR);
349 if (op == NULL) {
350 return NULL;
351 }
352
353 cmd = calloc(1, sizeof(async_command_t));
354 CRM_ASSERT(cmd != NULL);
355
356
357 cmd->action = crm_element_value_copy(op, F_STONITH_ACTION);
358 cmd->op = crm_element_value_copy(msg, F_STONITH_OPERATION);
359 cmd->client = crm_element_value_copy(msg, F_STONITH_CLIENTID);
360 if ((cmd->action == NULL) || (cmd->op == NULL) || (cmd->client == NULL)) {
361 free_async_command(cmd);
362 return NULL;
363 }
364
365 crm_element_value_int(msg, F_STONITH_CALLID, &(cmd->id));
366 crm_element_value_int(msg, F_STONITH_CALLOPTS, &(cmd->options));
367 crm_element_value_int(msg, F_STONITH_DELAY, &(cmd->start_delay));
368 crm_element_value_int(msg, F_STONITH_TIMEOUT, &(cmd->default_timeout));
369 cmd->timeout = cmd->default_timeout;
370
371 cmd->origin = crm_element_value_copy(msg, F_ORIG);
372 cmd->remote_op_id = crm_element_value_copy(msg, F_STONITH_REMOTE_OP_ID);
373 cmd->client_name = crm_element_value_copy(msg, F_STONITH_CLIENTNAME);
374 cmd->target = crm_element_value_copy(op, F_STONITH_TARGET);
375 cmd->device = crm_element_value_copy(op, F_STONITH_DEVICE);
376
377 cmd->done_cb = st_child_done;
378
379
380 cmd_list = g_list_append(cmd_list, cmd);
381
382 return cmd;
383 }
384
385 static int
386 get_action_limit(stonith_device_t * device)
387 {
388 const char *value = NULL;
389 int action_limit = 1;
390
391 value = g_hash_table_lookup(device->params, PCMK_STONITH_ACTION_LIMIT);
392 if ((value == NULL)
393 || (pcmk__scan_min_int(value, &action_limit, INT_MIN) != pcmk_rc_ok)
394 || (action_limit == 0)) {
395 action_limit = 1;
396 }
397 return action_limit;
398 }
399
400 static int
401 get_active_cmds(stonith_device_t * device)
402 {
403 int counter = 0;
404 GList *gIter = NULL;
405 GList *gIterNext = NULL;
406
407 CRM_CHECK(device != NULL, return 0);
408
409 for (gIter = cmd_list; gIter != NULL; gIter = gIterNext) {
410 async_command_t *cmd = gIter->data;
411
412 gIterNext = gIter->next;
413
414 if (cmd->active_on == device) {
415 counter++;
416 }
417 }
418
419 return counter;
420 }
421
422 static void
423 fork_cb(int pid, void *user_data)
424 {
425 async_command_t *cmd = (async_command_t *) user_data;
426 stonith_device_t * device =
427
428
429
430 cmd->activating_on?cmd->activating_on:cmd->active_on;
431
432 CRM_ASSERT(device);
433 crm_debug("Operation '%s' [%d]%s%s using %s now running with %ds timeout",
434 cmd->action, pid,
435 ((cmd->target == NULL)? "" : " targeting "),
436 pcmk__s(cmd->target, ""), device->id, cmd->timeout);
437 cmd->active_on = device;
438 cmd->activating_on = NULL;
439 }
440
441 static int
442 get_agent_metadata_cb(gpointer data) {
443 stonith_device_t *device = data;
444 guint period_ms;
445
446 switch (get_agent_metadata(device->agent, &device->agent_metadata)) {
447 case pcmk_rc_ok:
448 if (device->agent_metadata) {
449 read_action_metadata(device);
450 stonith__device_parameter_flags(&(device->flags), device->id,
451 device->agent_metadata);
452 }
453 return G_SOURCE_REMOVE;
454
455 case EAGAIN:
456 period_ms = pcmk__mainloop_timer_get_period(device->timer);
457 if (period_ms < 160 * 1000) {
458 mainloop_timer_set_period(device->timer, 2 * period_ms);
459 }
460 return G_SOURCE_CONTINUE;
461
462 default:
463 return G_SOURCE_REMOVE;
464 }
465 }
466
467
468
469
470
471
472
473
474
475
476 static void
477 report_internal_result(async_command_t *cmd, int exit_status,
478 int execution_status, const char *exit_reason)
479 {
480 pcmk__action_result_t result = PCMK__UNKNOWN_RESULT;
481
482 pcmk__set_result(&result, exit_status, execution_status, exit_reason);
483 cmd->done_cb(0, &result, cmd);
484 pcmk__reset_result(&result);
485 }
486
487 static gboolean
488 stonith_device_execute(stonith_device_t * device)
489 {
490 int exec_rc = 0;
491 const char *action_str = NULL;
492 const char *host_arg = NULL;
493 async_command_t *cmd = NULL;
494 stonith_action_t *action = NULL;
495 int active_cmds = 0;
496 int action_limit = 0;
497 GList *gIter = NULL;
498 GList *gIterNext = NULL;
499
500 CRM_CHECK(device != NULL, return FALSE);
501
502 active_cmds = get_active_cmds(device);
503 action_limit = get_action_limit(device);
504 if (action_limit > -1 && active_cmds >= action_limit) {
505 crm_trace("%s is over its action limit of %d (%u active action%s)",
506 device->id, action_limit, active_cmds,
507 pcmk__plural_s(active_cmds));
508 return TRUE;
509 }
510
511 for (gIter = device->pending_ops; gIter != NULL; gIter = gIterNext) {
512 async_command_t *pending_op = gIter->data;
513
514 gIterNext = gIter->next;
515
516 if (pending_op && pending_op->delay_id) {
517 crm_trace("Operation '%s'%s%s using %s was asked to run too early, "
518 "waiting for start delay of %ds",
519 pending_op->action,
520 ((pending_op->target == NULL)? "" : " targeting "),
521 pcmk__s(pending_op->target, ""),
522 device->id, pending_op->start_delay);
523 continue;
524 }
525
526 device->pending_ops = g_list_remove_link(device->pending_ops, gIter);
527 g_list_free_1(gIter);
528
529 cmd = pending_op;
530 break;
531 }
532
533 if (cmd == NULL) {
534 crm_trace("No actions using %s are needed", device->id);
535 return TRUE;
536 }
537
538 if (pcmk__str_any_of(device->agent, STONITH_WATCHDOG_AGENT,
539 STONITH_WATCHDOG_AGENT_INTERNAL, NULL)) {
540 if (pcmk__is_fencing_action(cmd->action)) {
541 if (node_does_watchdog_fencing(stonith_our_uname)) {
542 pcmk__panic(__func__);
543 goto done;
544 }
545 } else {
546 crm_info("Faking success for %s watchdog operation", cmd->action);
547 report_internal_result(cmd, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
548 goto done;
549 }
550 }
551
552 #if SUPPORT_CIBSECRETS
553 exec_rc = pcmk__substitute_secrets(device->id, device->params);
554 if (exec_rc != pcmk_rc_ok) {
555 if (pcmk__str_eq(cmd->action, PCMK_ACTION_STOP, pcmk__str_none)) {
556 crm_info("Proceeding with stop operation for %s "
557 "despite being unable to load CIB secrets (%s)",
558 device->id, pcmk_rc_str(exec_rc));
559 } else {
560 crm_err("Considering %s unconfigured "
561 "because unable to load CIB secrets: %s",
562 device->id, pcmk_rc_str(exec_rc));
563 report_internal_result(cmd, CRM_EX_ERROR, PCMK_EXEC_NO_SECRETS,
564 "Failed to get CIB secrets");
565 goto done;
566 }
567 }
568 #endif
569
570 action_str = cmd->action;
571 if (pcmk__str_eq(cmd->action, PCMK_ACTION_REBOOT, pcmk__str_none)
572 && !pcmk_is_set(device->flags, st_device_supports_reboot)) {
573
574 crm_notice("Remapping 'reboot' action%s%s using %s to 'off' "
575 "because agent '%s' does not support reboot",
576 ((cmd->target == NULL)? "" : " targeting "),
577 pcmk__s(cmd->target, ""), device->id, device->agent);
578 action_str = PCMK_ACTION_OFF;
579 }
580
581 if (pcmk_is_set(device->flags, st_device_supports_parameter_port)) {
582 host_arg = "port";
583
584 } else if (pcmk_is_set(device->flags, st_device_supports_parameter_plug)) {
585 host_arg = "plug";
586 }
587
588 action = stonith__action_create(device->agent, action_str, cmd->target,
589 cmd->target_nodeid, cmd->timeout,
590 device->params, device->aliases, host_arg);
591
592
593
594 cmd->activating_on = device;
595 exec_rc = stonith__execute_async(action, (void *)cmd, cmd->done_cb,
596 fork_cb);
597 if (exec_rc < 0) {
598 cmd->activating_on = NULL;
599 cmd->done_cb(0, stonith__action_result(action), cmd);
600 stonith__destroy_action(action);
601 }
602
603 done:
604
605
606
607 if (device->pending_ops) {
608 mainloop_set_trigger(device->work);
609 }
610 return TRUE;
611 }
612
613 static gboolean
614 stonith_device_dispatch(gpointer user_data)
615 {
616 return stonith_device_execute(user_data);
617 }
618
619 static gboolean
620 start_delay_helper(gpointer data)
621 {
622 async_command_t *cmd = data;
623 stonith_device_t *device = cmd_device(cmd);
624
625 cmd->delay_id = 0;
626 if (device) {
627 mainloop_set_trigger(device->work);
628 }
629
630 return FALSE;
631 }
632
633 static void
634 schedule_stonith_command(async_command_t * cmd, stonith_device_t * device)
635 {
636 int delay_max = 0;
637 int delay_base = 0;
638 int requested_delay = cmd->start_delay;
639
640 CRM_CHECK(cmd != NULL, return);
641 CRM_CHECK(device != NULL, return);
642
643 if (cmd->device) {
644 free(cmd->device);
645 }
646
647 if (device->include_nodeid && (cmd->target != NULL)) {
648 crm_node_t *node = crm_get_peer(0, cmd->target);
649
650 cmd->target_nodeid = node->id;
651 }
652
653 cmd->device = strdup(device->id);
654 cmd->timeout = get_action_timeout(device, cmd->action, cmd->default_timeout);
655
656 if (cmd->remote_op_id) {
657 crm_debug("Scheduling '%s' action%s%s using %s for remote peer %s "
658 "with op id %.8s and timeout %ds",
659 cmd->action,
660 (cmd->target == NULL)? "" : " targeting ",
661 pcmk__s(cmd->target, ""),
662 device->id, cmd->origin, cmd->remote_op_id, cmd->timeout);
663 } else {
664 crm_debug("Scheduling '%s' action%s%s using %s for %s with timeout %ds",
665 cmd->action,
666 (cmd->target == NULL)? "" : " targeting ",
667 pcmk__s(cmd->target, ""),
668 device->id, cmd->client, cmd->timeout);
669 }
670
671 device->pending_ops = g_list_append(device->pending_ops, cmd);
672 mainloop_set_trigger(device->work);
673
674
675 if (requested_delay < 0) {
676 return;
677 }
678
679 delay_max = get_action_delay_max(device, cmd->action);
680 delay_base = get_action_delay_base(device, cmd->action, cmd->target);
681 if (delay_max == 0) {
682 delay_max = delay_base;
683 }
684 if (delay_max < delay_base) {
685 crm_warn(PCMK_STONITH_DELAY_BASE " (%ds) is larger than "
686 PCMK_STONITH_DELAY_MAX " (%ds) for %s using %s "
687 "(limiting to maximum delay)",
688 delay_base, delay_max, cmd->action, device->id);
689 delay_base = delay_max;
690 }
691 if (delay_max > 0) {
692
693 cmd->start_delay +=
694 ((delay_max != delay_base)?(rand() % (delay_max - delay_base)):0)
695 + delay_base;
696 }
697
698 if (cmd->start_delay > 0) {
699 crm_notice("Delaying '%s' action%s%s using %s for %ds " CRM_XS
700 " timeout=%ds requested_delay=%ds base=%ds max=%ds",
701 cmd->action,
702 (cmd->target == NULL)? "" : " targeting ",
703 pcmk__s(cmd->target, ""),
704 device->id, cmd->start_delay, cmd->timeout,
705 requested_delay, delay_base, delay_max);
706 cmd->delay_id =
707 g_timeout_add_seconds(cmd->start_delay, start_delay_helper, cmd);
708 }
709 }
710
711 static void
712 free_device(gpointer data)
713 {
714 GList *gIter = NULL;
715 stonith_device_t *device = data;
716
717 g_hash_table_destroy(device->params);
718 g_hash_table_destroy(device->aliases);
719
720 for (gIter = device->pending_ops; gIter != NULL; gIter = gIter->next) {
721 async_command_t *cmd = gIter->data;
722
723 crm_warn("Removal of device '%s' purged operation '%s'", device->id, cmd->action);
724 report_internal_result(cmd, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE,
725 "Device was removed before action could be executed");
726 }
727 g_list_free(device->pending_ops);
728
729 g_list_free_full(device->targets, free);
730
731 if (device->timer) {
732 mainloop_timer_stop(device->timer);
733 mainloop_timer_del(device->timer);
734 }
735
736 mainloop_destroy_trigger(device->work);
737
738 free_xml(device->agent_metadata);
739 free(device->namespace);
740 if (device->on_target_actions != NULL) {
741 g_string_free(device->on_target_actions, TRUE);
742 }
743 free(device->agent);
744 free(device->id);
745 free(device);
746 }
747
748 void free_device_list(void)
749 {
750 if (device_list != NULL) {
751 g_hash_table_destroy(device_list);
752 device_list = NULL;
753 }
754 }
755
756 void
757 init_device_list(void)
758 {
759 if (device_list == NULL) {
760 device_list = pcmk__strkey_table(NULL, free_device);
761 }
762 }
763
764 static GHashTable *
765 build_port_aliases(const char *hostmap, GList ** targets)
766 {
767 char *name = NULL;
768 int last = 0, lpc = 0, max = 0, added = 0;
769 GHashTable *aliases = pcmk__strikey_table(free, free);
770
771 if (hostmap == NULL) {
772 return aliases;
773 }
774
775 max = strlen(hostmap);
776 for (; lpc <= max; lpc++) {
777 switch (hostmap[lpc]) {
778
779 case '\\':
780 lpc++;
781 break;
782
783
784 case '=':
785 case ':':
786 if (lpc > last) {
787 free(name);
788 name = calloc(1, 1 + lpc - last);
789 memcpy(name, hostmap + last, lpc - last);
790 }
791 last = lpc + 1;
792 break;
793
794
795
796 case 0:
797 case ';':
798 case ' ':
799 case '\t':
800 if (name) {
801 char *value = NULL;
802 int k = 0;
803
804 value = calloc(1, 1 + lpc - last);
805 memcpy(value, hostmap + last, lpc - last);
806
807 for (int i = 0; value[i] != '\0'; i++) {
808 if (value[i] != '\\') {
809 value[k++] = value[i];
810 }
811 }
812 value[k] = '\0';
813
814 crm_debug("Adding alias '%s'='%s'", name, value);
815 g_hash_table_replace(aliases, name, value);
816 if (targets) {
817 *targets = g_list_append(*targets, strdup(value));
818 }
819 value = NULL;
820 name = NULL;
821 added++;
822
823 } else if (lpc > last) {
824 crm_debug("Parse error at offset %d near '%s'", lpc - last, hostmap + last);
825 }
826
827 last = lpc + 1;
828 break;
829 }
830
831 if (hostmap[lpc] == 0) {
832 break;
833 }
834 }
835
836 if (added == 0) {
837 crm_info("No host mappings detected in '%s'", hostmap);
838 }
839
840 free(name);
841 return aliases;
842 }
843
844 GHashTable *metadata_cache = NULL;
845
846 void
847 free_metadata_cache(void) {
848 if (metadata_cache != NULL) {
849 g_hash_table_destroy(metadata_cache);
850 metadata_cache = NULL;
851 }
852 }
853
854 static void
855 init_metadata_cache(void) {
856 if (metadata_cache == NULL) {
857 metadata_cache = pcmk__strkey_table(free, free);
858 }
859 }
860
861 int
862 get_agent_metadata(const char *agent, xmlNode ** metadata)
863 {
864 char *buffer = NULL;
865
866 if (metadata == NULL) {
867 return EINVAL;
868 }
869 *metadata = NULL;
870 if (pcmk__str_eq(agent, STONITH_WATCHDOG_AGENT_INTERNAL, pcmk__str_none)) {
871 return pcmk_rc_ok;
872 }
873 init_metadata_cache();
874 buffer = g_hash_table_lookup(metadata_cache, agent);
875 if (buffer == NULL) {
876 stonith_t *st = stonith_api_new();
877 int rc;
878
879 if (st == NULL) {
880 crm_warn("Could not get agent meta-data: "
881 "API memory allocation failed");
882 return EAGAIN;
883 }
884 rc = st->cmds->metadata(st, st_opt_sync_call, agent,
885 NULL, &buffer, 10);
886 stonith_api_delete(st);
887 if (rc || !buffer) {
888 crm_err("Could not retrieve metadata for fencing agent %s", agent);
889 return EAGAIN;
890 }
891 g_hash_table_replace(metadata_cache, strdup(agent), buffer);
892 }
893
894 *metadata = string2xml(buffer);
895 return pcmk_rc_ok;
896 }
897
898 static gboolean
899 is_nodeid_required(xmlNode * xml)
900 {
901 xmlXPathObjectPtr xpath = NULL;
902
903 if (stand_alone) {
904 return FALSE;
905 }
906
907 if (!xml) {
908 return FALSE;
909 }
910
911 xpath = xpath_search(xml, "//parameter[@name='nodeid']");
912 if (numXpathResults(xpath) <= 0) {
913 freeXpathObject(xpath);
914 return FALSE;
915 }
916
917 freeXpathObject(xpath);
918 return TRUE;
919 }
920
921 static void
922 read_action_metadata(stonith_device_t *device)
923 {
924 xmlXPathObjectPtr xpath = NULL;
925 int max = 0;
926 int lpc = 0;
927
928 if (device->agent_metadata == NULL) {
929 return;
930 }
931
932 xpath = xpath_search(device->agent_metadata, "//action");
933 max = numXpathResults(xpath);
934
935 if (max <= 0) {
936 freeXpathObject(xpath);
937 return;
938 }
939
940 for (lpc = 0; lpc < max; lpc++) {
941 const char *action = NULL;
942 xmlNode *match = getXpathResult(xpath, lpc);
943
944 CRM_LOG_ASSERT(match != NULL);
945 if(match == NULL) { continue; };
946
947 action = crm_element_value(match, "name");
948
949 if (pcmk__str_eq(action, PCMK_ACTION_LIST, pcmk__str_none)) {
950 stonith__set_device_flags(device->flags, device->id,
951 st_device_supports_list);
952 } else if (pcmk__str_eq(action, PCMK_ACTION_STATUS, pcmk__str_none)) {
953 stonith__set_device_flags(device->flags, device->id,
954 st_device_supports_status);
955 } else if (pcmk__str_eq(action, PCMK_ACTION_REBOOT, pcmk__str_none)) {
956 stonith__set_device_flags(device->flags, device->id,
957 st_device_supports_reboot);
958 } else if (pcmk__str_eq(action, PCMK_ACTION_ON, pcmk__str_none)) {
959
960
961 if (pcmk__xe_attr_is_true(match, "automatic") || pcmk__xe_attr_is_true(match, "required")) {
962 device->automatic_unfencing = TRUE;
963 }
964 stonith__set_device_flags(device->flags, device->id,
965 st_device_supports_on);
966 }
967
968 if ((action != NULL) && pcmk__xe_attr_is_true(match, "on_target")) {
969 pcmk__add_word(&(device->on_target_actions), 64, action);
970 }
971 }
972
973 freeXpathObject(xpath);
974 }
975
976
977
978
979
980
981
982
983
984 static void
985 map_action(GHashTable *params, const char *action, const char *value)
986 {
987 char *key = crm_strdup_printf("pcmk_%s_action", action);
988
989 if (g_hash_table_lookup(params, key)) {
990 crm_warn("Ignoring %s='%s', see %s instead",
991 STONITH_ATTR_ACTION_OP, value, key);
992 free(key);
993 } else {
994 crm_warn("Mapping %s='%s' to %s='%s'",
995 STONITH_ATTR_ACTION_OP, value, key, value);
996 g_hash_table_insert(params, key, strdup(value));
997 }
998 }
999
1000
1001
1002
1003
1004
1005
1006
1007 static GHashTable *
1008 xml2device_params(const char *name, const xmlNode *dev)
1009 {
1010 GHashTable *params = xml2list(dev);
1011 const char *value;
1012
1013
1014
1015
1016
1017 value = g_hash_table_lookup(params, STONITH_ATTR_ACTION_OP);
1018 if (value != NULL) {
1019 crm_warn("%s has '%s' parameter, which should never be specified in configuration",
1020 name, STONITH_ATTR_ACTION_OP);
1021
1022 if (*value == '\0') {
1023 crm_warn("Ignoring empty '%s' parameter", STONITH_ATTR_ACTION_OP);
1024
1025 } else if (strcmp(value, PCMK_ACTION_REBOOT) == 0) {
1026 crm_warn("Ignoring %s='reboot' (see stonith-action cluster property instead)",
1027 STONITH_ATTR_ACTION_OP);
1028
1029 } else if (strcmp(value, PCMK_ACTION_OFF) == 0) {
1030 map_action(params, PCMK_ACTION_REBOOT, value);
1031
1032 } else {
1033 map_action(params, PCMK_ACTION_OFF, value);
1034 map_action(params, PCMK_ACTION_REBOOT, value);
1035 }
1036
1037 g_hash_table_remove(params, STONITH_ATTR_ACTION_OP);
1038 }
1039
1040 return params;
1041 }
1042
1043 static const char *
1044 target_list_type(stonith_device_t * dev)
1045 {
1046 const char *check_type = NULL;
1047
1048 check_type = g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_CHECK);
1049
1050 if (check_type == NULL) {
1051
1052 if (g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_LIST)) {
1053 check_type = "static-list";
1054 } else if (g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_MAP)) {
1055 check_type = "static-list";
1056 } else if (pcmk_is_set(dev->flags, st_device_supports_list)) {
1057 check_type = "dynamic-list";
1058 } else if (pcmk_is_set(dev->flags, st_device_supports_status)) {
1059 check_type = "status";
1060 } else {
1061 check_type = PCMK__VALUE_NONE;
1062 }
1063 }
1064
1065 return check_type;
1066 }
1067
1068 static stonith_device_t *
1069 build_device_from_xml(xmlNode *dev)
1070 {
1071 const char *value;
1072 stonith_device_t *device = NULL;
1073 char *agent = crm_element_value_copy(dev, "agent");
1074
1075 CRM_CHECK(agent != NULL, return device);
1076
1077 device = calloc(1, sizeof(stonith_device_t));
1078
1079 CRM_CHECK(device != NULL, {free(agent); return device;});
1080
1081 device->id = crm_element_value_copy(dev, XML_ATTR_ID);
1082 device->agent = agent;
1083 device->namespace = crm_element_value_copy(dev, "namespace");
1084 device->params = xml2device_params(device->id, dev);
1085
1086 value = g_hash_table_lookup(device->params, PCMK_STONITH_HOST_LIST);
1087 if (value) {
1088 device->targets = stonith__parse_targets(value);
1089 }
1090
1091 value = g_hash_table_lookup(device->params, PCMK_STONITH_HOST_MAP);
1092 device->aliases = build_port_aliases(value, &(device->targets));
1093
1094 value = target_list_type(device);
1095 if (!pcmk__str_eq(value, "static-list", pcmk__str_casei) && device->targets) {
1096
1097 g_list_free_full(device->targets, free);
1098 device->targets = NULL;
1099 }
1100 switch (get_agent_metadata(device->agent, &device->agent_metadata)) {
1101 case pcmk_rc_ok:
1102 if (device->agent_metadata) {
1103 read_action_metadata(device);
1104 stonith__device_parameter_flags(&(device->flags), device->id,
1105 device->agent_metadata);
1106 }
1107 break;
1108
1109 case EAGAIN:
1110 if (device->timer == NULL) {
1111 device->timer = mainloop_timer_add("get_agent_metadata", 10 * 1000,
1112 TRUE, get_agent_metadata_cb, device);
1113 }
1114 if (!mainloop_timer_running(device->timer)) {
1115 mainloop_timer_start(device->timer);
1116 }
1117 break;
1118
1119 default:
1120 break;
1121 }
1122
1123 value = g_hash_table_lookup(device->params, "nodeid");
1124 if (!value) {
1125 device->include_nodeid = is_nodeid_required(device->agent_metadata);
1126 }
1127
1128 value = crm_element_value(dev, "rsc_provides");
1129 if (pcmk__str_eq(value, PCMK__VALUE_UNFENCING, pcmk__str_casei)) {
1130 device->automatic_unfencing = TRUE;
1131 }
1132
1133 if (is_action_required(PCMK_ACTION_ON, device)) {
1134 crm_info("Fencing device '%s' requires unfencing", device->id);
1135 }
1136
1137 if (device->on_target_actions != NULL) {
1138 crm_info("Fencing device '%s' requires actions (%s) to be executed "
1139 "on target", device->id,
1140 (const char *) device->on_target_actions->str);
1141 }
1142
1143 device->work = mainloop_add_trigger(G_PRIORITY_HIGH, stonith_device_dispatch, device);
1144
1145
1146 return device;
1147 }
1148
1149 static void
1150 schedule_internal_command(const char *origin,
1151 stonith_device_t * device,
1152 const char *action,
1153 const char *target,
1154 int timeout,
1155 void *internal_user_data,
1156 void (*done_cb) (int pid,
1157 const pcmk__action_result_t *result,
1158 void *user_data))
1159 {
1160 async_command_t *cmd = NULL;
1161
1162 cmd = calloc(1, sizeof(async_command_t));
1163
1164 cmd->id = -1;
1165 cmd->default_timeout = timeout ? timeout : 60;
1166 cmd->timeout = cmd->default_timeout;
1167 cmd->action = strdup(action);
1168 pcmk__str_update(&cmd->target, target);
1169 cmd->device = strdup(device->id);
1170 cmd->origin = strdup(origin);
1171 cmd->client = strdup(crm_system_name);
1172 cmd->client_name = strdup(crm_system_name);
1173
1174 cmd->internal_user_data = internal_user_data;
1175 cmd->done_cb = done_cb;
1176
1177 schedule_stonith_command(cmd, device);
1178 }
1179
1180
1181 enum fence_status_code {
1182 fence_status_invalid = -1,
1183 fence_status_active = 0,
1184 fence_status_unknown = 1,
1185 fence_status_inactive = 2,
1186 };
1187
1188 static void
1189 status_search_cb(int pid, const pcmk__action_result_t *result, void *user_data)
1190 {
1191 async_command_t *cmd = user_data;
1192 struct device_search_s *search = cmd->internal_user_data;
1193 stonith_device_t *dev = cmd_device(cmd);
1194 gboolean can = FALSE;
1195
1196 free_async_command(cmd);
1197
1198 if (!dev) {
1199 search_devices_record_result(search, NULL, FALSE);
1200 return;
1201 }
1202
1203 mainloop_set_trigger(dev->work);
1204
1205 if (result->execution_status != PCMK_EXEC_DONE) {
1206 crm_warn("Assuming %s cannot fence %s "
1207 "because status could not be executed: %s%s%s%s",
1208 dev->id, search->host,
1209 pcmk_exec_status_str(result->execution_status),
1210 ((result->exit_reason == NULL)? "" : " ("),
1211 ((result->exit_reason == NULL)? "" : result->exit_reason),
1212 ((result->exit_reason == NULL)? "" : ")"));
1213 search_devices_record_result(search, dev->id, FALSE);
1214 return;
1215 }
1216
1217 switch (result->exit_status) {
1218 case fence_status_unknown:
1219 crm_trace("%s reported it cannot fence %s", dev->id, search->host);
1220 break;
1221
1222 case fence_status_active:
1223 case fence_status_inactive:
1224 crm_trace("%s reported it can fence %s", dev->id, search->host);
1225 can = TRUE;
1226 break;
1227
1228 default:
1229 crm_warn("Assuming %s cannot fence %s "
1230 "(status returned unknown code %d)",
1231 dev->id, search->host, result->exit_status);
1232 break;
1233 }
1234 search_devices_record_result(search, dev->id, can);
1235 }
1236
1237 static void
1238 dynamic_list_search_cb(int pid, const pcmk__action_result_t *result,
1239 void *user_data)
1240 {
1241 async_command_t *cmd = user_data;
1242 struct device_search_s *search = cmd->internal_user_data;
1243 stonith_device_t *dev = cmd_device(cmd);
1244 gboolean can_fence = FALSE;
1245
1246 free_async_command(cmd);
1247
1248
1249
1250
1251
1252
1253 if (!dev) {
1254 search_devices_record_result(search, NULL, FALSE);
1255 return;
1256 }
1257
1258 mainloop_set_trigger(dev->work);
1259
1260 if (pcmk__result_ok(result)) {
1261 crm_info("Refreshing target list for %s", dev->id);
1262 g_list_free_full(dev->targets, free);
1263 dev->targets = stonith__parse_targets(result->action_stdout);
1264 dev->targets_age = time(NULL);
1265
1266 } else if (dev->targets != NULL) {
1267 if (result->execution_status == PCMK_EXEC_DONE) {
1268 crm_info("Reusing most recent target list for %s "
1269 "because list returned error code %d",
1270 dev->id, result->exit_status);
1271 } else {
1272 crm_info("Reusing most recent target list for %s "
1273 "because list could not be executed: %s%s%s%s",
1274 dev->id, pcmk_exec_status_str(result->execution_status),
1275 ((result->exit_reason == NULL)? "" : " ("),
1276 ((result->exit_reason == NULL)? "" : result->exit_reason),
1277 ((result->exit_reason == NULL)? "" : ")"));
1278 }
1279
1280 } else {
1281 if (result->execution_status == PCMK_EXEC_DONE) {
1282 crm_warn("Assuming %s cannot fence %s "
1283 "because list returned error code %d",
1284 dev->id, search->host, result->exit_status);
1285 } else {
1286 crm_warn("Assuming %s cannot fence %s "
1287 "because list could not be executed: %s%s%s%s",
1288 dev->id, search->host,
1289 pcmk_exec_status_str(result->execution_status),
1290 ((result->exit_reason == NULL)? "" : " ("),
1291 ((result->exit_reason == NULL)? "" : result->exit_reason),
1292 ((result->exit_reason == NULL)? "" : ")"));
1293 }
1294
1295
1296
1297
1298 if (g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_CHECK) == NULL) {
1299 crm_notice("Switching to pcmk_host_check='status' for %s", dev->id);
1300 g_hash_table_replace(dev->params, strdup(PCMK_STONITH_HOST_CHECK),
1301 strdup("status"));
1302 }
1303 }
1304
1305 if (dev->targets) {
1306 const char *alias = g_hash_table_lookup(dev->aliases, search->host);
1307
1308 if (!alias) {
1309 alias = search->host;
1310 }
1311 if (pcmk__str_in_list(alias, dev->targets, pcmk__str_casei)) {
1312 can_fence = TRUE;
1313 }
1314 }
1315 search_devices_record_result(search, dev->id, can_fence);
1316 }
1317
1318
1319
1320
1321
1322 static int
1323 device_params_diff(GHashTable *first, GHashTable *second) {
1324 char *key = NULL;
1325 char *value = NULL;
1326 GHashTableIter gIter;
1327
1328 g_hash_table_iter_init(&gIter, first);
1329 while (g_hash_table_iter_next(&gIter, (void **)&key, (void **)&value)) {
1330
1331 if(strstr(key, "CRM_meta") == key) {
1332 continue;
1333 } else if(strcmp(key, "crm_feature_set") == 0) {
1334 continue;
1335 } else {
1336 char *other_value = g_hash_table_lookup(second, key);
1337
1338 if (!other_value || !pcmk__str_eq(other_value, value, pcmk__str_casei)) {
1339 crm_trace("Different value for %s: %s != %s", key, other_value, value);
1340 return 1;
1341 }
1342 }
1343 }
1344
1345 return 0;
1346 }
1347
1348
1349
1350
1351
1352 static stonith_device_t *
1353 device_has_duplicate(const stonith_device_t *device)
1354 {
1355 stonith_device_t *dup = g_hash_table_lookup(device_list, device->id);
1356
1357 if (!dup) {
1358 crm_trace("No match for %s", device->id);
1359 return NULL;
1360
1361 } else if (!pcmk__str_eq(dup->agent, device->agent, pcmk__str_casei)) {
1362 crm_trace("Different agent: %s != %s", dup->agent, device->agent);
1363 return NULL;
1364 }
1365
1366
1367 if (device_params_diff(device->params, dup->params) ||
1368 device_params_diff(dup->params, device->params)) {
1369 return NULL;
1370 }
1371
1372 crm_trace("Match");
1373 return dup;
1374 }
1375
1376 int
1377 stonith_device_register(xmlNode *dev, gboolean from_cib)
1378 {
1379 stonith_device_t *dup = NULL;
1380 stonith_device_t *device = build_device_from_xml(dev);
1381 guint ndevices = 0;
1382 int rv = pcmk_ok;
1383
1384 CRM_CHECK(device != NULL, return -ENOMEM);
1385
1386
1387 if (pcmk__str_eq(device->id, STONITH_WATCHDOG_ID, pcmk__str_none) ||
1388 pcmk__str_any_of(device->agent, STONITH_WATCHDOG_AGENT,
1389 STONITH_WATCHDOG_AGENT_INTERNAL, NULL)) do {
1390 if (stonith_watchdog_timeout_ms <= 0) {
1391 crm_err("Ignoring watchdog fence device without "
1392 "stonith-watchdog-timeout set.");
1393 rv = -ENODEV;
1394
1395 } else if (!pcmk__str_any_of(device->agent, STONITH_WATCHDOG_AGENT,
1396 STONITH_WATCHDOG_AGENT_INTERNAL, NULL)) {
1397 crm_err("Ignoring watchdog fence device with unknown "
1398 "agent '%s' unequal '" STONITH_WATCHDOG_AGENT "'.",
1399 device->agent?device->agent:"");
1400 rv = -ENODEV;
1401
1402 } else if (!pcmk__str_eq(device->id, STONITH_WATCHDOG_ID,
1403 pcmk__str_none)) {
1404 crm_err("Ignoring watchdog fence device "
1405 "named %s !='"STONITH_WATCHDOG_ID"'.",
1406 device->id?device->id:"");
1407 rv = -ENODEV;
1408
1409 } else {
1410 if (pcmk__str_eq(device->agent, STONITH_WATCHDOG_AGENT,
1411 pcmk__str_none)) {
1412
1413
1414
1415 g_list_free_full(stonith_watchdog_targets, free);
1416 stonith_watchdog_targets = device->targets;
1417 device->targets = NULL;
1418 }
1419 if (node_does_watchdog_fencing(stonith_our_uname)) {
1420 g_list_free_full(device->targets, free);
1421 device->targets = stonith__parse_targets(stonith_our_uname);
1422 g_hash_table_replace(device->params,
1423 strdup(PCMK_STONITH_HOST_LIST),
1424 strdup(stonith_our_uname));
1425
1426 break;
1427 }
1428
1429 crm_debug("Skip registration of watchdog fence device on node not in host-list.");
1430
1431 device->targets = NULL;
1432 stonith_device_remove(device->id, from_cib);
1433 }
1434 free_device(device);
1435 return rv;
1436 } while (0);
1437
1438 dup = device_has_duplicate(device);
1439 if (dup) {
1440 ndevices = g_hash_table_size(device_list);
1441 crm_debug("Device '%s' already in device list (%d active device%s)",
1442 device->id, ndevices, pcmk__plural_s(ndevices));
1443 free_device(device);
1444 device = dup;
1445 dup = g_hash_table_lookup(device_list, device->id);
1446 dup->dirty = FALSE;
1447
1448 } else {
1449 stonith_device_t *old = g_hash_table_lookup(device_list, device->id);
1450
1451 if (from_cib && old && old->api_registered) {
1452
1453
1454
1455
1456 crm_info("Overwriting existing entry for %s from CIB", device->id);
1457 device->pending_ops = old->pending_ops;
1458 device->api_registered = TRUE;
1459 old->pending_ops = NULL;
1460 if (device->pending_ops) {
1461 mainloop_set_trigger(device->work);
1462 }
1463 }
1464 g_hash_table_replace(device_list, device->id, device);
1465
1466 ndevices = g_hash_table_size(device_list);
1467 crm_notice("Added '%s' to device list (%d active device%s)",
1468 device->id, ndevices, pcmk__plural_s(ndevices));
1469 }
1470
1471 if (from_cib) {
1472 device->cib_registered = TRUE;
1473 } else {
1474 device->api_registered = TRUE;
1475 }
1476
1477 return pcmk_ok;
1478 }
1479
1480 void
1481 stonith_device_remove(const char *id, bool from_cib)
1482 {
1483 stonith_device_t *device = g_hash_table_lookup(device_list, id);
1484 guint ndevices = 0;
1485
1486 if (!device) {
1487 ndevices = g_hash_table_size(device_list);
1488 crm_info("Device '%s' not found (%d active device%s)",
1489 id, ndevices, pcmk__plural_s(ndevices));
1490 return;
1491 }
1492
1493 if (from_cib) {
1494 device->cib_registered = FALSE;
1495 } else {
1496 device->verified = FALSE;
1497 device->api_registered = FALSE;
1498 }
1499
1500 if (!device->cib_registered && !device->api_registered) {
1501 g_hash_table_remove(device_list, id);
1502 ndevices = g_hash_table_size(device_list);
1503 crm_info("Removed '%s' from device list (%d active device%s)",
1504 id, ndevices, pcmk__plural_s(ndevices));
1505 } else {
1506 crm_trace("Not removing '%s' from device list (%d active) because "
1507 "still registered via:%s%s",
1508 id, g_hash_table_size(device_list),
1509 (device->cib_registered? " cib" : ""),
1510 (device->api_registered? " api" : ""));
1511 }
1512 }
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523 static int
1524 count_active_levels(const stonith_topology_t *tp)
1525 {
1526 int lpc = 0;
1527 int count = 0;
1528
1529 for (lpc = 0; lpc < ST_LEVEL_MAX; lpc++) {
1530 if (tp->levels[lpc] != NULL) {
1531 count++;
1532 }
1533 }
1534 return count;
1535 }
1536
1537 static void
1538 free_topology_entry(gpointer data)
1539 {
1540 stonith_topology_t *tp = data;
1541
1542 int lpc = 0;
1543
1544 for (lpc = 0; lpc < ST_LEVEL_MAX; lpc++) {
1545 if (tp->levels[lpc] != NULL) {
1546 g_list_free_full(tp->levels[lpc], free);
1547 }
1548 }
1549 free(tp->target);
1550 free(tp->target_value);
1551 free(tp->target_pattern);
1552 free(tp->target_attribute);
1553 free(tp);
1554 }
1555
1556 void
1557 free_topology_list(void)
1558 {
1559 if (topology != NULL) {
1560 g_hash_table_destroy(topology);
1561 topology = NULL;
1562 }
1563 }
1564
1565 void
1566 init_topology_list(void)
1567 {
1568 if (topology == NULL) {
1569 topology = pcmk__strkey_table(NULL, free_topology_entry);
1570 }
1571 }
1572
1573 char *
1574 stonith_level_key(const xmlNode *level, enum fenced_target_by mode)
1575 {
1576 if (mode == fenced_target_by_unknown) {
1577 mode = unpack_level_kind(level);
1578 }
1579 switch (mode) {
1580 case fenced_target_by_name:
1581 return crm_element_value_copy(level, XML_ATTR_STONITH_TARGET);
1582
1583 case fenced_target_by_pattern:
1584 return crm_element_value_copy(level, XML_ATTR_STONITH_TARGET_PATTERN);
1585
1586 case fenced_target_by_attribute:
1587 return crm_strdup_printf("%s=%s",
1588 crm_element_value(level, XML_ATTR_STONITH_TARGET_ATTRIBUTE),
1589 crm_element_value(level, XML_ATTR_STONITH_TARGET_VALUE));
1590
1591 default:
1592 return crm_strdup_printf("unknown-%s", ID(level));
1593 }
1594 }
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604 static enum fenced_target_by
1605 unpack_level_kind(const xmlNode *level)
1606 {
1607 if (crm_element_value(level, XML_ATTR_STONITH_TARGET) != NULL) {
1608 return fenced_target_by_name;
1609 }
1610 if (crm_element_value(level, XML_ATTR_STONITH_TARGET_PATTERN) != NULL) {
1611 return fenced_target_by_pattern;
1612 }
1613 if (!stand_alone
1614 && (crm_element_value(level, XML_ATTR_STONITH_TARGET_ATTRIBUTE) != NULL)
1615 && (crm_element_value(level, XML_ATTR_STONITH_TARGET_VALUE) != NULL)) {
1616 return fenced_target_by_attribute;
1617 }
1618 return fenced_target_by_unknown;
1619 }
1620
1621 static stonith_key_value_t *
1622 parse_device_list(const char *devices)
1623 {
1624 int lpc = 0;
1625 int max = 0;
1626 int last = 0;
1627 stonith_key_value_t *output = NULL;
1628
1629 if (devices == NULL) {
1630 return output;
1631 }
1632
1633 max = strlen(devices);
1634 for (lpc = 0; lpc <= max; lpc++) {
1635 if (devices[lpc] == ',' || devices[lpc] == 0) {
1636 char *line = strndup(devices + last, lpc - last);
1637
1638 output = stonith_key_value_add(output, NULL, line);
1639 free(line);
1640
1641 last = lpc + 1;
1642 }
1643 }
1644
1645 return output;
1646 }
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661 static xmlNode *
1662 unpack_level_request(xmlNode *xml, enum fenced_target_by *mode, char **target,
1663 int *id, char **desc)
1664 {
1665 enum fenced_target_by local_mode = fenced_target_by_unknown;
1666 char *local_target = NULL;
1667 int local_id = 0;
1668
1669
1670
1671
1672
1673 if ((xml != NULL) && !pcmk__xe_is(xml, XML_TAG_FENCING_LEVEL)) {
1674 xml = get_xpath_object("//" XML_TAG_FENCING_LEVEL, xml, LOG_WARNING);
1675 }
1676
1677 if (xml == NULL) {
1678 if (desc != NULL) {
1679 *desc = crm_strdup_printf("missing");
1680 }
1681 } else {
1682 local_mode = unpack_level_kind(xml);
1683 local_target = stonith_level_key(xml, local_mode);
1684 crm_element_value_int(xml, XML_ATTR_STONITH_INDEX, &local_id);
1685 if (desc != NULL) {
1686 *desc = crm_strdup_printf("%s[%d]", local_target, local_id);
1687 }
1688 }
1689
1690 if (mode != NULL) {
1691 *mode = local_mode;
1692 }
1693 if (id != NULL) {
1694 *id = local_id;
1695 }
1696
1697 if (target != NULL) {
1698 *target = local_target;
1699 } else {
1700 free(local_target);
1701 }
1702
1703 return xml;
1704 }
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719 void
1720 fenced_register_level(xmlNode *msg, char **desc, pcmk__action_result_t *result)
1721 {
1722 int id = 0;
1723 xmlNode *level;
1724 enum fenced_target_by mode;
1725 char *target;
1726
1727 stonith_topology_t *tp;
1728 stonith_key_value_t *dIter = NULL;
1729 stonith_key_value_t *devices = NULL;
1730
1731 CRM_CHECK((msg != NULL) && (result != NULL), return);
1732
1733 level = unpack_level_request(msg, &mode, &target, &id, desc);
1734 if (level == NULL) {
1735 fenced_set_protocol_error(result);
1736 return;
1737 }
1738
1739
1740 if (pcmk__str_empty(ID(level))) {
1741 crm_warn("Ignoring registration for topology level without ID");
1742 free(target);
1743 crm_log_xml_trace(level, "Bad level");
1744 pcmk__format_result(result, CRM_EX_INVALID_PARAM, PCMK_EXEC_INVALID,
1745 "Topology level is invalid without ID");
1746 return;
1747 }
1748
1749
1750 if (mode == fenced_target_by_unknown) {
1751 crm_warn("Ignoring registration for topology level '%s' "
1752 "without valid target", ID(level));
1753 free(target);
1754 crm_log_xml_trace(level, "Bad level");
1755 pcmk__format_result(result, CRM_EX_INVALID_PARAM, PCMK_EXEC_INVALID,
1756 "Invalid target for topology level '%s'",
1757 ID(level));
1758 return;
1759 }
1760
1761
1762 if ((id <= 0) || (id >= ST_LEVEL_MAX)) {
1763 crm_warn("Ignoring topology registration for %s with invalid level %d",
1764 target, id);
1765 free(target);
1766 crm_log_xml_trace(level, "Bad level");
1767 pcmk__format_result(result, CRM_EX_INVALID_PARAM, PCMK_EXEC_INVALID,
1768 "Invalid level number '%s' for topology level '%s'",
1769 pcmk__s(crm_element_value(level,
1770 XML_ATTR_STONITH_INDEX),
1771 ""),
1772 ID(level));
1773 return;
1774 }
1775
1776
1777 tp = g_hash_table_lookup(topology, target);
1778 if (tp == NULL) {
1779 tp = calloc(1, sizeof(stonith_topology_t));
1780 if (tp == NULL) {
1781 pcmk__set_result(result, CRM_EX_ERROR, PCMK_EXEC_ERROR,
1782 strerror(ENOMEM));
1783 free(target);
1784 return;
1785 }
1786 tp->kind = mode;
1787 tp->target = target;
1788 tp->target_value = crm_element_value_copy(level, XML_ATTR_STONITH_TARGET_VALUE);
1789 tp->target_pattern = crm_element_value_copy(level, XML_ATTR_STONITH_TARGET_PATTERN);
1790 tp->target_attribute = crm_element_value_copy(level, XML_ATTR_STONITH_TARGET_ATTRIBUTE);
1791
1792 g_hash_table_replace(topology, tp->target, tp);
1793 crm_trace("Added %s (%d) to the topology (%d active entries)",
1794 target, (int) mode, g_hash_table_size(topology));
1795 } else {
1796 free(target);
1797 }
1798
1799 if (tp->levels[id] != NULL) {
1800 crm_info("Adding to the existing %s[%d] topology entry",
1801 tp->target, id);
1802 }
1803
1804 devices = parse_device_list(crm_element_value(level, XML_ATTR_STONITH_DEVICES));
1805 for (dIter = devices; dIter; dIter = dIter->next) {
1806 const char *device = dIter->value;
1807
1808 crm_trace("Adding device '%s' for %s[%d]", device, tp->target, id);
1809 tp->levels[id] = g_list_append(tp->levels[id], strdup(device));
1810 }
1811 stonith_key_value_freeall(devices, 1, 1);
1812
1813 {
1814 int nlevels = count_active_levels(tp);
1815
1816 crm_info("Target %s has %d active fencing level%s",
1817 tp->target, nlevels, pcmk__plural_s(nlevels));
1818 }
1819
1820 pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
1821 }
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835 void
1836 fenced_unregister_level(xmlNode *msg, char **desc,
1837 pcmk__action_result_t *result)
1838 {
1839 int id = -1;
1840 stonith_topology_t *tp;
1841 char *target;
1842 xmlNode *level = NULL;
1843
1844 CRM_CHECK(result != NULL, return);
1845
1846 level = unpack_level_request(msg, NULL, &target, &id, desc);
1847 if (level == NULL) {
1848 fenced_set_protocol_error(result);
1849 return;
1850 }
1851
1852
1853 if ((id < 0) || (id >= ST_LEVEL_MAX)) {
1854 crm_warn("Ignoring topology unregistration for %s with invalid level %d",
1855 target, id);
1856 free(target);
1857 crm_log_xml_trace(level, "Bad level");
1858 pcmk__format_result(result, CRM_EX_INVALID_PARAM, PCMK_EXEC_INVALID,
1859 "Invalid level number '%s' for topology level %s",
1860 pcmk__s(crm_element_value(level,
1861 XML_ATTR_STONITH_INDEX),
1862 "<null>"),
1863
1864
1865 pcmk__s(ID(level), ""));
1866 return;
1867 }
1868
1869 tp = g_hash_table_lookup(topology, target);
1870 if (tp == NULL) {
1871 guint nentries = g_hash_table_size(topology);
1872
1873 crm_info("No fencing topology found for %s (%d active %s)",
1874 target, nentries,
1875 pcmk__plural_alt(nentries, "entry", "entries"));
1876
1877 } else if (id == 0 && g_hash_table_remove(topology, target)) {
1878 guint nentries = g_hash_table_size(topology);
1879
1880 crm_info("Removed all fencing topology entries related to %s "
1881 "(%d active %s remaining)", target, nentries,
1882 pcmk__plural_alt(nentries, "entry", "entries"));
1883
1884 } else if (tp->levels[id] != NULL) {
1885 guint nlevels;
1886
1887 g_list_free_full(tp->levels[id], free);
1888 tp->levels[id] = NULL;
1889
1890 nlevels = count_active_levels(tp);
1891 crm_info("Removed level %d from fencing topology for %s "
1892 "(%d active level%s remaining)",
1893 id, target, nlevels, pcmk__plural_s(nlevels));
1894 }
1895
1896 free(target);
1897 pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
1898 }
1899
1900 static char *
1901 list_to_string(GList *list, const char *delim, gboolean terminate_with_delim)
1902 {
1903 int max = g_list_length(list);
1904 size_t delim_len = delim?strlen(delim):0;
1905 size_t alloc_size = 1 + (max?((max-1+(terminate_with_delim?1:0))*delim_len):0);
1906 char *rv;
1907 GList *gIter;
1908
1909 for (gIter = list; gIter != NULL; gIter = gIter->next) {
1910 const char *value = (const char *) gIter->data;
1911
1912 alloc_size += strlen(value);
1913 }
1914 rv = calloc(alloc_size, sizeof(char));
1915 if (rv) {
1916 char *pos = rv;
1917 const char *lead_delim = "";
1918
1919 for (gIter = list; gIter != NULL; gIter = gIter->next) {
1920 const char *value = (const char *) gIter->data;
1921
1922 pos = &pos[sprintf(pos, "%s%s", lead_delim, value)];
1923 lead_delim = delim;
1924 }
1925 if (max && terminate_with_delim) {
1926 sprintf(pos, "%s", delim);
1927 }
1928 }
1929 return rv;
1930 }
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947 static void
1948 execute_agent_action(xmlNode *msg, pcmk__action_result_t *result)
1949 {
1950 xmlNode *dev = get_xpath_object("//" F_STONITH_DEVICE, msg, LOG_ERR);
1951 xmlNode *op = get_xpath_object("//@" F_STONITH_ACTION, msg, LOG_ERR);
1952 const char *id = crm_element_value(dev, F_STONITH_DEVICE);
1953 const char *action = crm_element_value(op, F_STONITH_ACTION);
1954 async_command_t *cmd = NULL;
1955 stonith_device_t *device = NULL;
1956
1957 if ((id == NULL) || (action == NULL)) {
1958 crm_info("Malformed API action request: device %s, action %s",
1959 (id? id : "not specified"),
1960 (action? action : "not specified"));
1961 fenced_set_protocol_error(result);
1962 return;
1963 }
1964
1965 if (pcmk__str_eq(id, STONITH_WATCHDOG_ID, pcmk__str_none)) {
1966
1967 if (stonith_watchdog_timeout_ms <= 0) {
1968 pcmk__set_result(result, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE,
1969 "Watchdog fence device not configured");
1970 return;
1971
1972 } else if (pcmk__str_eq(action, PCMK_ACTION_LIST, pcmk__str_none)) {
1973 pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
1974 pcmk__set_result_output(result,
1975 list_to_string(stonith_watchdog_targets,
1976 "\n", TRUE),
1977 NULL);
1978 return;
1979
1980 } else if (pcmk__str_eq(action, PCMK_ACTION_MONITOR, pcmk__str_none)) {
1981 pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
1982 return;
1983 }
1984 }
1985
1986 device = g_hash_table_lookup(device_list, id);
1987 if (device == NULL) {
1988 crm_info("Ignoring API '%s' action request because device %s not found",
1989 action, id);
1990 pcmk__format_result(result, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE,
1991 "'%s' not found", id);
1992 return;
1993
1994 } else if (!device->api_registered
1995 && (strcmp(action, PCMK_ACTION_MONITOR) == 0)) {
1996
1997 crm_info("Ignoring API '%s' action request because device %s not active",
1998 action, id);
1999 pcmk__format_result(result, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE,
2000 "'%s' not active", id);
2001 return;
2002 }
2003
2004 cmd = create_async_command(msg);
2005 if (cmd == NULL) {
2006 crm_log_xml_warn(msg, "invalid");
2007 fenced_set_protocol_error(result);
2008 return;
2009 }
2010
2011 schedule_stonith_command(cmd, device);
2012 pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_PENDING, NULL);
2013 }
2014
2015 static void
2016 search_devices_record_result(struct device_search_s *search, const char *device, gboolean can_fence)
2017 {
2018 search->replies_received++;
2019 if (can_fence && device) {
2020 if (search->support_action_only != st_device_supports_none) {
2021 stonith_device_t *dev = g_hash_table_lookup(device_list, device);
2022 if (dev && !pcmk_is_set(dev->flags, search->support_action_only)) {
2023 return;
2024 }
2025 }
2026 search->capable = g_list_append(search->capable, strdup(device));
2027 }
2028
2029 if (search->replies_needed == search->replies_received) {
2030
2031 guint ndevices = g_list_length(search->capable);
2032
2033 crm_debug("Search found %d device%s that can perform '%s' targeting %s",
2034 ndevices, pcmk__plural_s(ndevices),
2035 (search->action? search->action : "unknown action"),
2036 (search->host? search->host : "any node"));
2037
2038 search->callback(search->capable, search->user_data);
2039 free(search->host);
2040 free(search->action);
2041 free(search);
2042 }
2043 }
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056 static gboolean
2057 localhost_is_eligible(const stonith_device_t *device, const char *action,
2058 const char *target, gboolean allow_suicide)
2059 {
2060 gboolean localhost_is_target = pcmk__str_eq(target, stonith_our_uname,
2061 pcmk__str_casei);
2062
2063 if ((device != NULL) && (action != NULL)
2064 && (device->on_target_actions != NULL)
2065 && (strstr((const char*) device->on_target_actions->str,
2066 action) != NULL)) {
2067
2068 if (!localhost_is_target) {
2069 crm_trace("Operation '%s' using %s can only be executed for local "
2070 "host, not %s", action, device->id, target);
2071 return FALSE;
2072 }
2073
2074 } else if (localhost_is_target && !allow_suicide) {
2075 crm_trace("'%s' operation does not support self-fencing", action);
2076 return FALSE;
2077 }
2078 return TRUE;
2079 }
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093 static bool
2094 localhost_is_eligible_with_remap(const stonith_device_t *device,
2095 const char *action, const char *target,
2096 gboolean allow_self)
2097 {
2098
2099 if (localhost_is_eligible(device, action, target, allow_self)) {
2100 return true;
2101 }
2102
2103
2104
2105 if (pcmk__str_eq(action, PCMK_ACTION_REBOOT, pcmk__str_none)) {
2106
2107
2108
2109
2110
2111 if (localhost_is_eligible(device, PCMK_ACTION_OFF, target, allow_self)
2112 || localhost_is_eligible(device, PCMK_ACTION_ON, target, FALSE)) {
2113 return true;
2114 }
2115 }
2116
2117 return false;
2118 }
2119
2120 static void
2121 can_fence_host_with_device(stonith_device_t *dev,
2122 struct device_search_s *search)
2123 {
2124 gboolean can = FALSE;
2125 const char *check_type = "Internal bug";
2126 const char *target = NULL;
2127 const char *alias = NULL;
2128 const char *dev_id = "Unspecified device";
2129 const char *action = (search == NULL)? NULL : search->action;
2130
2131 CRM_CHECK((dev != NULL) && (action != NULL), goto search_report_results);
2132
2133 if (dev->id != NULL) {
2134 dev_id = dev->id;
2135 }
2136
2137 target = search->host;
2138 if (target == NULL) {
2139 can = TRUE;
2140 check_type = "No target";
2141 goto search_report_results;
2142 }
2143
2144
2145
2146
2147 if (pcmk__str_eq(action, PCMK_ACTION_ON, pcmk__str_none)
2148 && !pcmk_is_set(dev->flags, st_device_supports_on)) {
2149 check_type = "Agent does not support 'on'";
2150 goto search_report_results;
2151
2152 } else if (!localhost_is_eligible_with_remap(dev, action, target,
2153 search->allow_suicide)) {
2154 check_type = "This node is not allowed to execute action";
2155 goto search_report_results;
2156 }
2157
2158
2159 check_type = target_list_type(dev);
2160 alias = g_hash_table_lookup(dev->aliases, target);
2161 if (pcmk__str_eq(check_type, PCMK__VALUE_NONE, pcmk__str_casei)) {
2162 can = TRUE;
2163
2164 } else if (pcmk__str_eq(check_type, "static-list", pcmk__str_casei)) {
2165 if (pcmk__str_in_list(target, dev->targets, pcmk__str_casei)) {
2166 can = TRUE;
2167 } else if (g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_MAP)
2168 && g_hash_table_lookup(dev->aliases, target)) {
2169 can = TRUE;
2170 }
2171
2172 } else if (pcmk__str_eq(check_type, "dynamic-list", pcmk__str_casei)) {
2173 time_t now = time(NULL);
2174
2175 if (dev->targets == NULL || dev->targets_age + 60 < now) {
2176 int device_timeout = get_action_timeout(dev, PCMK_ACTION_LIST,
2177 search->per_device_timeout);
2178
2179 if (device_timeout > search->per_device_timeout) {
2180 crm_notice("Since the pcmk_list_timeout(%ds) parameter of %s is larger than stonith-timeout(%ds), timeout may occur",
2181 device_timeout, dev_id, search->per_device_timeout);
2182 }
2183
2184 crm_trace("Running '%s' to check whether %s is eligible to fence %s (%s)",
2185 check_type, dev_id, target, action);
2186
2187 schedule_internal_command(__func__, dev, PCMK_ACTION_LIST, NULL,
2188 search->per_device_timeout, search, dynamic_list_search_cb);
2189
2190
2191 return;
2192 }
2193
2194 if (pcmk__str_in_list(((alias == NULL)? target : alias), dev->targets,
2195 pcmk__str_casei)) {
2196 can = TRUE;
2197 }
2198
2199 } else if (pcmk__str_eq(check_type, "status", pcmk__str_casei)) {
2200 int device_timeout = get_action_timeout(dev, check_type, search->per_device_timeout);
2201
2202 if (device_timeout > search->per_device_timeout) {
2203 crm_notice("Since the pcmk_status_timeout(%ds) parameter of %s is larger than stonith-timeout(%ds), timeout may occur",
2204 device_timeout, dev_id, search->per_device_timeout);
2205 }
2206
2207 crm_trace("Running '%s' to check whether %s is eligible to fence %s (%s)",
2208 check_type, dev_id, target, action);
2209 schedule_internal_command(__func__, dev, PCMK_ACTION_STATUS, target,
2210 search->per_device_timeout, search, status_search_cb);
2211
2212 return;
2213 } else {
2214 crm_err("Invalid value for " PCMK_STONITH_HOST_CHECK ": %s", check_type);
2215 check_type = "Invalid " PCMK_STONITH_HOST_CHECK;
2216 }
2217
2218 search_report_results:
2219 crm_info("%s is%s eligible to fence (%s) %s%s%s%s: %s",
2220 dev_id, (can? "" : " not"), pcmk__s(action, "unspecified action"),
2221 pcmk__s(target, "unspecified target"),
2222 (alias == NULL)? "" : " (as '", pcmk__s(alias, ""),
2223 (alias == NULL)? "" : "')", check_type);
2224 search_devices_record_result(search, ((dev == NULL)? NULL : dev_id), can);
2225 }
2226
2227 static void
2228 search_devices(gpointer key, gpointer value, gpointer user_data)
2229 {
2230 stonith_device_t *dev = value;
2231 struct device_search_s *search = user_data;
2232
2233 can_fence_host_with_device(dev, search);
2234 }
2235
2236 #define DEFAULT_QUERY_TIMEOUT 20
2237 static void
2238 get_capable_devices(const char *host, const char *action, int timeout, bool suicide, void *user_data,
2239 void (*callback) (GList * devices, void *user_data), uint32_t support_action_only)
2240 {
2241 struct device_search_s *search;
2242 guint ndevices = g_hash_table_size(device_list);
2243
2244 if (ndevices == 0) {
2245 callback(NULL, user_data);
2246 return;
2247 }
2248
2249 search = calloc(1, sizeof(struct device_search_s));
2250 if (!search) {
2251 crm_crit("Cannot search for capable fence devices: %s",
2252 strerror(ENOMEM));
2253 callback(NULL, user_data);
2254 return;
2255 }
2256
2257 pcmk__str_update(&search->host, host);
2258 pcmk__str_update(&search->action, action);
2259 search->per_device_timeout = timeout;
2260 search->allow_suicide = suicide;
2261 search->callback = callback;
2262 search->user_data = user_data;
2263 search->support_action_only = support_action_only;
2264
2265
2266
2267
2268 search->replies_needed = ndevices;
2269
2270 crm_debug("Searching %d device%s to see which can execute '%s' targeting %s",
2271 ndevices, pcmk__plural_s(ndevices),
2272 (search->action? search->action : "unknown action"),
2273 (search->host? search->host : "any node"));
2274 g_hash_table_foreach(device_list, search_devices, search);
2275 }
2276
2277 struct st_query_data {
2278 xmlNode *reply;
2279 char *remote_peer;
2280 char *client_id;
2281 char *target;
2282 char *action;
2283 int call_options;
2284 };
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295 static void
2296 add_action_specific_attributes(xmlNode *xml, const char *action,
2297 const stonith_device_t *device,
2298 const char *target)
2299 {
2300 int action_specific_timeout;
2301 int delay_max;
2302 int delay_base;
2303
2304 CRM_CHECK(xml && action && device, return);
2305
2306 if (is_action_required(action, device)) {
2307 crm_trace("Action '%s' is required using %s", action, device->id);
2308 crm_xml_add_int(xml, F_STONITH_DEVICE_REQUIRED, 1);
2309 }
2310
2311 action_specific_timeout = get_action_timeout(device, action, 0);
2312 if (action_specific_timeout) {
2313 crm_trace("Action '%s' has timeout %dms using %s",
2314 action, action_specific_timeout, device->id);
2315 crm_xml_add_int(xml, F_STONITH_ACTION_TIMEOUT, action_specific_timeout);
2316 }
2317
2318 delay_max = get_action_delay_max(device, action);
2319 if (delay_max > 0) {
2320 crm_trace("Action '%s' has maximum random delay %ds using %s",
2321 action, delay_max, device->id);
2322 crm_xml_add_int(xml, F_STONITH_DELAY_MAX, delay_max);
2323 }
2324
2325 delay_base = get_action_delay_base(device, action, target);
2326 if (delay_base > 0) {
2327 crm_xml_add_int(xml, F_STONITH_DELAY_BASE, delay_base);
2328 }
2329
2330 if ((delay_max > 0) && (delay_base == 0)) {
2331 crm_trace("Action '%s' has maximum random delay %ds using %s",
2332 action, delay_max, device->id);
2333 } else if ((delay_max == 0) && (delay_base > 0)) {
2334 crm_trace("Action '%s' has a static delay of %ds using %s",
2335 action, delay_base, device->id);
2336 } else if ((delay_max > 0) && (delay_base > 0)) {
2337 crm_trace("Action '%s' has a minimum delay of %ds and a randomly chosen "
2338 "maximum delay of %ds using %s",
2339 action, delay_base, delay_max, device->id);
2340 }
2341 }
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353 static void
2354 add_disallowed(xmlNode *xml, const char *action, const stonith_device_t *device,
2355 const char *target, gboolean allow_suicide)
2356 {
2357 if (!localhost_is_eligible(device, action, target, allow_suicide)) {
2358 crm_trace("Action '%s' using %s is disallowed for local host",
2359 action, device->id);
2360 pcmk__xe_set_bool_attr(xml, F_STONITH_ACTION_DISALLOWED, true);
2361 }
2362 }
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374 static void
2375 add_action_reply(xmlNode *xml, const char *action,
2376 const stonith_device_t *device, const char *target,
2377 gboolean allow_suicide)
2378 {
2379 xmlNode *child = create_xml_node(xml, F_STONITH_ACTION);
2380
2381 crm_xml_add(child, XML_ATTR_ID, action);
2382 add_action_specific_attributes(child, action, device, target);
2383 add_disallowed(child, action, device, target, allow_suicide);
2384 }
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395 static void
2396 stonith_send_reply(const xmlNode *reply, int call_options,
2397 const char *remote_peer, pcmk__client_t *client)
2398 {
2399 CRM_CHECK((reply != NULL) && ((remote_peer != NULL) || (client != NULL)),
2400 return);
2401
2402 if (remote_peer == NULL) {
2403 do_local_reply(reply, client, call_options);
2404 } else {
2405 send_cluster_message(crm_get_peer(0, remote_peer), crm_msg_stonith_ng,
2406 reply, FALSE);
2407 }
2408 }
2409
2410 static void
2411 stonith_query_capable_device_cb(GList * devices, void *user_data)
2412 {
2413 struct st_query_data *query = user_data;
2414 int available_devices = 0;
2415 xmlNode *dev = NULL;
2416 xmlNode *list = NULL;
2417 GList *lpc = NULL;
2418 pcmk__client_t *client = NULL;
2419
2420 if (query->client_id != NULL) {
2421 client = pcmk__find_client_by_id(query->client_id);
2422 if ((client == NULL) && (query->remote_peer == NULL)) {
2423 crm_trace("Skipping reply to %s: no longer a client",
2424 query->client_id);
2425 goto done;
2426 }
2427 }
2428
2429
2430 list = create_xml_node(NULL, __func__);
2431 crm_xml_add(list, F_STONITH_TARGET, query->target);
2432 for (lpc = devices; lpc != NULL; lpc = lpc->next) {
2433 stonith_device_t *device = g_hash_table_lookup(device_list, lpc->data);
2434 const char *action = query->action;
2435
2436 if (!device) {
2437
2438
2439 continue;
2440 }
2441
2442 available_devices++;
2443
2444 dev = create_xml_node(list, F_STONITH_DEVICE);
2445 crm_xml_add(dev, XML_ATTR_ID, device->id);
2446 crm_xml_add(dev, "namespace", device->namespace);
2447 crm_xml_add(dev, "agent", device->agent);
2448 crm_xml_add_int(dev, F_STONITH_DEVICE_VERIFIED, device->verified);
2449 crm_xml_add_int(dev, F_STONITH_DEVICE_SUPPORT_FLAGS, device->flags);
2450
2451
2452
2453
2454 if (!pcmk_is_set(device->flags, st_device_supports_reboot)
2455 && pcmk__str_eq(query->action, PCMK_ACTION_REBOOT,
2456 pcmk__str_none)) {
2457 crm_trace("%s doesn't support reboot, using values for off instead",
2458 device->id);
2459 action = PCMK_ACTION_OFF;
2460 }
2461
2462
2463 add_action_specific_attributes(dev, action, device, query->target);
2464 if (pcmk__str_eq(query->action, PCMK_ACTION_REBOOT, pcmk__str_none)) {
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476 add_disallowed(dev, action, device, query->target,
2477 pcmk_is_set(query->call_options, st_opt_allow_suicide));
2478 add_action_reply(dev, PCMK_ACTION_OFF, device, query->target,
2479 pcmk_is_set(query->call_options, st_opt_allow_suicide));
2480 add_action_reply(dev, PCMK_ACTION_ON, device, query->target, FALSE);
2481 }
2482
2483
2484 if (query->target == NULL) {
2485 xmlNode *attrs = create_xml_node(dev, XML_TAG_ATTRS);
2486
2487 g_hash_table_foreach(device->params, hash2field, attrs);
2488 }
2489 }
2490
2491 crm_xml_add_int(list, F_STONITH_AVAILABLE_DEVICES, available_devices);
2492 if (query->target) {
2493 crm_debug("Found %d matching device%s for target '%s'",
2494 available_devices, pcmk__plural_s(available_devices),
2495 query->target);
2496 } else {
2497 crm_debug("%d device%s installed",
2498 available_devices, pcmk__plural_s(available_devices));
2499 }
2500
2501 if (list != NULL) {
2502 crm_log_xml_trace(list, "Add query results");
2503 add_message_xml(query->reply, F_STONITH_CALLDATA, list);
2504 }
2505
2506 stonith_send_reply(query->reply, query->call_options, query->remote_peer,
2507 client);
2508
2509 done:
2510 free_xml(query->reply);
2511 free(query->remote_peer);
2512 free(query->client_id);
2513 free(query->target);
2514 free(query->action);
2515 free(query);
2516 free_xml(list);
2517 g_list_free_full(devices, free);
2518 }
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530 static void
2531 log_async_result(const async_command_t *cmd,
2532 const pcmk__action_result_t *result,
2533 int pid, const char *next, bool op_merged)
2534 {
2535 int log_level = LOG_ERR;
2536 int output_log_level = LOG_NEVER;
2537 guint devices_remaining = g_list_length(cmd->next_device_iter);
2538
2539 GString *msg = g_string_sized_new(80);
2540
2541
2542 if (pcmk__result_ok(result)) {
2543 log_level = (cmd->target == NULL)? LOG_DEBUG : LOG_NOTICE;
2544 if ((result->action_stdout != NULL)
2545 && !pcmk__str_eq(cmd->action, "metadata", pcmk__str_none)) {
2546 output_log_level = LOG_DEBUG;
2547 }
2548 next = NULL;
2549 } else {
2550 log_level = (cmd->target == NULL)? LOG_NOTICE : LOG_ERR;
2551 if ((result->action_stdout != NULL)
2552 && !pcmk__str_eq(cmd->action, "metadata", pcmk__str_none)) {
2553 output_log_level = LOG_WARNING;
2554 }
2555 }
2556
2557
2558 pcmk__g_strcat(msg, "Operation '", cmd->action, "' ", NULL);
2559 if (pid != 0) {
2560 g_string_append_printf(msg, "[%d] ", pid);
2561 }
2562 if (cmd->target != NULL) {
2563 pcmk__g_strcat(msg, "targeting ", cmd->target, " ", NULL);
2564 }
2565 if (cmd->device != NULL) {
2566 pcmk__g_strcat(msg, "using ", cmd->device, " ", NULL);
2567 }
2568
2569
2570 if (result->execution_status == PCMK_EXEC_DONE) {
2571 g_string_append_printf(msg, "returned %d", result->exit_status);
2572 } else {
2573 pcmk__g_strcat(msg, "could not be executed: ",
2574 pcmk_exec_status_str(result->execution_status), NULL);
2575 }
2576
2577
2578 if (result->exit_reason != NULL) {
2579 pcmk__g_strcat(msg, " (", result->exit_reason, ")", NULL);
2580 }
2581 if (next != NULL) {
2582 pcmk__g_strcat(msg, ", retrying with ", next, NULL);
2583 }
2584 if (devices_remaining > 0) {
2585 g_string_append_printf(msg, " (%u device%s remaining)",
2586 (unsigned int) devices_remaining,
2587 pcmk__plural_s(devices_remaining));
2588 }
2589 g_string_append_printf(msg, " " CRM_XS " %scall %d from %s",
2590 (op_merged? "merged " : ""), cmd->id,
2591 cmd->client_name);
2592
2593
2594 do_crm_log(log_level, "%s", msg->str);
2595 g_string_free(msg, TRUE);
2596
2597
2598 if (output_log_level != LOG_NEVER) {
2599 char *prefix = crm_strdup_printf("%s[%d]", cmd->device, pid);
2600
2601 crm_log_output(output_log_level, prefix, result->action_stdout);
2602 free(prefix);
2603 }
2604 }
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615 static void
2616 send_async_reply(const async_command_t *cmd, const pcmk__action_result_t *result,
2617 int pid, bool merged)
2618 {
2619 xmlNode *reply = NULL;
2620 pcmk__client_t *client = NULL;
2621
2622 CRM_CHECK((cmd != NULL) && (result != NULL), return);
2623
2624 log_async_result(cmd, result, pid, NULL, merged);
2625
2626 if (cmd->client != NULL) {
2627 client = pcmk__find_client_by_id(cmd->client);
2628 if ((client == NULL) && (cmd->origin == NULL)) {
2629 crm_trace("Skipping reply to %s: no longer a client", cmd->client);
2630 return;
2631 }
2632 }
2633
2634 reply = construct_async_reply(cmd, result);
2635 if (merged) {
2636 pcmk__xe_set_bool_attr(reply, F_STONITH_MERGED, true);
2637 }
2638
2639 if (!stand_alone && pcmk__is_fencing_action(cmd->action)
2640 && pcmk__str_eq(cmd->origin, cmd->target, pcmk__str_casei)) {
2641
2642
2643
2644 crm_trace("Broadcast '%s' result for %s (target was also originator)",
2645 cmd->action, cmd->target);
2646 crm_xml_add(reply, F_SUBTYPE, "broadcast");
2647 crm_xml_add(reply, F_STONITH_OPERATION, T_STONITH_NOTIFY);
2648 send_cluster_message(NULL, crm_msg_stonith_ng, reply, FALSE);
2649 } else {
2650
2651 stonith_send_reply(reply, cmd->options, cmd->origin, client);
2652 }
2653
2654 crm_log_xml_trace(reply, "Reply");
2655 free_xml(reply);
2656
2657 if (stand_alone) {
2658
2659 xmlNode *notify_data = create_xml_node(NULL, T_STONITH_NOTIFY_FENCE);
2660
2661 stonith__xe_set_result(notify_data, result);
2662 crm_xml_add(notify_data, F_STONITH_TARGET, cmd->target);
2663 crm_xml_add(notify_data, F_STONITH_OPERATION, cmd->op);
2664 crm_xml_add(notify_data, F_STONITH_DELEGATE, "localhost");
2665 crm_xml_add(notify_data, F_STONITH_DEVICE, cmd->device);
2666 crm_xml_add(notify_data, F_STONITH_REMOTE_OP_ID, cmd->remote_op_id);
2667 crm_xml_add(notify_data, F_STONITH_ORIGIN, cmd->client);
2668
2669 fenced_send_notification(T_STONITH_NOTIFY_FENCE, result, notify_data);
2670 fenced_send_notification(T_STONITH_NOTIFY_HISTORY, NULL, NULL);
2671 }
2672 }
2673
2674 static void
2675 cancel_stonith_command(async_command_t * cmd)
2676 {
2677 stonith_device_t *device = cmd_device(cmd);
2678
2679 if (device) {
2680 crm_trace("Cancel scheduled '%s' action using %s",
2681 cmd->action, device->id);
2682 device->pending_ops = g_list_remove(device->pending_ops, cmd);
2683 }
2684 }
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705 static void
2706 reply_to_duplicates(async_command_t *cmd, const pcmk__action_result_t *result,
2707 int pid)
2708 {
2709 GList *next = NULL;
2710
2711 for (GList *iter = cmd_list; iter != NULL; iter = next) {
2712 async_command_t *cmd_other = iter->data;
2713
2714 next = iter->next;
2715
2716 if (cmd == cmd_other) {
2717 continue;
2718 }
2719
2720
2721
2722
2723
2724
2725
2726 if (pcmk__str_eq(cmd->client, cmd_other->client, pcmk__str_casei) ||
2727 !pcmk__str_eq(cmd->target, cmd_other->target, pcmk__str_casei) ||
2728 !pcmk__str_eq(cmd->action, cmd_other->action, pcmk__str_none) ||
2729 !pcmk__str_eq(cmd->device, cmd_other->device, pcmk__str_casei)) {
2730
2731 continue;
2732 }
2733
2734 crm_notice("Merging fencing action '%s'%s%s originating from "
2735 "client %s with identical fencing request from client %s",
2736 cmd_other->action,
2737 (cmd_other->target == NULL)? "" : " targeting ",
2738 pcmk__s(cmd_other->target, ""), cmd_other->client_name,
2739 cmd->client_name);
2740
2741
2742 cmd_list = g_list_remove_link(cmd_list, iter);
2743 send_async_reply(cmd_other, result, pid, true);
2744 cancel_stonith_command(cmd_other);
2745
2746 free_async_command(cmd_other);
2747 g_list_free_1(iter);
2748 }
2749 }
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759 static stonith_device_t *
2760 next_required_device(async_command_t *cmd)
2761 {
2762 for (GList *iter = cmd->next_device_iter; iter != NULL; iter = iter->next) {
2763 stonith_device_t *next_device = g_hash_table_lookup(device_list,
2764 iter->data);
2765
2766 if (is_action_required(cmd->action, next_device)) {
2767
2768
2769
2770 cmd->next_device_iter = iter->next;
2771 return next_device;
2772 }
2773 }
2774 return NULL;
2775 }
2776
2777 static void
2778 st_child_done(int pid, const pcmk__action_result_t *result, void *user_data)
2779 {
2780 async_command_t *cmd = user_data;
2781
2782 stonith_device_t *device = NULL;
2783 stonith_device_t *next_device = NULL;
2784
2785 CRM_CHECK(cmd != NULL, return);
2786
2787 device = cmd_device(cmd);
2788 cmd->active_on = NULL;
2789
2790
2791 if (device) {
2792 if (!device->verified && pcmk__result_ok(result)
2793 && pcmk__strcase_any_of(cmd->action, PCMK_ACTION_LIST,
2794 PCMK_ACTION_MONITOR, PCMK_ACTION_STATUS,
2795 NULL)) {
2796
2797 device->verified = TRUE;
2798 }
2799
2800 mainloop_set_trigger(device->work);
2801 }
2802
2803 if (pcmk__result_ok(result)) {
2804 next_device = next_required_device(cmd);
2805
2806 } else if ((cmd->next_device_iter != NULL)
2807 && !is_action_required(cmd->action, device)) {
2808
2809
2810 next_device = g_hash_table_lookup(device_list,
2811 cmd->next_device_iter->data);
2812 cmd->next_device_iter = cmd->next_device_iter->next;
2813 }
2814
2815 if (next_device == NULL) {
2816 send_async_reply(cmd, result, pid, false);
2817 if (pcmk__result_ok(result)) {
2818 reply_to_duplicates(cmd, result, pid);
2819 }
2820 free_async_command(cmd);
2821
2822 } else {
2823 log_async_result(cmd, result, pid, next_device->id, false);
2824 schedule_stonith_command(cmd, next_device);
2825 }
2826 }
2827
2828 static gint
2829 sort_device_priority(gconstpointer a, gconstpointer b)
2830 {
2831 const stonith_device_t *dev_a = a;
2832 const stonith_device_t *dev_b = b;
2833
2834 if (dev_a->priority > dev_b->priority) {
2835 return -1;
2836 } else if (dev_a->priority < dev_b->priority) {
2837 return 1;
2838 }
2839 return 0;
2840 }
2841
2842 static void
2843 stonith_fence_get_devices_cb(GList * devices, void *user_data)
2844 {
2845 async_command_t *cmd = user_data;
2846 stonith_device_t *device = NULL;
2847 guint ndevices = g_list_length(devices);
2848
2849 crm_info("Found %d matching device%s for target '%s'",
2850 ndevices, pcmk__plural_s(ndevices), cmd->target);
2851
2852 if (devices != NULL) {
2853
2854 devices = g_list_sort(devices, sort_device_priority);
2855 device = g_hash_table_lookup(device_list, devices->data);
2856 }
2857
2858 if (device == NULL) {
2859 pcmk__action_result_t result = PCMK__UNKNOWN_RESULT;
2860
2861 pcmk__format_result(&result, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE,
2862 "No device configured for target '%s'",
2863 cmd->target);
2864 send_async_reply(cmd, &result, 0, false);
2865 pcmk__reset_result(&result);
2866 free_async_command(cmd);
2867 g_list_free_full(devices, free);
2868
2869 } else {
2870 cmd->device_list = devices;
2871 cmd->next_device_iter = devices->next;
2872 schedule_stonith_command(cmd, device);
2873 }
2874 }
2875
2876
2877
2878
2879
2880
2881
2882
2883 static void
2884 fence_locally(xmlNode *msg, pcmk__action_result_t *result)
2885 {
2886 const char *device_id = NULL;
2887 stonith_device_t *device = NULL;
2888 async_command_t *cmd = NULL;
2889 xmlNode *dev = NULL;
2890
2891 CRM_CHECK((msg != NULL) && (result != NULL), return);
2892
2893 dev = get_xpath_object("//@" F_STONITH_TARGET, msg, LOG_ERR);
2894
2895 cmd = create_async_command(msg);
2896 if (cmd == NULL) {
2897 crm_log_xml_warn(msg, "invalid");
2898 fenced_set_protocol_error(result);
2899 return;
2900 }
2901
2902 device_id = crm_element_value(dev, F_STONITH_DEVICE);
2903 if (device_id != NULL) {
2904 device = g_hash_table_lookup(device_list, device_id);
2905 if (device == NULL) {
2906 crm_err("Requested device '%s' is not available", device_id);
2907 pcmk__format_result(result, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE,
2908 "Requested device '%s' not found", device_id);
2909 return;
2910 }
2911 schedule_stonith_command(cmd, device);
2912
2913 } else {
2914 const char *host = crm_element_value(dev, F_STONITH_TARGET);
2915
2916 if (pcmk_is_set(cmd->options, st_opt_cs_nodeid)) {
2917 int nodeid = 0;
2918 crm_node_t *node = NULL;
2919
2920 pcmk__scan_min_int(host, &nodeid, 0);
2921 node = pcmk__search_known_node_cache(nodeid, NULL, CRM_GET_PEER_ANY);
2922 if (node != NULL) {
2923 host = node->uname;
2924 }
2925 }
2926
2927
2928 get_capable_devices(host, cmd->action, cmd->default_timeout,
2929 TRUE, cmd, stonith_fence_get_devices_cb,
2930 fenced_support_flag(cmd->action));
2931 }
2932
2933 pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_PENDING, NULL);
2934 }
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950 xmlNode *
2951 fenced_construct_reply(const xmlNode *request, xmlNode *data,
2952 const pcmk__action_result_t *result)
2953 {
2954 xmlNode *reply = NULL;
2955
2956 reply = create_xml_node(NULL, T_STONITH_REPLY);
2957
2958 crm_xml_add(reply, "st_origin", __func__);
2959 crm_xml_add(reply, F_TYPE, T_STONITH_NG);
2960 stonith__xe_set_result(reply, result);
2961
2962 if (request == NULL) {
2963
2964
2965
2966
2967
2968
2969 crm_warn("Missing request information for client notifications for "
2970 "operation with result '%s' (initiated before we came up?)",
2971 pcmk_exec_status_str(result->execution_status));
2972
2973 } else {
2974 const char *name = NULL;
2975 const char *value = NULL;
2976
2977
2978 const char *names[] = {
2979 F_STONITH_OPERATION,
2980 F_STONITH_CALLID,
2981 F_STONITH_CLIENTID,
2982 F_STONITH_CLIENTNAME,
2983 F_STONITH_REMOTE_OP_ID,
2984 F_STONITH_CALLOPTS
2985 };
2986
2987 for (int lpc = 0; lpc < PCMK__NELEM(names); lpc++) {
2988 name = names[lpc];
2989 value = crm_element_value(request, name);
2990 crm_xml_add(reply, name, value);
2991 }
2992 if (data != NULL) {
2993 add_message_xml(reply, F_STONITH_CALLDATA, data);
2994 }
2995 }
2996 return reply;
2997 }
2998
2999
3000
3001
3002
3003
3004
3005
3006 static xmlNode *
3007 construct_async_reply(const async_command_t *cmd,
3008 const pcmk__action_result_t *result)
3009 {
3010 xmlNode *reply = create_xml_node(NULL, T_STONITH_REPLY);
3011
3012 crm_xml_add(reply, "st_origin", __func__);
3013 crm_xml_add(reply, F_TYPE, T_STONITH_NG);
3014 crm_xml_add(reply, F_STONITH_OPERATION, cmd->op);
3015 crm_xml_add(reply, F_STONITH_DEVICE, cmd->device);
3016 crm_xml_add(reply, F_STONITH_REMOTE_OP_ID, cmd->remote_op_id);
3017 crm_xml_add(reply, F_STONITH_CLIENTID, cmd->client);
3018 crm_xml_add(reply, F_STONITH_CLIENTNAME, cmd->client_name);
3019 crm_xml_add(reply, F_STONITH_TARGET, cmd->target);
3020 crm_xml_add(reply, F_STONITH_ACTION, cmd->op);
3021 crm_xml_add(reply, F_STONITH_ORIGIN, cmd->origin);
3022 crm_xml_add_int(reply, F_STONITH_CALLID, cmd->id);
3023 crm_xml_add_int(reply, F_STONITH_CALLOPTS, cmd->options);
3024
3025 stonith__xe_set_result(reply, result);
3026 return reply;
3027 }
3028
3029 bool fencing_peer_active(crm_node_t *peer)
3030 {
3031 if (peer == NULL) {
3032 return FALSE;
3033 } else if (peer->uname == NULL) {
3034 return FALSE;
3035 } else if (pcmk_is_set(peer->processes, crm_get_cluster_proc())) {
3036 return TRUE;
3037 }
3038 return FALSE;
3039 }
3040
3041 void
3042 set_fencing_completed(remote_fencing_op_t *op)
3043 {
3044 struct timespec tv;
3045
3046 qb_util_timespec_from_epoch_get(&tv);
3047 op->completed = tv.tv_sec;
3048 op->completed_nsec = tv.tv_nsec;
3049 }
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060 static const char *
3061 check_alternate_host(const char *target)
3062 {
3063 if (pcmk__str_eq(target, stonith_our_uname, pcmk__str_casei)) {
3064 GHashTableIter gIter;
3065 crm_node_t *entry = NULL;
3066
3067 g_hash_table_iter_init(&gIter, crm_peer_cache);
3068 while (g_hash_table_iter_next(&gIter, NULL, (void **)&entry)) {
3069 if (fencing_peer_active(entry)
3070 && !pcmk__str_eq(entry->uname, target, pcmk__str_casei)) {
3071 crm_notice("Forwarding self-fencing request to %s",
3072 entry->uname);
3073 return entry->uname;
3074 }
3075 }
3076 crm_warn("Will handle own fencing because no peer can");
3077 }
3078 return NULL;
3079 }
3080
3081 static void
3082 remove_relay_op(xmlNode * request)
3083 {
3084 xmlNode *dev = get_xpath_object("//@" F_STONITH_ACTION, request, LOG_TRACE);
3085 const char *relay_op_id = NULL;
3086 const char *op_id = NULL;
3087 const char *client_name = NULL;
3088 const char *target = NULL;
3089 remote_fencing_op_t *relay_op = NULL;
3090
3091 if (dev) {
3092 target = crm_element_value(dev, F_STONITH_TARGET);
3093 }
3094
3095 relay_op_id = crm_element_value(request, F_STONITH_REMOTE_OP_ID_RELAY);
3096 op_id = crm_element_value(request, F_STONITH_REMOTE_OP_ID);
3097 client_name = crm_element_value(request, F_STONITH_CLIENTNAME);
3098
3099
3100 if (relay_op_id && target && pcmk__str_eq(target, stonith_our_uname, pcmk__str_casei)) {
3101 relay_op = g_hash_table_lookup(stonith_remote_op_list, relay_op_id);
3102
3103 if (relay_op) {
3104 GHashTableIter iter;
3105 remote_fencing_op_t *list_op = NULL;
3106 g_hash_table_iter_init(&iter, stonith_remote_op_list);
3107
3108
3109 while (g_hash_table_iter_next(&iter, NULL, (void **)&list_op)) {
3110 GList *dup_iter = NULL;
3111 if (list_op != relay_op) {
3112 for (dup_iter = list_op->duplicates; dup_iter != NULL; dup_iter = dup_iter->next) {
3113 remote_fencing_op_t *other = dup_iter->data;
3114 if (other == relay_op) {
3115 other->duplicates = g_list_remove(other->duplicates, relay_op);
3116 break;
3117 }
3118 }
3119 }
3120 }
3121 crm_debug("Deleting relay op %s ('%s'%s%s for %s), "
3122 "replaced by op %s ('%s'%s%s for %s)",
3123 relay_op->id, relay_op->action,
3124 (relay_op->target == NULL)? "" : " targeting ",
3125 pcmk__s(relay_op->target, ""),
3126 relay_op->client_name, op_id, relay_op->action,
3127 (target == NULL)? "" : " targeting ", pcmk__s(target, ""),
3128 client_name);
3129
3130 g_hash_table_remove(stonith_remote_op_list, relay_op_id);
3131 }
3132 }
3133 }
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149 static inline bool
3150 is_privileged(const pcmk__client_t *c, const char *op)
3151 {
3152 if ((c == NULL) || pcmk_is_set(c->flags, pcmk__client_privileged)) {
3153 return true;
3154 } else {
3155 crm_warn("Rejecting IPC request '%s' from unprivileged client %s",
3156 pcmk__s(op, ""), pcmk__client_name(c));
3157 return false;
3158 }
3159 }
3160
3161
3162 static xmlNode *
3163 handle_register_request(pcmk__request_t *request)
3164 {
3165 xmlNode *reply = create_xml_node(NULL, "reply");
3166
3167 CRM_ASSERT(request->ipc_client != NULL);
3168 crm_xml_add(reply, F_STONITH_OPERATION, CRM_OP_REGISTER);
3169 crm_xml_add(reply, F_STONITH_CLIENTID, request->ipc_client->id);
3170 pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
3171 pcmk__set_request_flags(request, pcmk__request_reuse_options);
3172 return reply;
3173 }
3174
3175
3176 static xmlNode *
3177 handle_agent_request(pcmk__request_t *request)
3178 {
3179 execute_agent_action(request->xml, &request->result);
3180 if (request->result.execution_status == PCMK_EXEC_PENDING) {
3181 return NULL;
3182 }
3183 return fenced_construct_reply(request->xml, NULL, &request->result);
3184 }
3185
3186
3187 static xmlNode *
3188 handle_update_timeout_request(pcmk__request_t *request)
3189 {
3190 const char *call_id = crm_element_value(request->xml, F_STONITH_CALLID);
3191 const char *client_id = crm_element_value(request->xml, F_STONITH_CLIENTID);
3192 int op_timeout = 0;
3193
3194 crm_element_value_int(request->xml, F_STONITH_TIMEOUT, &op_timeout);
3195 do_stonith_async_timeout_update(client_id, call_id, op_timeout);
3196 pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
3197 return NULL;
3198 }
3199
3200
3201 static xmlNode *
3202 handle_query_request(pcmk__request_t *request)
3203 {
3204 int timeout = 0;
3205 xmlNode *dev = NULL;
3206 const char *action = NULL;
3207 const char *target = NULL;
3208 const char *client_id = crm_element_value(request->xml, F_STONITH_CLIENTID);
3209 struct st_query_data *query = NULL;
3210
3211 if (request->peer != NULL) {
3212
3213 create_remote_stonith_op(client_id, request->xml, TRUE);
3214 }
3215
3216
3217 remove_relay_op(request->xml);
3218
3219 pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
3220
3221 dev = get_xpath_object("//@" F_STONITH_ACTION, request->xml, LOG_NEVER);
3222 if (dev != NULL) {
3223 const char *device = crm_element_value(dev, F_STONITH_DEVICE);
3224
3225 if (pcmk__str_eq(device, "manual_ack", pcmk__str_casei)) {
3226 return NULL;
3227 }
3228 target = crm_element_value(dev, F_STONITH_TARGET);
3229 action = crm_element_value(dev, F_STONITH_ACTION);
3230 }
3231
3232 crm_log_xml_trace(request->xml, "Query");
3233
3234 query = calloc(1, sizeof(struct st_query_data));
3235 CRM_ASSERT(query != NULL);
3236
3237 query->reply = fenced_construct_reply(request->xml, NULL, &request->result);
3238 pcmk__str_update(&query->remote_peer, request->peer);
3239 pcmk__str_update(&query->client_id, client_id);
3240 pcmk__str_update(&query->target, target);
3241 pcmk__str_update(&query->action, action);
3242 query->call_options = request->call_options;
3243
3244 crm_element_value_int(request->xml, F_STONITH_TIMEOUT, &timeout);
3245 get_capable_devices(target, action, timeout,
3246 pcmk_is_set(query->call_options, st_opt_allow_suicide),
3247 query, stonith_query_capable_device_cb, st_device_supports_none);
3248 return NULL;
3249 }
3250
3251
3252 static xmlNode *
3253 handle_notify_request(pcmk__request_t *request)
3254 {
3255 const char *flag_name = NULL;
3256
3257 CRM_ASSERT(request->ipc_client != NULL);
3258 flag_name = crm_element_value(request->xml, F_STONITH_NOTIFY_ACTIVATE);
3259 if (flag_name != NULL) {
3260 crm_debug("Enabling %s callbacks for client %s",
3261 flag_name, pcmk__request_origin(request));
3262 pcmk__set_client_flags(request->ipc_client, get_stonith_flag(flag_name));
3263 }
3264
3265 flag_name = crm_element_value(request->xml, F_STONITH_NOTIFY_DEACTIVATE);
3266 if (flag_name != NULL) {
3267 crm_debug("Disabling %s callbacks for client %s",
3268 flag_name, pcmk__request_origin(request));
3269 pcmk__clear_client_flags(request->ipc_client,
3270 get_stonith_flag(flag_name));
3271 }
3272
3273 pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
3274 pcmk__set_request_flags(request, pcmk__request_reuse_options);
3275
3276 return pcmk__ipc_create_ack(request->ipc_flags, "ack", NULL, CRM_EX_OK);
3277 }
3278
3279
3280 static xmlNode *
3281 handle_relay_request(pcmk__request_t *request)
3282 {
3283 xmlNode *dev = get_xpath_object("//@" F_STONITH_TARGET, request->xml,
3284 LOG_TRACE);
3285
3286 crm_notice("Received forwarded fencing request from "
3287 "%s %s to fence (%s) peer %s",
3288 pcmk__request_origin_type(request),
3289 pcmk__request_origin(request),
3290 crm_element_value(dev, F_STONITH_ACTION),
3291 crm_element_value(dev, F_STONITH_TARGET));
3292
3293 if (initiate_remote_stonith_op(NULL, request->xml, FALSE) == NULL) {
3294 fenced_set_protocol_error(&request->result);
3295 return fenced_construct_reply(request->xml, NULL, &request->result);
3296 }
3297
3298 pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_PENDING, NULL);
3299 return NULL;
3300 }
3301
3302
3303 static xmlNode *
3304 handle_fence_request(pcmk__request_t *request)
3305 {
3306 if ((request->peer != NULL) || stand_alone) {
3307 fence_locally(request->xml, &request->result);
3308
3309 } else if (pcmk_is_set(request->call_options, st_opt_manual_ack)) {
3310 switch (fenced_handle_manual_confirmation(request->ipc_client,
3311 request->xml)) {
3312 case pcmk_rc_ok:
3313 pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE,
3314 NULL);
3315 break;
3316 case EINPROGRESS:
3317 pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_PENDING,
3318 NULL);
3319 break;
3320 default:
3321 fenced_set_protocol_error(&request->result);
3322 break;
3323 }
3324
3325 } else {
3326 const char *alternate_host = NULL;
3327 xmlNode *dev = get_xpath_object("//@" F_STONITH_TARGET, request->xml,
3328 LOG_TRACE);
3329 const char *target = crm_element_value(dev, F_STONITH_TARGET);
3330 const char *action = crm_element_value(dev, F_STONITH_ACTION);
3331 const char *device = crm_element_value(dev, F_STONITH_DEVICE);
3332
3333 if (request->ipc_client != NULL) {
3334 int tolerance = 0;
3335
3336 crm_notice("Client %s wants to fence (%s) %s using %s",
3337 pcmk__request_origin(request), action,
3338 target, (device? device : "any device"));
3339 crm_element_value_int(dev, F_STONITH_TOLERANCE, &tolerance);
3340 if (stonith_check_fence_tolerance(tolerance, target, action)) {
3341 pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE,
3342 NULL);
3343 return fenced_construct_reply(request->xml, NULL,
3344 &request->result);
3345 }
3346 alternate_host = check_alternate_host(target);
3347
3348 } else {
3349 crm_notice("Peer %s wants to fence (%s) '%s' with device '%s'",
3350 request->peer, action, target,
3351 (device == NULL)? "(any)" : device);
3352 }
3353
3354 if (alternate_host != NULL) {
3355 const char *client_id = NULL;
3356 remote_fencing_op_t *op = NULL;
3357
3358 if (request->ipc_client->id == 0) {
3359 client_id = crm_element_value(request->xml, F_STONITH_CLIENTID);
3360 } else {
3361 client_id = request->ipc_client->id;
3362 }
3363
3364
3365
3366
3367
3368 op = create_remote_stonith_op(client_id, request->xml, FALSE);
3369
3370 crm_xml_add(request->xml, F_STONITH_OPERATION, STONITH_OP_RELAY);
3371 crm_xml_add(request->xml, F_STONITH_CLIENTID,
3372 request->ipc_client->id);
3373 crm_xml_add(request->xml, F_STONITH_REMOTE_OP_ID, op->id);
3374 send_cluster_message(crm_get_peer(0, alternate_host),
3375 crm_msg_stonith_ng, request->xml, FALSE);
3376 pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_PENDING,
3377 NULL);
3378
3379 } else if (initiate_remote_stonith_op(request->ipc_client, request->xml,
3380 FALSE) == NULL) {
3381 fenced_set_protocol_error(&request->result);
3382
3383 } else {
3384 pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_PENDING,
3385 NULL);
3386 }
3387 }
3388
3389 if (request->result.execution_status == PCMK_EXEC_PENDING) {
3390 return NULL;
3391 }
3392 return fenced_construct_reply(request->xml, NULL, &request->result);
3393 }
3394
3395
3396 static xmlNode *
3397 handle_history_request(pcmk__request_t *request)
3398 {
3399 xmlNode *reply = NULL;
3400 xmlNode *data = NULL;
3401
3402 stonith_fence_history(request->xml, &data, request->peer,
3403 request->call_options);
3404 pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
3405 if (!pcmk_is_set(request->call_options, st_opt_discard_reply)) {
3406
3407
3408
3409 reply = fenced_construct_reply(request->xml, data, &request->result);
3410 }
3411 free_xml(data);
3412 return reply;
3413 }
3414
3415
3416 static xmlNode *
3417 handle_device_add_request(pcmk__request_t *request)
3418 {
3419 const char *op = crm_element_value(request->xml, F_STONITH_OPERATION);
3420 xmlNode *dev = get_xpath_object("//" F_STONITH_DEVICE, request->xml,
3421 LOG_ERR);
3422
3423 if (is_privileged(request->ipc_client, op)) {
3424 int rc = stonith_device_register(dev, FALSE);
3425
3426 pcmk__set_result(&request->result,
3427 ((rc == pcmk_ok)? CRM_EX_OK : CRM_EX_ERROR),
3428 stonith__legacy2status(rc),
3429 ((rc == pcmk_ok)? NULL : pcmk_strerror(rc)));
3430 } else {
3431 pcmk__set_result(&request->result, CRM_EX_INSUFFICIENT_PRIV,
3432 PCMK_EXEC_INVALID,
3433 "Unprivileged users must register device via CIB");
3434 }
3435 fenced_send_device_notification(op, &request->result,
3436 (dev == NULL)? NULL : ID(dev));
3437 return fenced_construct_reply(request->xml, NULL, &request->result);
3438 }
3439
3440
3441 static xmlNode *
3442 handle_device_delete_request(pcmk__request_t *request)
3443 {
3444 xmlNode *dev = get_xpath_object("//" F_STONITH_DEVICE, request->xml,
3445 LOG_ERR);
3446 const char *device_id = crm_element_value(dev, XML_ATTR_ID);
3447 const char *op = crm_element_value(request->xml, F_STONITH_OPERATION);
3448
3449 if (is_privileged(request->ipc_client, op)) {
3450 stonith_device_remove(device_id, false);
3451 pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
3452 } else {
3453 pcmk__set_result(&request->result, CRM_EX_INSUFFICIENT_PRIV,
3454 PCMK_EXEC_INVALID,
3455 "Unprivileged users must delete device via CIB");
3456 }
3457 fenced_send_device_notification(op, &request->result, device_id);
3458 return fenced_construct_reply(request->xml, NULL, &request->result);
3459 }
3460
3461
3462 static xmlNode *
3463 handle_level_add_request(pcmk__request_t *request)
3464 {
3465 char *desc = NULL;
3466 const char *op = crm_element_value(request->xml, F_STONITH_OPERATION);
3467
3468 if (is_privileged(request->ipc_client, op)) {
3469 fenced_register_level(request->xml, &desc, &request->result);
3470 } else {
3471 unpack_level_request(request->xml, NULL, NULL, NULL, &desc);
3472 pcmk__set_result(&request->result, CRM_EX_INSUFFICIENT_PRIV,
3473 PCMK_EXEC_INVALID,
3474 "Unprivileged users must add level via CIB");
3475 }
3476 fenced_send_level_notification(op, &request->result, desc);
3477 free(desc);
3478 return fenced_construct_reply(request->xml, NULL, &request->result);
3479 }
3480
3481
3482 static xmlNode *
3483 handle_level_delete_request(pcmk__request_t *request)
3484 {
3485 char *desc = NULL;
3486 const char *op = crm_element_value(request->xml, F_STONITH_OPERATION);
3487
3488 if (is_privileged(request->ipc_client, op)) {
3489 fenced_unregister_level(request->xml, &desc, &request->result);
3490 } else {
3491 unpack_level_request(request->xml, NULL, NULL, NULL, &desc);
3492 pcmk__set_result(&request->result, CRM_EX_INSUFFICIENT_PRIV,
3493 PCMK_EXEC_INVALID,
3494 "Unprivileged users must delete level via CIB");
3495 }
3496 fenced_send_level_notification(op, &request->result, desc);
3497 free(desc);
3498 return fenced_construct_reply(request->xml, NULL, &request->result);
3499 }
3500
3501
3502 static xmlNode *
3503 handle_cache_request(pcmk__request_t *request)
3504 {
3505 int node_id = 0;
3506 const char *name = NULL;
3507
3508 crm_element_value_int(request->xml, XML_ATTR_ID, &node_id);
3509 name = crm_element_value(request->xml, XML_ATTR_UNAME);
3510 reap_crm_member(node_id, name);
3511 pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
3512 return NULL;
3513 }
3514
3515 static xmlNode *
3516 handle_unknown_request(pcmk__request_t *request)
3517 {
3518 crm_err("Unknown IPC request %s from %s %s",
3519 request->op, pcmk__request_origin_type(request),
3520 pcmk__request_origin(request));
3521 pcmk__format_result(&request->result, CRM_EX_PROTOCOL, PCMK_EXEC_INVALID,
3522 "Unknown IPC request type '%s' (bug?)", request->op);
3523 return fenced_construct_reply(request->xml, NULL, &request->result);
3524 }
3525
3526 static void
3527 fenced_register_handlers(void)
3528 {
3529 pcmk__server_command_t handlers[] = {
3530 { CRM_OP_REGISTER, handle_register_request },
3531 { STONITH_OP_EXEC, handle_agent_request },
3532 { STONITH_OP_TIMEOUT_UPDATE, handle_update_timeout_request },
3533 { STONITH_OP_QUERY, handle_query_request },
3534 { T_STONITH_NOTIFY, handle_notify_request },
3535 { STONITH_OP_RELAY, handle_relay_request },
3536 { STONITH_OP_FENCE, handle_fence_request },
3537 { STONITH_OP_FENCE_HISTORY, handle_history_request },
3538 { STONITH_OP_DEVICE_ADD, handle_device_add_request },
3539 { STONITH_OP_DEVICE_DEL, handle_device_delete_request },
3540 { STONITH_OP_LEVEL_ADD, handle_level_add_request },
3541 { STONITH_OP_LEVEL_DEL, handle_level_delete_request },
3542 { CRM_OP_RM_NODE_CACHE, handle_cache_request },
3543 { NULL, handle_unknown_request },
3544 };
3545
3546 fenced_handlers = pcmk__register_handlers(handlers);
3547 }
3548
3549 void
3550 fenced_unregister_handlers(void)
3551 {
3552 if (fenced_handlers != NULL) {
3553 g_hash_table_destroy(fenced_handlers);
3554 fenced_handlers = NULL;
3555 }
3556 }
3557
3558 static void
3559 handle_request(pcmk__request_t *request)
3560 {
3561 xmlNode *reply = NULL;
3562 const char *reason = NULL;
3563
3564 if (fenced_handlers == NULL) {
3565 fenced_register_handlers();
3566 }
3567 reply = pcmk__process_request(request, fenced_handlers);
3568 if (reply != NULL) {
3569 if (pcmk_is_set(request->flags, pcmk__request_reuse_options)
3570 && (request->ipc_client != NULL)) {
3571
3572
3573
3574
3575 pcmk__ipc_send_xml(request->ipc_client, request->ipc_id, reply,
3576 request->ipc_flags);
3577 request->ipc_client->request_id = 0;
3578 } else {
3579 stonith_send_reply(reply, request->call_options,
3580 request->peer, request->ipc_client);
3581 }
3582 free_xml(reply);
3583 }
3584
3585 reason = request->result.exit_reason;
3586 crm_debug("Processed %s request from %s %s: %s%s%s%s",
3587 request->op, pcmk__request_origin_type(request),
3588 pcmk__request_origin(request),
3589 pcmk_exec_status_str(request->result.execution_status),
3590 (reason == NULL)? "" : " (",
3591 (reason == NULL)? "" : reason,
3592 (reason == NULL)? "" : ")");
3593 }
3594
3595 static void
3596 handle_reply(pcmk__client_t *client, xmlNode *request, const char *remote_peer)
3597 {
3598
3599 char *op = crm_element_value_copy(request, F_STONITH_OPERATION);
3600
3601 if (pcmk__str_eq(op, STONITH_OP_QUERY, pcmk__str_none)) {
3602 process_remote_stonith_query(request);
3603 } else if (pcmk__str_any_of(op, T_STONITH_NOTIFY, STONITH_OP_FENCE, NULL)) {
3604 fenced_process_fencing_reply(request);
3605 } else {
3606 crm_err("Ignoring unknown %s reply from %s %s",
3607 pcmk__s(op, "untyped"), ((client == NULL)? "peer" : "client"),
3608 ((client == NULL)? remote_peer : pcmk__client_name(client)));
3609 crm_log_xml_warn(request, "UnknownOp");
3610 free(op);
3611 return;
3612 }
3613 crm_debug("Processed %s reply from %s %s",
3614 op, ((client == NULL)? "peer" : "client"),
3615 ((client == NULL)? remote_peer : pcmk__client_name(client)));
3616 free(op);
3617 }
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629 void
3630 stonith_command(pcmk__client_t *client, uint32_t id, uint32_t flags,
3631 xmlNode *message, const char *remote_peer)
3632 {
3633 int call_options = st_opt_none;
3634 bool is_reply = false;
3635
3636 CRM_CHECK(message != NULL, return);
3637
3638 if (get_xpath_object("//" T_STONITH_REPLY, message, LOG_NEVER) != NULL) {
3639 is_reply = true;
3640 }
3641 crm_element_value_int(message, F_STONITH_CALLOPTS, &call_options);
3642 crm_debug("Processing %ssynchronous %s %s %u from %s %s",
3643 pcmk_is_set(call_options, st_opt_sync_call)? "" : "a",
3644 crm_element_value(message, F_STONITH_OPERATION),
3645 (is_reply? "reply" : "request"), id,
3646 ((client == NULL)? "peer" : "client"),
3647 ((client == NULL)? remote_peer : pcmk__client_name(client)));
3648
3649 if (pcmk_is_set(call_options, st_opt_sync_call)) {
3650 CRM_ASSERT(client == NULL || client->request_id == id);
3651 }
3652
3653 if (is_reply) {
3654 handle_reply(client, message, remote_peer);
3655 } else {
3656 pcmk__request_t request = {
3657 .ipc_client = client,
3658 .ipc_id = id,
3659 .ipc_flags = flags,
3660 .peer = remote_peer,
3661 .xml = message,
3662 .call_options = call_options,
3663 .result = PCMK__UNKNOWN_RESULT,
3664 };
3665
3666 request.op = crm_element_value_copy(request.xml, F_STONITH_OPERATION);
3667 CRM_CHECK(request.op != NULL, return);
3668
3669 if (pcmk_is_set(request.call_options, st_opt_sync_call)) {
3670 pcmk__set_request_flags(&request, pcmk__request_sync);
3671 }
3672
3673 handle_request(&request);
3674 pcmk__reset_request(&request);
3675 }
3676 }
![[first]](../icons/first.png){kind=icon}
![[previous]](../icons/n_left.png){kind=icon}
![[next]](../icons/n_right.png){kind=icon}
![[last]](../icons/last.png){kind=icon}
![[top]](../icons/n_top.png){kind=icon}
![[bottom]](../icons/bottom.png){kind=icon}
![[index]](../icons/index.png){kind=icon}
![[help]](../icons/help.png){kind=icon}
*/
![[next]](../icons/right.png)
![[first]](../icons/n_first.png)
![[top]](../icons/top.png)
![[previous]](../icons/left.png)
![[last]](../icons/n_last.png)
![[bottom]](../icons/n_bottom.png){kind=icon}