lib/pengine/bundle.c

/* */
This source file includes following definitions.
pe__bundle_max
pe__bundled_resource
pe__get_rsc_in_container
pe__node_is_bundle_instance
pe__first_container
pe__foreach_bundle_replica
pe__foreach_const_bundle_replica
next_ip
allocate_ip
create_resource
valid_network
create_ip_resource
container_agent_str
create_container_resource
disallow_node
create_remote_resource
create_replica_resources
mount_add
mount_free
port_free
replica_for_remote
pe__bundle_needs_remote_name
pe__add_bundle_remote_name
pe__unpack_bundle
replica_resource_active
pe__bundle_active
pe__find_bundle_replica
print_rsc_in_list
bundle_print_xml
PCMK__OUTPUT_ARGS
pe__bundle_replica_output_html
get_unmanaged_str
PCMK__OUTPUT_ARGS
pe__bundle_replica_output_text
PCMK__OUTPUT_ARGS
print_bundle_replica
pe__print_bundle
free_bundle_replica
pe__free_bundle
pe__bundle_resource_state
pe_bundle_replicas
pe__count_bundle
pe__bundle_is_filtered
pe__bundle_containers
pe__bundle_active_node
pe__bundle_max_per_node
   1 /*
   2  * Copyright 2004-2023 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU Lesser General Public License
   7  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <ctype.h>
  13 #include <stdint.h>
  14 
  15 #include <crm/pengine/rules.h>
  16 #include <crm/pengine/status.h>
  17 #include <crm/pengine/internal.h>
  18 #include <crm/msg_xml.h>
  19 #include <crm/common/output.h>
  20 #include <crm/common/xml_internal.h>
  21 #include <pe_status_private.h>
  22 
  23 enum pe__bundle_mount_flags {
  24     pe__bundle_mount_none       = 0x00,
  25 
  26     // mount instance-specific subdirectory rather than source directly
  27     pe__bundle_mount_subdir     = 0x01
  28 };
  29 
  30 typedef struct {
  31     char *source;
  32     char *target;
  33     char *options;
  34     uint32_t flags; // bitmask of pe__bundle_mount_flags
  35 } pe__bundle_mount_t;
  36 
  37 typedef struct {
  38     char *source;
  39     char *target;
  40 } pe__bundle_port_t;
  41 
  42 enum pe__container_agent {
  43     PE__CONTAINER_AGENT_UNKNOWN,
  44     PE__CONTAINER_AGENT_DOCKER,
  45     PE__CONTAINER_AGENT_RKT,
  46     PE__CONTAINER_AGENT_PODMAN,
  47 };
  48 
  49 #define PE__CONTAINER_AGENT_UNKNOWN_S "unknown"
  50 #define PE__CONTAINER_AGENT_DOCKER_S  "docker"
  51 #define PE__CONTAINER_AGENT_RKT_S     "rkt"
  52 #define PE__CONTAINER_AGENT_PODMAN_S  "podman"
  53 
  54 typedef struct pe__bundle_variant_data_s {
  55         int promoted_max;
  56         int nreplicas;
  57         int nreplicas_per_host;
  58         char *prefix;
  59         char *image;
  60         const char *ip_last;
  61         char *host_network;
  62         char *host_netmask;
  63         char *control_port;
  64         char *container_network;
  65         char *ip_range_start;
  66         gboolean add_host;
  67         gchar *container_host_options;
  68         char *container_command;
  69         char *launcher_options;
  70         const char *attribute_target;
  71 
  72         pcmk_resource_t *child;
  73 
  74         GList *replicas;    // pe__bundle_replica_t *
  75         GList *ports;       // pe__bundle_port_t *
  76         GList *mounts;      // pe__bundle_mount_t *
  77 
  78         enum pe__container_agent agent_type;
  79 } pe__bundle_variant_data_t;
  80 
  81 #define get_bundle_variant_data(data, rsc)                      \
  82     CRM_ASSERT(rsc != NULL);                                    \
  83     CRM_ASSERT(rsc->variant == pcmk_rsc_variant_bundle);        \
  84     CRM_ASSERT(rsc->variant_opaque != NULL);                    \
  85     data = (pe__bundle_variant_data_t *) rsc->variant_opaque;
  86 
  87 /*!
  88  * \internal
  89  * \brief Get maximum number of bundle replicas allowed to run
  90  *
  91  * \param[in] rsc  Bundle or bundled resource to check
  92  *
  93  * \return Maximum replicas for bundle corresponding to \p rsc
  94  */
  95 int
  96 pe__bundle_max(const pcmk_resource_t *rsc)
     /*  */
  97 {
  98     const pe__bundle_variant_data_t *bundle_data = NULL;
  99 
 100     get_bundle_variant_data(bundle_data, pe__const_top_resource(rsc, true));
 101     return bundle_data->nreplicas;
 102 }
 103 
 104 /*!
 105  * \internal
 106  * \brief Get the resource inside a bundle
 107  *
 108  * \param[in] bundle  Bundle to check
 109  *
 110  * \return Resource inside \p bundle if any, otherwise NULL
 111  */
 112 pcmk_resource_t *
 113 pe__bundled_resource(const pcmk_resource_t *rsc)
     /*  */
 114 {
 115     const pe__bundle_variant_data_t *bundle_data = NULL;
 116 
 117     get_bundle_variant_data(bundle_data, pe__const_top_resource(rsc, true));
 118     return bundle_data->child;
 119 }
 120 
 121 /*!
 122  * \internal
 123  * \brief Get containerized resource corresponding to a given bundle container
 124  *
 125  * \param[in] instance  Collective instance that might be a bundle container
 126  *
 127  * \return Bundled resource instance inside \p instance if it is a bundle
 128  *         container instance, otherwise NULL
 129  */
 130 const pcmk_resource_t *
 131 pe__get_rsc_in_container(const pcmk_resource_t *instance)
     /*  */
 132 {
 133     const pe__bundle_variant_data_t *data = NULL;
 134     const pcmk_resource_t *top = pe__const_top_resource(instance, true);
 135 
 136     if ((top == NULL) || (top->variant != pcmk_rsc_variant_bundle)) {
 137         return NULL;
 138     }
 139     get_bundle_variant_data(data, top);
 140 
 141     for (const GList *iter = data->replicas; iter != NULL; iter = iter->next) {
 142         const pe__bundle_replica_t *replica = iter->data;
 143 
 144         if (instance == replica->container) {
 145             return replica->child;
 146         }
 147     }
 148     return NULL;
 149 }
 150 
 151 /*!
 152  * \internal
 153  * \brief Check whether a given node is created by a bundle
 154  *
 155  * \param[in] bundle  Bundle resource to check
 156  * \param[in] node    Node to check
 157  *
 158  * \return true if \p node is an instance of \p bundle, otherwise false
 159  */
 160 bool
 161 pe__node_is_bundle_instance(const pcmk_resource_t *bundle,
     /*  */
 162                             const pcmk_node_t *node)
 163 {
 164     pe__bundle_variant_data_t *bundle_data = NULL;
 165 
 166     get_bundle_variant_data(bundle_data, bundle);
 167     for (GList *iter = bundle_data->replicas; iter != NULL; iter = iter->next) {
 168         pe__bundle_replica_t *replica = iter->data;
 169 
 170         if (pe__same_node(node, replica->node)) {
 171             return true;
 172         }
 173     }
 174     return false;
 175 }
 176 
 177 /*!
 178  * \internal
 179  * \brief Get the container of a bundle's first replica
 180  *
 181  * \param[in] bundle  Bundle resource to get container for
 182  *
 183  * \return Container resource from first replica of \p bundle if any,
 184  *         otherwise NULL
 185  */
 186 pcmk_resource_t *
 187 pe__first_container(const pcmk_resource_t *bundle)
     /*  */
 188 {
 189     const pe__bundle_variant_data_t *bundle_data = NULL;
 190     const pe__bundle_replica_t *replica = NULL;
 191 
 192     get_bundle_variant_data(bundle_data, bundle);
 193     if (bundle_data->replicas == NULL) {
 194         return NULL;
 195     }
 196     replica = bundle_data->replicas->data;
 197     return replica->container;
 198 }
 199 
 200 /*!
 201  * \internal
 202  * \brief Iterate over bundle replicas
 203  *
 204  * \param[in,out] bundle     Bundle to iterate over
 205  * \param[in]     fn         Function to call for each replica (its return value
 206  *                           indicates whether to continue iterating)
 207  * \param[in,out] user_data  Pointer to pass to \p fn
 208  */
 209 void
 210 pe__foreach_bundle_replica(pcmk_resource_t *bundle,
     /*  */
 211                            bool (*fn)(pe__bundle_replica_t *, void *),
 212                            void *user_data)
 213 {
 214     const pe__bundle_variant_data_t *bundle_data = NULL;
 215 
 216     get_bundle_variant_data(bundle_data, bundle);
 217     for (GList *iter = bundle_data->replicas; iter != NULL; iter = iter->next) {
 218         if (!fn((pe__bundle_replica_t *) iter->data, user_data)) {
 219             break;
 220         }
 221     }
 222 }
 223 
 224 /*!
 225  * \internal
 226  * \brief Iterate over const bundle replicas
 227  *
 228  * \param[in]     bundle     Bundle to iterate over
 229  * \param[in]     fn         Function to call for each replica (its return value
 230  *                           indicates whether to continue iterating)
 231  * \param[in,out] user_data  Pointer to pass to \p fn
 232  */
 233 void
 234 pe__foreach_const_bundle_replica(const pcmk_resource_t *bundle,
     /*  */
 235                                  bool (*fn)(const pe__bundle_replica_t *,
 236                                             void *),
 237                                  void *user_data)
 238 {
 239     const pe__bundle_variant_data_t *bundle_data = NULL;
 240 
 241     get_bundle_variant_data(bundle_data, bundle);
 242     for (const GList *iter = bundle_data->replicas; iter != NULL;
 243          iter = iter->next) {
 244 
 245         if (!fn((const pe__bundle_replica_t *) iter->data, user_data)) {
 246             break;
 247         }
 248     }
 249 }
 250 
 251 static char *
 252 next_ip(const char *last_ip)
     /*  */
 253 {
 254     unsigned int oct1 = 0;
 255     unsigned int oct2 = 0;
 256     unsigned int oct3 = 0;
 257     unsigned int oct4 = 0;
 258     int rc = sscanf(last_ip, "%u.%u.%u.%u", &oct1, &oct2, &oct3, &oct4);
 259 
 260     if (rc != 4) {
 261         /*@ TODO check for IPv6 */
 262         return NULL;
 263 
 264     } else if (oct3 > 253) {
 265         return NULL;
 266 
 267     } else if (oct4 > 253) {
 268         ++oct3;
 269         oct4 = 1;
 270 
 271     } else {
 272         ++oct4;
 273     }
 274 
 275     return crm_strdup_printf("%u.%u.%u.%u", oct1, oct2, oct3, oct4);
 276 }
 277 
 278 static void
 279 allocate_ip(pe__bundle_variant_data_t *data, pe__bundle_replica_t *replica,
     /*  */
 280             GString *buffer)
 281 {
 282     if(data->ip_range_start == NULL) {
 283         return;
 284 
 285     } else if(data->ip_last) {
 286         replica->ipaddr = next_ip(data->ip_last);
 287 
 288     } else {
 289         replica->ipaddr = strdup(data->ip_range_start);
 290     }
 291 
 292     data->ip_last = replica->ipaddr;
 293     switch (data->agent_type) {
 294         case PE__CONTAINER_AGENT_DOCKER:
 295         case PE__CONTAINER_AGENT_PODMAN:
 296             if (data->add_host) {
 297                 g_string_append_printf(buffer, " --add-host=%s-%d:%s",
 298                                        data->prefix, replica->offset,
 299                                        replica->ipaddr);
 300             } else {
 301                 g_string_append_printf(buffer, " --hosts-entry=%s=%s-%d",
 302                                        replica->ipaddr, data->prefix,
 303                                        replica->offset);
 304             }
 305             break;
 306 
 307         case PE__CONTAINER_AGENT_RKT:
 308             g_string_append_printf(buffer, " --hosts-entry=%s=%s-%d",
 309                                    replica->ipaddr, data->prefix,
 310                                    replica->offset);
 311             break;
 312 
 313         default: // PE__CONTAINER_AGENT_UNKNOWN
 314             break;
 315     }
 316 }
 317 
 318 static xmlNode *
 319 create_resource(const char *name, const char *provider, const char *kind)
     /*  */
 320 {
 321     xmlNode *rsc = create_xml_node(NULL, XML_CIB_TAG_RESOURCE);
 322 
 323     crm_xml_add(rsc, XML_ATTR_ID, name);
 324     crm_xml_add(rsc, XML_AGENT_ATTR_CLASS, PCMK_RESOURCE_CLASS_OCF);
 325     crm_xml_add(rsc, XML_AGENT_ATTR_PROVIDER, provider);
 326     crm_xml_add(rsc, XML_ATTR_TYPE, kind);
 327 
 328     return rsc;
 329 }
 330 
 331 /*!
 332  * \internal
 333  * \brief Check whether cluster can manage resource inside container
 334  *
 335  * \param[in,out] data  Container variant data
 336  *
 337  * \return TRUE if networking configuration is acceptable, FALSE otherwise
 338  *
 339  * \note The resource is manageable if an IP range or control port has been
 340  *       specified. If a control port is used without an IP range, replicas per
 341  *       host must be 1.
 342  */
 343 static bool
 344 valid_network(pe__bundle_variant_data_t *data)
     /*  */
 345 {
 346     if(data->ip_range_start) {
 347         return TRUE;
 348     }
 349     if(data->control_port) {
 350         if(data->nreplicas_per_host > 1) {
 351             pe_err("Specifying the 'control-port' for %s requires 'replicas-per-host=1'", data->prefix);
 352             data->nreplicas_per_host = 1;
 353             // @TODO to be sure:
 354             // pe__clear_resource_flags(rsc, pcmk_rsc_unique);
 355         }
 356         return TRUE;
 357     }
 358     return FALSE;
 359 }
 360 
 361 static int
 362 create_ip_resource(pcmk_resource_t *parent, pe__bundle_variant_data_t *data,
     /*  */
 363                    pe__bundle_replica_t *replica)
 364 {
 365     if(data->ip_range_start) {
 366         char *id = NULL;
 367         xmlNode *xml_ip = NULL;
 368         xmlNode *xml_obj = NULL;
 369 
 370         id = crm_strdup_printf("%s-ip-%s", data->prefix, replica->ipaddr);
 371         crm_xml_sanitize_id(id);
 372         xml_ip = create_resource(id, "heartbeat", "IPaddr2");
 373         free(id);
 374 
 375         xml_obj = create_xml_node(xml_ip, XML_TAG_ATTR_SETS);
 376         crm_xml_set_id(xml_obj, "%s-attributes-%d",
 377                        data->prefix, replica->offset);
 378 
 379         crm_create_nvpair_xml(xml_obj, NULL, "ip", replica->ipaddr);
 380         if(data->host_network) {
 381             crm_create_nvpair_xml(xml_obj, NULL, "nic", data->host_network);
 382         }
 383 
 384         if(data->host_netmask) {
 385             crm_create_nvpair_xml(xml_obj, NULL,
 386                                   "cidr_netmask", data->host_netmask);
 387 
 388         } else {
 389             crm_create_nvpair_xml(xml_obj, NULL, "cidr_netmask", "32");
 390         }
 391 
 392         xml_obj = create_xml_node(xml_ip, "operations");
 393         crm_create_op_xml(xml_obj, ID(xml_ip), PCMK_ACTION_MONITOR, "60s",
 394                           NULL);
 395 
 396         // TODO: Other ops? Timeouts and intervals from underlying resource?
 397 
 398         if (pe__unpack_resource(xml_ip, &replica->ip, parent,
 399                                 parent->cluster) != pcmk_rc_ok) {
 400             return pcmk_rc_unpack_error;
 401         }
 402 
 403         parent->children = g_list_append(parent->children, replica->ip);
 404     }
 405     return pcmk_rc_ok;
 406 }
 407 
 408 static const char*
 409 container_agent_str(enum pe__container_agent t)
     /*  */
 410 {
 411     switch (t) {
 412         case PE__CONTAINER_AGENT_DOCKER: return PE__CONTAINER_AGENT_DOCKER_S;
 413         case PE__CONTAINER_AGENT_RKT:    return PE__CONTAINER_AGENT_RKT_S;
 414         case PE__CONTAINER_AGENT_PODMAN: return PE__CONTAINER_AGENT_PODMAN_S;
 415         default: // PE__CONTAINER_AGENT_UNKNOWN
 416             break;
 417     }
 418     return PE__CONTAINER_AGENT_UNKNOWN_S;
 419 }
 420 
 421 static int
 422 create_container_resource(pcmk_resource_t *parent,
     /*  */
 423                           const pe__bundle_variant_data_t *data,
 424                           pe__bundle_replica_t *replica)
 425 {
 426     char *id = NULL;
 427     xmlNode *xml_container = NULL;
 428     xmlNode *xml_obj = NULL;
 429 
 430     // Agent-specific
 431     const char *hostname_opt = NULL;
 432     const char *env_opt = NULL;
 433     const char *agent_str = NULL;
 434     int volid = 0;  // rkt-only
 435 
 436     GString *buffer = NULL;
 437     GString *dbuffer = NULL;
 438 
 439     // Where syntax differences are drop-in replacements, set them now
 440     switch (data->agent_type) {
 441         case PE__CONTAINER_AGENT_DOCKER:
 442         case PE__CONTAINER_AGENT_PODMAN:
 443             hostname_opt = "-h ";
 444             env_opt = "-e ";
 445             break;
 446         case PE__CONTAINER_AGENT_RKT:
 447             hostname_opt = "--hostname=";
 448             env_opt = "--environment=";
 449             break;
 450         default:    // PE__CONTAINER_AGENT_UNKNOWN
 451             return pcmk_rc_unpack_error;
 452     }
 453     agent_str = container_agent_str(data->agent_type);
 454 
 455     buffer = g_string_sized_new(4096);
 456 
 457     id = crm_strdup_printf("%s-%s-%d", data->prefix, agent_str,
 458                            replica->offset);
 459     crm_xml_sanitize_id(id);
 460     xml_container = create_resource(id, "heartbeat", agent_str);
 461     free(id);
 462 
 463     xml_obj = create_xml_node(xml_container, XML_TAG_ATTR_SETS);
 464     crm_xml_set_id(xml_obj, "%s-attributes-%d", data->prefix, replica->offset);
 465 
 466     crm_create_nvpair_xml(xml_obj, NULL, "image", data->image);
 467     crm_create_nvpair_xml(xml_obj, NULL, "allow_pull", XML_BOOLEAN_TRUE);
 468     crm_create_nvpair_xml(xml_obj, NULL, "force_kill", XML_BOOLEAN_FALSE);
 469     crm_create_nvpair_xml(xml_obj, NULL, "reuse", XML_BOOLEAN_FALSE);
 470 
 471     if (data->agent_type == PE__CONTAINER_AGENT_DOCKER) {
 472         g_string_append(buffer, " --restart=no");
 473     }
 474 
 475     /* Set a container hostname only if we have an IP to map it to. The user can
 476      * set -h or --uts=host themselves if they want a nicer name for logs, but
 477      * this makes applications happy who need their  hostname to match the IP
 478      * they bind to.
 479      */
 480     if (data->ip_range_start != NULL) {
 481         g_string_append_printf(buffer, " %s%s-%d", hostname_opt, data->prefix,
 482                                replica->offset);
 483     }
 484     pcmk__g_strcat(buffer, " ", env_opt, "PCMK_stderr=1", NULL);
 485 
 486     if (data->container_network != NULL) {
 487         pcmk__g_strcat(buffer, " --net=", data->container_network, NULL);
 488     }
 489 
 490     if (data->control_port != NULL) {
 491         pcmk__g_strcat(buffer, " ", env_opt, "PCMK_" PCMK__ENV_REMOTE_PORT "=",
 492                        data->control_port, NULL);
 493     } else {
 494         g_string_append_printf(buffer, " %sPCMK_" PCMK__ENV_REMOTE_PORT "=%d",
 495                                env_opt, DEFAULT_REMOTE_PORT);
 496     }
 497 
 498     for (GList *iter = data->mounts; iter != NULL; iter = iter->next) {
 499         pe__bundle_mount_t *mount = (pe__bundle_mount_t *) iter->data;
 500         char *source = NULL;
 501 
 502         if (pcmk_is_set(mount->flags, pe__bundle_mount_subdir)) {
 503             source = crm_strdup_printf("%s/%s-%d", mount->source, data->prefix,
 504                                        replica->offset);
 505             pcmk__add_separated_word(&dbuffer, 1024, source, ",");
 506         }
 507 
 508         switch (data->agent_type) {
 509             case PE__CONTAINER_AGENT_DOCKER:
 510             case PE__CONTAINER_AGENT_PODMAN:
 511                 pcmk__g_strcat(buffer,
 512                                " -v ", pcmk__s(source, mount->source),
 513                                ":", mount->target, NULL);
 514 
 515                 if (mount->options != NULL) {
 516                     pcmk__g_strcat(buffer, ":", mount->options, NULL);
 517                 }
 518                 break;
 519             case PE__CONTAINER_AGENT_RKT:
 520                 g_string_append_printf(buffer,
 521                                        " --volume vol%d,kind=host,"
 522                                        "source=%s%s%s "
 523                                        "--mount volume=vol%d,target=%s",
 524                                        volid, pcmk__s(source, mount->source),
 525                                        (mount->options != NULL)? "," : "",
 526                                        pcmk__s(mount->options, ""),
 527                                        volid, mount->target);
 528                 volid++;
 529                 break;
 530             default:
 531                 break;
 532         }
 533         free(source);
 534     }
 535 
 536     for (GList *iter = data->ports; iter != NULL; iter = iter->next) {
 537         pe__bundle_port_t *port = (pe__bundle_port_t *) iter->data;
 538 
 539         switch (data->agent_type) {
 540             case PE__CONTAINER_AGENT_DOCKER:
 541             case PE__CONTAINER_AGENT_PODMAN:
 542                 if (replica->ipaddr != NULL) {
 543                     pcmk__g_strcat(buffer,
 544                                    " -p ", replica->ipaddr, ":", port->source,
 545                                    ":", port->target, NULL);
 546 
 547                 } else if (!pcmk__str_eq(data->container_network, "host",
 548                                          pcmk__str_none)) {
 549                     // No need to do port mapping if net == host
 550                     pcmk__g_strcat(buffer,
 551                                    " -p ", port->source, ":", port->target,
 552                                    NULL);
 553                 }
 554                 break;
 555             case PE__CONTAINER_AGENT_RKT:
 556                 if (replica->ipaddr != NULL) {
 557                     pcmk__g_strcat(buffer,
 558                                    " --port=", port->target,
 559                                    ":", replica->ipaddr, ":", port->source,
 560                                    NULL);
 561                 } else {
 562                     pcmk__g_strcat(buffer,
 563                                    " --port=", port->target, ":", port->source,
 564                                    NULL);
 565                 }
 566                 break;
 567             default:
 568                 break;
 569         }
 570     }
 571 
 572     /* @COMPAT: We should use pcmk__add_word() here, but we can't yet, because
 573      * it would cause restarts during rolling upgrades.
 574      *
 575      * In a previous version of the container resource creation logic, if
 576      * data->launcher_options is not NULL, we append
 577      * (" %s", data->launcher_options) even if data->launcher_options is an
 578      * empty string. Likewise for data->container_host_options. Using
 579      *
 580      *     pcmk__add_word(buffer, 0, data->launcher_options)
 581      *
 582      * removes that extra trailing space, causing a resource definition change.
 583      */
 584     if (data->launcher_options != NULL) {
 585         pcmk__g_strcat(buffer, " ", data->launcher_options, NULL);
 586     }
 587 
 588     if (data->container_host_options != NULL) {
 589         pcmk__g_strcat(buffer, " ", data->container_host_options, NULL);
 590     }
 591 
 592     crm_create_nvpair_xml(xml_obj, NULL, "run_opts",
 593                           (const char *) buffer->str);
 594     g_string_free(buffer, TRUE);
 595 
 596     crm_create_nvpair_xml(xml_obj, NULL, "mount_points",
 597                           (dbuffer != NULL)? (const char *) dbuffer->str : "");
 598     if (dbuffer != NULL) {
 599         g_string_free(dbuffer, TRUE);
 600     }
 601 
 602     if (replica->child != NULL) {
 603         if (data->container_command != NULL) {
 604             crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
 605                                   data->container_command);
 606         } else {
 607             crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
 608                                   SBIN_DIR "/pacemaker-remoted");
 609         }
 610 
 611         /* TODO: Allow users to specify their own?
 612          *
 613          * We just want to know if the container is alive; we'll monitor the
 614          * child independently.
 615          */
 616         crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
 617 #if 0
 618         /* @TODO Consider supporting the use case where we can start and stop
 619          * resources, but not proxy local commands (such as setting node
 620          * attributes), by running the local executor in stand-alone mode.
 621          * However, this would probably be better done via ACLs as with other
 622          * Pacemaker Remote nodes.
 623          */
 624     } else if ((child != NULL) && data->untrusted) {
 625         crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
 626                               CRM_DAEMON_DIR "/pacemaker-execd");
 627         crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd",
 628                               CRM_DAEMON_DIR "/pacemaker/cts-exec-helper -c poke");
 629 #endif
 630     } else {
 631         if (data->container_command != NULL) {
 632             crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
 633                                   data->container_command);
 634         }
 635 
 636         /* TODO: Allow users to specify their own?
 637          *
 638          * We don't know what's in the container, so we just want to know if it
 639          * is alive.
 640          */
 641         crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
 642     }
 643 
 644     xml_obj = create_xml_node(xml_container, "operations");
 645     crm_create_op_xml(xml_obj, ID(xml_container), PCMK_ACTION_MONITOR, "60s",
 646                       NULL);
 647 
 648     // TODO: Other ops? Timeouts and intervals from underlying resource?
 649     if (pe__unpack_resource(xml_container, &replica->container, parent,
 650                             parent->cluster) != pcmk_rc_ok) {
 651         return pcmk_rc_unpack_error;
 652     }
 653     pe__set_resource_flags(replica->container, pcmk_rsc_replica_container);
 654     parent->children = g_list_append(parent->children, replica->container);
 655 
 656     return pcmk_rc_ok;
 657 }
 658 
 659 /*!
 660  * \brief Ban a node from a resource's (and its children's) allowed nodes list
 661  *
 662  * \param[in,out] rsc    Resource to modify
 663  * \param[in]     uname  Name of node to ban
 664  */
 665 static void
 666 disallow_node(pcmk_resource_t *rsc, const char *uname)
     /*  */
 667 {
 668     gpointer match = g_hash_table_lookup(rsc->allowed_nodes, uname);
 669 
 670     if (match) {
 671         ((pcmk_node_t *) match)->weight = -INFINITY;
 672         ((pcmk_node_t *) match)->rsc_discover_mode = pcmk_probe_never;
 673     }
 674     if (rsc->children) {
 675         g_list_foreach(rsc->children, (GFunc) disallow_node, (gpointer) uname);
 676     }
 677 }
 678 
 679 static int
 680 create_remote_resource(pcmk_resource_t *parent, pe__bundle_variant_data_t *data,
     /*  */
 681                        pe__bundle_replica_t *replica)
 682 {
 683     if (replica->child && valid_network(data)) {
 684         GHashTableIter gIter;
 685         pcmk_node_t *node = NULL;
 686         xmlNode *xml_remote = NULL;
 687         char *id = crm_strdup_printf("%s-%d", data->prefix, replica->offset);
 688         char *port_s = NULL;
 689         const char *uname = NULL;
 690         const char *connect_name = NULL;
 691 
 692         if (pe_find_resource(parent->cluster->resources, id) != NULL) {
 693             free(id);
 694             // The biggest hammer we have
 695             id = crm_strdup_printf("pcmk-internal-%s-remote-%d",
 696                                    replica->child->id, replica->offset);
 697             //@TODO return error instead of asserting?
 698             CRM_ASSERT(pe_find_resource(parent->cluster->resources,
 699                                         id) == NULL);
 700         }
 701 
 702         /* REMOTE_CONTAINER_HACK: Using "#uname" as the server name when the
 703          * connection does not have its own IP is a magic string that we use to
 704          * support nested remotes (i.e. a bundle running on a remote node).
 705          */
 706         connect_name = (replica->ipaddr? replica->ipaddr : "#uname");
 707 
 708         if (data->control_port == NULL) {
 709             port_s = pcmk__itoa(DEFAULT_REMOTE_PORT);
 710         }
 711 
 712         /* This sets replica->container as replica->remote's container, which is
 713          * similar to what happens with guest nodes. This is how the scheduler
 714          * knows that the bundle node is fenced by recovering the container, and
 715          * that remote should be ordered relative to the container.
 716          */
 717         xml_remote = pe_create_remote_xml(NULL, id, replica->container->id,
 718                                           NULL, NULL, NULL,
 719                                           connect_name, (data->control_port?
 720                                           data->control_port : port_s));
 721         free(port_s);
 722 
 723         /* Abandon our created ID, and pull the copy from the XML, because we
 724          * need something that will get freed during scheduler data cleanup to
 725          * use as the node ID and uname.
 726          */
 727         free(id);
 728         id = NULL;
 729         uname = ID(xml_remote);
 730 
 731         /* Ensure a node has been created for the guest (it may have already
 732          * been, if it has a permanent node attribute), and ensure its weight is
 733          * -INFINITY so no other resources can run on it.
 734          */
 735         node = pe_find_node(parent->cluster->nodes, uname);
 736         if (node == NULL) {
 737             node = pe_create_node(uname, uname, "remote", "-INFINITY",
 738                                   parent->cluster);
 739         } else {
 740             node->weight = -INFINITY;
 741         }
 742         node->rsc_discover_mode = pcmk_probe_never;
 743 
 744         /* unpack_remote_nodes() ensures that each remote node and guest node
 745          * has a pcmk_node_t entry. Ideally, it would do the same for bundle
 746          * nodes. Unfortunately, a bundle has to be mostly unpacked before it's
 747          * obvious what nodes will be needed, so we do it just above.
 748          *
 749          * Worse, that means that the node may have been utilized while
 750          * unpacking other resources, without our weight correction. The most
 751          * likely place for this to happen is when pe__unpack_resource() calls
 752          * resource_location() to set a default score in symmetric clusters.
 753          * This adds a node *copy* to each resource's allowed nodes, and these
 754          * copies will have the wrong weight.
 755          *
 756          * As a hacky workaround, fix those copies here.
 757          *
 758          * @TODO Possible alternative: ensure bundles are unpacked before other
 759          * resources, so the weight is correct before any copies are made.
 760          */
 761         g_list_foreach(parent->cluster->resources, (GFunc) disallow_node,
 762                        (gpointer) uname);
 763 
 764         replica->node = pe__copy_node(node);
 765         replica->node->weight = 500;
 766         replica->node->rsc_discover_mode = pcmk_probe_exclusive;
 767 
 768         /* Ensure the node shows up as allowed and with the correct discovery set */
 769         if (replica->child->allowed_nodes != NULL) {
 770             g_hash_table_destroy(replica->child->allowed_nodes);
 771         }
 772         replica->child->allowed_nodes = pcmk__strkey_table(NULL, free);
 773         g_hash_table_insert(replica->child->allowed_nodes,
 774                             (gpointer) replica->node->details->id,
 775                             pe__copy_node(replica->node));
 776 
 777         {
 778             pcmk_node_t *copy = pe__copy_node(replica->node);
 779             copy->weight = -INFINITY;
 780             g_hash_table_insert(replica->child->parent->allowed_nodes,
 781                                 (gpointer) replica->node->details->id, copy);
 782         }
 783         if (pe__unpack_resource(xml_remote, &replica->remote, parent,
 784                                 parent->cluster) != pcmk_rc_ok) {
 785             return pcmk_rc_unpack_error;
 786         }
 787 
 788         g_hash_table_iter_init(&gIter, replica->remote->allowed_nodes);
 789         while (g_hash_table_iter_next(&gIter, NULL, (void **)&node)) {
 790             if (pe__is_guest_or_remote_node(node)) {
 791                 /* Remote resources can only run on 'normal' cluster node */
 792                 node->weight = -INFINITY;
 793             }
 794         }
 795 
 796         replica->node->details->remote_rsc = replica->remote;
 797 
 798         // Ensure pe__is_guest_node() functions correctly immediately
 799         replica->remote->container = replica->container;
 800 
 801         /* A bundle's #kind is closer to "container" (guest node) than the
 802          * "remote" set by pe_create_node().
 803          */
 804         g_hash_table_insert(replica->node->details->attrs,
 805                             strdup(CRM_ATTR_KIND), strdup("container"));
 806 
 807         /* One effect of this is that setup_container() will add
 808          * replica->remote to replica->container's fillers, which will make
 809          * pe__resource_contains_guest_node() true for replica->container.
 810          *
 811          * replica->child does NOT get added to replica->container's fillers.
 812          * The only noticeable effect if it did would be for its fail count to
 813          * be taken into account when checking replica->container's migration
 814          * threshold.
 815          */
 816         parent->children = g_list_append(parent->children, replica->remote);
 817     }
 818     return pcmk_rc_ok;
 819 }
 820 
 821 static int
 822 create_replica_resources(pcmk_resource_t *parent, pe__bundle_variant_data_t *data,
     /*  */
 823                          pe__bundle_replica_t *replica)
 824 {
 825     int rc = pcmk_rc_ok;
 826 
 827     rc = create_container_resource(parent, data, replica);
 828     if (rc != pcmk_rc_ok) {
 829         return rc;
 830     }
 831 
 832     rc = create_ip_resource(parent, data, replica);
 833     if (rc != pcmk_rc_ok) {
 834         return rc;
 835     }
 836 
 837     rc = create_remote_resource(parent, data, replica);
 838     if (rc != pcmk_rc_ok) {
 839         return rc;
 840     }
 841 
 842     if ((replica->child != NULL) && (replica->ipaddr != NULL)) {
 843         add_hash_param(replica->child->meta, "external-ip", replica->ipaddr);
 844     }
 845 
 846     if (replica->remote != NULL) {
 847         /*
 848          * Allow the remote connection resource to be allocated to a
 849          * different node than the one on which the container is active.
 850          *
 851          * This makes it possible to have Pacemaker Remote nodes running
 852          * containers with pacemaker-remoted inside in order to start
 853          * services inside those containers.
 854          */
 855         pe__set_resource_flags(replica->remote,
 856                                pcmk_rsc_remote_nesting_allowed);
 857     }
 858     return rc;
 859 }
 860 
 861 static void
 862 mount_add(pe__bundle_variant_data_t *bundle_data, const char *source,
     /*  */
 863           const char *target, const char *options, uint32_t flags)
 864 {
 865     pe__bundle_mount_t *mount = calloc(1, sizeof(pe__bundle_mount_t));
 866 
 867     CRM_ASSERT(mount != NULL);
 868     mount->source = strdup(source);
 869     mount->target = strdup(target);
 870     pcmk__str_update(&mount->options, options);
 871     mount->flags = flags;
 872     bundle_data->mounts = g_list_append(bundle_data->mounts, mount);
 873 }
 874 
 875 static void
 876 mount_free(pe__bundle_mount_t *mount)
     /*  */
 877 {
 878     free(mount->source);
 879     free(mount->target);
 880     free(mount->options);
 881     free(mount);
 882 }
 883 
 884 static void
 885 port_free(pe__bundle_port_t *port)
     /*  */
 886 {
 887     free(port->source);
 888     free(port->target);
 889     free(port);
 890 }
 891 
 892 static pe__bundle_replica_t *
 893 replica_for_remote(pcmk_resource_t *remote)
     /*  */
 894 {
 895     pcmk_resource_t *top = remote;
 896     pe__bundle_variant_data_t *bundle_data = NULL;
 897 
 898     if (top == NULL) {
 899         return NULL;
 900     }
 901 
 902     while (top->parent != NULL) {
 903         top = top->parent;
 904     }
 905 
 906     get_bundle_variant_data(bundle_data, top);
 907     for (GList *gIter = bundle_data->replicas; gIter != NULL;
 908          gIter = gIter->next) {
 909         pe__bundle_replica_t *replica = gIter->data;
 910 
 911         if (replica->remote == remote) {
 912             return replica;
 913         }
 914     }
 915     CRM_LOG_ASSERT(FALSE);
 916     return NULL;
 917 }
 918 
 919 bool
 920 pe__bundle_needs_remote_name(pcmk_resource_t *rsc)
     /*  */
 921 {
 922     const char *value;
 923     GHashTable *params = NULL;
 924 
 925     if (rsc == NULL) {
 926         return false;
 927     }
 928 
 929     // Use NULL node since pcmk__bundle_expand() uses that to set value
 930     params = pe_rsc_params(rsc, NULL, rsc->cluster);
 931     value = g_hash_table_lookup(params, XML_RSC_ATTR_REMOTE_RA_ADDR);
 932 
 933     return pcmk__str_eq(value, "#uname", pcmk__str_casei)
 934            && xml_contains_remote_node(rsc->xml);
 935 }
 936 
 937 const char *
 938 pe__add_bundle_remote_name(pcmk_resource_t *rsc, pcmk_scheduler_t *scheduler,
     /*  */
 939                            xmlNode *xml, const char *field)
 940 {
 941     // REMOTE_CONTAINER_HACK: Allow remote nodes that start containers with pacemaker remote inside
 942 
 943     pcmk_node_t *node = NULL;
 944     pe__bundle_replica_t *replica = NULL;
 945 
 946     if (!pe__bundle_needs_remote_name(rsc)) {
 947         return NULL;
 948     }
 949 
 950     replica = replica_for_remote(rsc);
 951     if (replica == NULL) {
 952         return NULL;
 953     }
 954 
 955     node = replica->container->allocated_to;
 956     if (node == NULL) {
 957         /* If it won't be running anywhere after the
 958          * transition, go with where it's running now.
 959          */
 960         node = pe__current_node(replica->container);
 961     }
 962 
 963     if(node == NULL) {
 964         crm_trace("Cannot determine address for bundle connection %s", rsc->id);
 965         return NULL;
 966     }
 967 
 968     crm_trace("Setting address for bundle connection %s to bundle host %s",
 969               rsc->id, pe__node_name(node));
 970     if(xml != NULL && field != NULL) {
 971         crm_xml_add(xml, field, node->details->uname);
 972     }
 973 
 974     return node->details->uname;
 975 }
 976 
 977 #define pe__set_bundle_mount_flags(mount_xml, flags, flags_to_set) do {     \
 978         flags = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE,           \
 979                                    "Bundle mount", ID(mount_xml), flags,    \
 980                                    (flags_to_set), #flags_to_set);          \
 981     } while (0)
 982 
 983 gboolean
 984 pe__unpack_bundle(pcmk_resource_t *rsc, pcmk_scheduler_t *scheduler)
     /*  */
 985 {
 986     const char *value = NULL;
 987     xmlNode *xml_obj = NULL;
 988     xmlNode *xml_resource = NULL;
 989     pe__bundle_variant_data_t *bundle_data = NULL;
 990     bool need_log_mount = TRUE;
 991 
 992     CRM_ASSERT(rsc != NULL);
 993     pe_rsc_trace(rsc, "Processing resource %s...", rsc->id);
 994 
 995     bundle_data = calloc(1, sizeof(pe__bundle_variant_data_t));
 996     rsc->variant_opaque = bundle_data;
 997     bundle_data->prefix = strdup(rsc->id);
 998 
 999     xml_obj = first_named_child(rsc->xml, PE__CONTAINER_AGENT_DOCKER_S);
1000     if (xml_obj != NULL) {
1001         bundle_data->agent_type = PE__CONTAINER_AGENT_DOCKER;
1002     } else {
1003         xml_obj = first_named_child(rsc->xml, PE__CONTAINER_AGENT_RKT_S);
1004         if (xml_obj != NULL) {
1005             bundle_data->agent_type = PE__CONTAINER_AGENT_RKT;
1006         } else {
1007             xml_obj = first_named_child(rsc->xml, PE__CONTAINER_AGENT_PODMAN_S);
1008             if (xml_obj != NULL) {
1009                 bundle_data->agent_type = PE__CONTAINER_AGENT_PODMAN;
1010             } else {
1011                 return FALSE;
1012             }
1013         }
1014     }
1015 
1016     // Use 0 for default, minimum, and invalid promoted-max
1017     value = crm_element_value(xml_obj, PCMK_META_PROMOTED_MAX);
1018     if (value == NULL) {
1019         // @COMPAT deprecated since 2.0.0
1020         value = crm_element_value(xml_obj, "masters");
1021     }
1022     pcmk__scan_min_int(value, &bundle_data->promoted_max, 0);
1023 
1024     // Default replicas to promoted-max if it was specified and 1 otherwise
1025     value = crm_element_value(xml_obj, "replicas");
1026     if ((value == NULL) && (bundle_data->promoted_max > 0)) {
1027         bundle_data->nreplicas = bundle_data->promoted_max;
1028     } else {
1029         pcmk__scan_min_int(value, &bundle_data->nreplicas, 1);
1030     }
1031 
1032     /*
1033      * Communication between containers on the same host via the
1034      * floating IPs only works if the container is started with:
1035      *   --userland-proxy=false --ip-masq=false
1036      */
1037     value = crm_element_value(xml_obj, "replicas-per-host");
1038     pcmk__scan_min_int(value, &bundle_data->nreplicas_per_host, 1);
1039     if (bundle_data->nreplicas_per_host == 1) {
1040         pe__clear_resource_flags(rsc, pcmk_rsc_unique);
1041     }
1042 
1043     bundle_data->container_command = crm_element_value_copy(xml_obj, "run-command");
1044     bundle_data->launcher_options = crm_element_value_copy(xml_obj, "options");
1045     bundle_data->image = crm_element_value_copy(xml_obj, "image");
1046     bundle_data->container_network = crm_element_value_copy(xml_obj, "network");
1047 
1048     xml_obj = first_named_child(rsc->xml, "network");
1049     if(xml_obj) {
1050 
1051         bundle_data->ip_range_start = crm_element_value_copy(xml_obj, "ip-range-start");
1052         bundle_data->host_netmask = crm_element_value_copy(xml_obj, "host-netmask");
1053         bundle_data->host_network = crm_element_value_copy(xml_obj, "host-interface");
1054         bundle_data->control_port = crm_element_value_copy(xml_obj, "control-port");
1055         value = crm_element_value(xml_obj, "add-host");
1056         if (crm_str_to_boolean(value, &bundle_data->add_host) != 1) {
1057             bundle_data->add_host = TRUE;
1058         }
1059 
1060         for (xmlNode *xml_child = pcmk__xe_first_child(xml_obj); xml_child != NULL;
1061              xml_child = pcmk__xe_next(xml_child)) {
1062 
1063             pe__bundle_port_t *port = calloc(1, sizeof(pe__bundle_port_t));
1064             port->source = crm_element_value_copy(xml_child, "port");
1065 
1066             if(port->source == NULL) {
1067                 port->source = crm_element_value_copy(xml_child, "range");
1068             } else {
1069                 port->target = crm_element_value_copy(xml_child, "internal-port");
1070             }
1071 
1072             if(port->source != NULL && strlen(port->source) > 0) {
1073                 if(port->target == NULL) {
1074                     port->target = strdup(port->source);
1075                 }
1076                 bundle_data->ports = g_list_append(bundle_data->ports, port);
1077 
1078             } else {
1079                 pe_err("Invalid port directive %s", ID(xml_child));
1080                 port_free(port);
1081             }
1082         }
1083     }
1084 
1085     xml_obj = first_named_child(rsc->xml, "storage");
1086     for (xmlNode *xml_child = pcmk__xe_first_child(xml_obj); xml_child != NULL;
1087          xml_child = pcmk__xe_next(xml_child)) {
1088 
1089         const char *source = crm_element_value(xml_child, "source-dir");
1090         const char *target = crm_element_value(xml_child, "target-dir");
1091         const char *options = crm_element_value(xml_child, "options");
1092         int flags = pe__bundle_mount_none;
1093 
1094         if (source == NULL) {
1095             source = crm_element_value(xml_child, "source-dir-root");
1096             pe__set_bundle_mount_flags(xml_child, flags,
1097                                        pe__bundle_mount_subdir);
1098         }
1099 
1100         if (source && target) {
1101             mount_add(bundle_data, source, target, options, flags);
1102             if (strcmp(target, "/var/log") == 0) {
1103                 need_log_mount = FALSE;
1104             }
1105         } else {
1106             pe_err("Invalid mount directive %s", ID(xml_child));
1107         }
1108     }
1109 
1110     xml_obj = first_named_child(rsc->xml, "primitive");
1111     if (xml_obj && valid_network(bundle_data)) {
1112         char *value = NULL;
1113         xmlNode *xml_set = NULL;
1114 
1115         xml_resource = create_xml_node(NULL, XML_CIB_TAG_INCARNATION);
1116 
1117         /* @COMPAT We no longer use the <master> tag, but we need to keep it as
1118          * part of the resource name, so that bundles don't restart in a rolling
1119          * upgrade. (It also avoids needing to change regression tests.)
1120          */
1121         crm_xml_set_id(xml_resource, "%s-%s", bundle_data->prefix,
1122                       (bundle_data->promoted_max? "master"
1123                       : (const char *)xml_resource->name));
1124 
1125         xml_set = create_xml_node(xml_resource, XML_TAG_META_SETS);
1126         crm_xml_set_id(xml_set, "%s-%s-meta", bundle_data->prefix, xml_resource->name);
1127 
1128         crm_create_nvpair_xml(xml_set, NULL,
1129                               XML_RSC_ATTR_ORDERED, XML_BOOLEAN_TRUE);
1130 
1131         value = pcmk__itoa(bundle_data->nreplicas);
1132         crm_create_nvpair_xml(xml_set, NULL, PCMK_META_CLONE_MAX, value);
1133         free(value);
1134 
1135         value = pcmk__itoa(bundle_data->nreplicas_per_host);
1136         crm_create_nvpair_xml(xml_set, NULL, PCMK_META_CLONE_NODE_MAX, value);
1137         free(value);
1138 
1139         crm_create_nvpair_xml(xml_set, NULL, XML_RSC_ATTR_UNIQUE,
1140                               pcmk__btoa(bundle_data->nreplicas_per_host > 1));
1141 
1142         if (bundle_data->promoted_max) {
1143             crm_create_nvpair_xml(xml_set, NULL,
1144                                   XML_RSC_ATTR_PROMOTABLE, XML_BOOLEAN_TRUE);
1145 
1146             value = pcmk__itoa(bundle_data->promoted_max);
1147             crm_create_nvpair_xml(xml_set, NULL, PCMK_META_PROMOTED_MAX, value);
1148             free(value);
1149         }
1150 
1151         //crm_xml_add(xml_obj, XML_ATTR_ID, bundle_data->prefix);
1152         add_node_copy(xml_resource, xml_obj);
1153 
1154     } else if(xml_obj) {
1155         pe_err("Cannot control %s inside %s without either ip-range-start or control-port",
1156                rsc->id, ID(xml_obj));
1157         return FALSE;
1158     }
1159 
1160     if(xml_resource) {
1161         int lpc = 0;
1162         GList *childIter = NULL;
1163         pe__bundle_port_t *port = NULL;
1164         GString *buffer = NULL;
1165 
1166         if (pe__unpack_resource(xml_resource, &(bundle_data->child), rsc,
1167                                 scheduler) != pcmk_rc_ok) {
1168             return FALSE;
1169         }
1170 
1171         /* Currently, we always map the default authentication key location
1172          * into the same location inside the container.
1173          *
1174          * Ideally, we would respect the host's PCMK_authkey_location, but:
1175          * - it may be different on different nodes;
1176          * - the actual connection will do extra checking to make sure the key
1177          *   file exists and is readable, that we can't do here on the DC
1178          * - tools such as crm_resource and crm_simulate may not have the same
1179          *   environment variables as the cluster, causing operation digests to
1180          *   differ
1181          *
1182          * Always using the default location inside the container is fine,
1183          * because we control the pacemaker_remote environment, and it avoids
1184          * having to pass another environment variable to the container.
1185          *
1186          * @TODO A better solution may be to have only pacemaker_remote use the
1187          * environment variable, and have the cluster nodes use a new
1188          * cluster option for key location. This would introduce the limitation
1189          * of the location being the same on all cluster nodes, but that's
1190          * reasonable.
1191          */
1192         mount_add(bundle_data, DEFAULT_REMOTE_KEY_LOCATION,
1193                   DEFAULT_REMOTE_KEY_LOCATION, NULL, pe__bundle_mount_none);
1194 
1195         if (need_log_mount) {
1196             mount_add(bundle_data, CRM_BUNDLE_DIR, "/var/log", NULL,
1197                       pe__bundle_mount_subdir);
1198         }
1199 
1200         port = calloc(1, sizeof(pe__bundle_port_t));
1201         if(bundle_data->control_port) {
1202             port->source = strdup(bundle_data->control_port);
1203         } else {
1204             /* If we wanted to respect PCMK_remote_port, we could use
1205              * crm_default_remote_port() here and elsewhere in this file instead
1206              * of DEFAULT_REMOTE_PORT.
1207              *
1208              * However, it gains nothing, since we control both the container
1209              * environment and the connection resource parameters, and the user
1210              * can use a different port if desired by setting control-port.
1211              */
1212             port->source = pcmk__itoa(DEFAULT_REMOTE_PORT);
1213         }
1214         port->target = strdup(port->source);
1215         bundle_data->ports = g_list_append(bundle_data->ports, port);
1216 
1217         buffer = g_string_sized_new(1024);
1218         for (childIter = bundle_data->child->children; childIter != NULL;
1219              childIter = childIter->next) {
1220 
1221             pe__bundle_replica_t *replica = calloc(1, sizeof(pe__bundle_replica_t));
1222 
1223             replica->child = childIter->data;
1224             replica->child->exclusive_discover = TRUE;
1225             replica->offset = lpc++;
1226 
1227             // Ensure the child's notify gets set based on the underlying primitive's value
1228             if (pcmk_is_set(replica->child->flags, pcmk_rsc_notify)) {
1229                 pe__set_resource_flags(bundle_data->child, pcmk_rsc_notify);
1230             }
1231 
1232             allocate_ip(bundle_data, replica, buffer);
1233             bundle_data->replicas = g_list_append(bundle_data->replicas,
1234                                                   replica);
1235             bundle_data->attribute_target = g_hash_table_lookup(replica->child->meta,
1236                                                                 XML_RSC_ATTR_TARGET);
1237         }
1238         bundle_data->container_host_options = g_string_free(buffer, FALSE);
1239 
1240         if (bundle_data->attribute_target) {
1241             g_hash_table_replace(rsc->meta, strdup(XML_RSC_ATTR_TARGET),
1242                                  strdup(bundle_data->attribute_target));
1243             g_hash_table_replace(bundle_data->child->meta,
1244                                  strdup(XML_RSC_ATTR_TARGET),
1245                                  strdup(bundle_data->attribute_target));
1246         }
1247 
1248     } else {
1249         // Just a naked container, no pacemaker-remote
1250         GString *buffer = g_string_sized_new(1024);
1251 
1252         for (int lpc = 0; lpc < bundle_data->nreplicas; lpc++) {
1253             pe__bundle_replica_t *replica = calloc(1, sizeof(pe__bundle_replica_t));
1254 
1255             replica->offset = lpc;
1256             allocate_ip(bundle_data, replica, buffer);
1257             bundle_data->replicas = g_list_append(bundle_data->replicas,
1258                                                   replica);
1259         }
1260         bundle_data->container_host_options = g_string_free(buffer, FALSE);
1261     }
1262 
1263     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1264          gIter = gIter->next) {
1265         pe__bundle_replica_t *replica = gIter->data;
1266 
1267         if (create_replica_resources(rsc, bundle_data, replica) != pcmk_rc_ok) {
1268             pe_err("Failed unpacking resource %s", rsc->id);
1269             rsc->fns->free(rsc);
1270             return FALSE;
1271         }
1272 
1273         /* Utilization needs special handling for bundles. It makes no sense for
1274          * the inner primitive to have utilization, because it is tied
1275          * one-to-one to the guest node created by the container resource -- and
1276          * there's no way to set capacities for that guest node anyway.
1277          *
1278          * What the user really wants is to configure utilization for the
1279          * container. However, the schema only allows utilization for
1280          * primitives, and the container resource is implicit anyway, so the
1281          * user can *only* configure utilization for the inner primitive. If
1282          * they do, move the primitive's utilization values to the container.
1283          *
1284          * @TODO This means that bundles without an inner primitive can't have
1285          * utilization. An alternative might be to allow utilization values in
1286          * the top-level bundle XML in the schema, and copy those to each
1287          * container.
1288          */
1289         if (replica->child != NULL) {
1290             GHashTable *empty = replica->container->utilization;
1291 
1292             replica->container->utilization = replica->child->utilization;
1293             replica->child->utilization = empty;
1294         }
1295     }
1296 
1297     if (bundle_data->child) {
1298         rsc->children = g_list_append(rsc->children, bundle_data->child);
1299     }
1300     return TRUE;
1301 }
1302 
1303 static int
1304 replica_resource_active(pcmk_resource_t *rsc, gboolean all)
     /*  */
1305 {
1306     if (rsc) {
1307         gboolean child_active = rsc->fns->active(rsc, all);
1308 
1309         if (child_active && !all) {
1310             return TRUE;
1311         } else if (!child_active && all) {
1312             return FALSE;
1313         }
1314     }
1315     return -1;
1316 }
1317 
1318 gboolean
1319 pe__bundle_active(pcmk_resource_t *rsc, gboolean all)
     /*  */
1320 {
1321     pe__bundle_variant_data_t *bundle_data = NULL;
1322     GList *iter = NULL;
1323 
1324     get_bundle_variant_data(bundle_data, rsc);
1325     for (iter = bundle_data->replicas; iter != NULL; iter = iter->next) {
1326         pe__bundle_replica_t *replica = iter->data;
1327         int rsc_active;
1328 
1329         rsc_active = replica_resource_active(replica->ip, all);
1330         if (rsc_active >= 0) {
1331             return (gboolean) rsc_active;
1332         }
1333 
1334         rsc_active = replica_resource_active(replica->child, all);
1335         if (rsc_active >= 0) {
1336             return (gboolean) rsc_active;
1337         }
1338 
1339         rsc_active = replica_resource_active(replica->container, all);
1340         if (rsc_active >= 0) {
1341             return (gboolean) rsc_active;
1342         }
1343 
1344         rsc_active = replica_resource_active(replica->remote, all);
1345         if (rsc_active >= 0) {
1346             return (gboolean) rsc_active;
1347         }
1348     }
1349 
1350     /* If "all" is TRUE, we've already checked that no resources were inactive,
1351      * so return TRUE; if "all" is FALSE, we didn't find any active resources,
1352      * so return FALSE.
1353      */
1354     return all;
1355 }
1356 
1357 /*!
1358  * \internal
1359  * \brief Find the bundle replica corresponding to a given node
1360  *
1361  * \param[in] bundle  Top-level bundle resource
1362  * \param[in] node    Node to search for
1363  *
1364  * \return Bundle replica if found, NULL otherwise
1365  */
1366 pcmk_resource_t *
1367 pe__find_bundle_replica(const pcmk_resource_t *bundle, const pcmk_node_t *node)
     /*  */
1368 {
1369     pe__bundle_variant_data_t *bundle_data = NULL;
1370     CRM_ASSERT(bundle && node);
1371 
1372     get_bundle_variant_data(bundle_data, bundle);
1373     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1374          gIter = gIter->next) {
1375         pe__bundle_replica_t *replica = gIter->data;
1376 
1377         CRM_ASSERT(replica && replica->node);
1378         if (replica->node->details == node->details) {
1379             return replica->child;
1380         }
1381     }
1382     return NULL;
1383 }
1384 
1385 /*!
1386  * \internal
1387  * \deprecated This function will be removed in a future release
1388  */
1389 static void
1390 print_rsc_in_list(pcmk_resource_t *rsc, const char *pre_text, long options,
     /*  */
1391                   void *print_data)
1392 {
1393     if (rsc != NULL) {
1394         if (options & pe_print_html) {
1395             status_print("<li>");
1396         }
1397         rsc->fns->print(rsc, pre_text, options, print_data);
1398         if (options & pe_print_html) {
1399             status_print("</li>\n");
1400         }
1401     }
1402 }
1403 
1404 /*!
1405  * \internal
1406  * \deprecated This function will be removed in a future release
1407  */
1408 static void
1409 bundle_print_xml(pcmk_resource_t *rsc, const char *pre_text, long options,
     /*  */
1410                  void *print_data)
1411 {
1412     pe__bundle_variant_data_t *bundle_data = NULL;
1413     char *child_text = NULL;
1414     CRM_CHECK(rsc != NULL, return);
1415 
1416     if (pre_text == NULL) {
1417         pre_text = "";
1418     }
1419     child_text = crm_strdup_printf("%s        ", pre_text);
1420 
1421     get_bundle_variant_data(bundle_data, rsc);
1422 
1423     status_print("%s<bundle ", pre_text);
1424     status_print(XML_ATTR_ID "=\"%s\" ", rsc->id);
1425     status_print("type=\"%s\" ", container_agent_str(bundle_data->agent_type));
1426     status_print("image=\"%s\" ", bundle_data->image);
1427     status_print("unique=\"%s\" ", pe__rsc_bool_str(rsc, pcmk_rsc_unique));
1428     status_print("managed=\"%s\" ",
1429                  pe__rsc_bool_str(rsc, pcmk_rsc_managed));
1430     status_print("failed=\"%s\" ", pe__rsc_bool_str(rsc, pcmk_rsc_failed));
1431     status_print(">\n");
1432 
1433     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1434          gIter = gIter->next) {
1435         pe__bundle_replica_t *replica = gIter->data;
1436 
1437         CRM_ASSERT(replica);
1438         status_print("%s    <replica " XML_ATTR_ID "=\"%d\">\n",
1439                      pre_text, replica->offset);
1440         print_rsc_in_list(replica->ip, child_text, options, print_data);
1441         print_rsc_in_list(replica->child, child_text, options, print_data);
1442         print_rsc_in_list(replica->container, child_text, options, print_data);
1443         print_rsc_in_list(replica->remote, child_text, options, print_data);
1444         status_print("%s    </replica>\n", pre_text);
1445     }
1446     status_print("%s</bundle>\n", pre_text);
1447     free(child_text);
1448 }
1449 
1450 PCMK__OUTPUT_ARGS("bundle", "uint32_t", "pcmk_resource_t *", "GList *",
     /*  */
1451                   "GList *")
1452 int
1453 pe__bundle_xml(pcmk__output_t *out, va_list args)
1454 {
1455     uint32_t show_opts = va_arg(args, uint32_t);
1456     pcmk_resource_t *rsc = va_arg(args, pcmk_resource_t *);
1457     GList *only_node = va_arg(args, GList *);
1458     GList *only_rsc = va_arg(args, GList *);
1459 
1460     pe__bundle_variant_data_t *bundle_data = NULL;
1461     int rc = pcmk_rc_no_output;
1462     gboolean printed_header = FALSE;
1463     gboolean print_everything = TRUE;
1464 
1465     const char *desc = NULL;
1466 
1467     CRM_ASSERT(rsc != NULL);
1468     
1469     get_bundle_variant_data(bundle_data, rsc);
1470 
1471     if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
1472         return rc;
1473     }
1474 
1475     print_everything = pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches);
1476 
1477     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1478          gIter = gIter->next) {
1479         pe__bundle_replica_t *replica = gIter->data;
1480         char *id = NULL;
1481         gboolean print_ip, print_child, print_ctnr, print_remote;
1482 
1483         CRM_ASSERT(replica);
1484 
1485         if (pcmk__rsc_filtered_by_node(replica->container, only_node)) {
1486             continue;
1487         }
1488 
1489         print_ip = replica->ip != NULL &&
1490                    !replica->ip->fns->is_filtered(replica->ip, only_rsc, print_everything);
1491         print_child = replica->child != NULL &&
1492                       !replica->child->fns->is_filtered(replica->child, only_rsc, print_everything);
1493         print_ctnr = !replica->container->fns->is_filtered(replica->container, only_rsc, print_everything);
1494         print_remote = replica->remote != NULL &&
1495                        !replica->remote->fns->is_filtered(replica->remote, only_rsc, print_everything);
1496 
1497         if (!print_everything && !print_ip && !print_child && !print_ctnr && !print_remote) {
1498             continue;
1499         }
1500 
1501         if (!printed_header) {
1502             printed_header = TRUE;
1503 
1504             desc = pe__resource_description(rsc, show_opts);
1505 
1506             rc = pe__name_and_nvpairs_xml(out, true, "bundle", 8,
1507                      "id", rsc->id,
1508                      "type", container_agent_str(bundle_data->agent_type),
1509                      "image", bundle_data->image,
1510                      "unique", pe__rsc_bool_str(rsc, pcmk_rsc_unique),
1511                      "maintenance",
1512                      pe__rsc_bool_str(rsc, pcmk_rsc_maintenance),
1513                      "managed", pe__rsc_bool_str(rsc, pcmk_rsc_managed),
1514                      "failed", pe__rsc_bool_str(rsc, pcmk_rsc_failed),
1515                      "description", desc);
1516             CRM_ASSERT(rc == pcmk_rc_ok);
1517         }
1518 
1519         id = pcmk__itoa(replica->offset);
1520         rc = pe__name_and_nvpairs_xml(out, true, "replica", 1, "id", id);
1521         free(id);
1522         CRM_ASSERT(rc == pcmk_rc_ok);
1523 
1524         if (print_ip) {
1525             out->message(out, crm_map_element_name(replica->ip->xml), show_opts,
1526                          replica->ip, only_node, only_rsc);
1527         }
1528 
1529         if (print_child) {
1530             out->message(out, crm_map_element_name(replica->child->xml), show_opts,
1531                          replica->child, only_node, only_rsc);
1532         }
1533 
1534         if (print_ctnr) {
1535             out->message(out, crm_map_element_name(replica->container->xml), show_opts,
1536                          replica->container, only_node, only_rsc);
1537         }
1538 
1539         if (print_remote) {
1540             out->message(out, crm_map_element_name(replica->remote->xml), show_opts,
1541                          replica->remote, only_node, only_rsc);
1542         }
1543 
1544         pcmk__output_xml_pop_parent(out); // replica
1545     }
1546 
1547     if (printed_header) {
1548         pcmk__output_xml_pop_parent(out); // bundle
1549     }
1550 
1551     return rc;
1552 }
1553 
1554 static void
1555 pe__bundle_replica_output_html(pcmk__output_t *out, pe__bundle_replica_t *replica,
     /*  */
1556                                pcmk_node_t *node, uint32_t show_opts)
1557 {
1558     pcmk_resource_t *rsc = replica->child;
1559 
1560     int offset = 0;
1561     char buffer[LINE_MAX];
1562 
1563     if(rsc == NULL) {
1564         rsc = replica->container;
1565     }
1566 
1567     if (replica->remote) {
1568         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1569                            rsc_printable_id(replica->remote));
1570     } else {
1571         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1572                            rsc_printable_id(replica->container));
1573     }
1574     if (replica->ipaddr) {
1575         offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
1576                            replica->ipaddr);
1577     }
1578 
1579     pe__common_output_html(out, rsc, buffer, node, show_opts);
1580 }
1581 
1582 /*!
1583  * \internal
1584  * \brief Get a string describing a resource's unmanaged state or lack thereof
1585  *
1586  * \param[in] rsc  Resource to describe
1587  *
1588  * \return A string indicating that a resource is in maintenance mode or
1589  *         otherwise unmanaged, or an empty string otherwise
1590  */
1591 static const char *
1592 get_unmanaged_str(const pcmk_resource_t *rsc)
     /*  */
1593 {
1594     if (pcmk_is_set(rsc->flags, pcmk_rsc_maintenance)) {
1595         return " (maintenance)";
1596     }
1597     if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
1598         return " (unmanaged)";
1599     }
1600     return "";
1601 }
1602 
1603 PCMK__OUTPUT_ARGS("bundle", "uint32_t", "pcmk_resource_t *", "GList *",
     /*  */
1604                   "GList *")
1605 int
1606 pe__bundle_html(pcmk__output_t *out, va_list args)
1607 {
1608     uint32_t show_opts = va_arg(args, uint32_t);
1609     pcmk_resource_t *rsc = va_arg(args, pcmk_resource_t *);
1610     GList *only_node = va_arg(args, GList *);
1611     GList *only_rsc = va_arg(args, GList *);
1612 
1613     const char *desc = NULL;
1614     pe__bundle_variant_data_t *bundle_data = NULL;
1615     int rc = pcmk_rc_no_output;
1616     gboolean print_everything = TRUE;
1617 
1618     CRM_ASSERT(rsc != NULL);
1619 
1620     get_bundle_variant_data(bundle_data, rsc);
1621 
1622     desc = pe__resource_description(rsc, show_opts);
1623 
1624     if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
1625         return rc;
1626     }
1627 
1628     print_everything = pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches);
1629 
1630     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1631          gIter = gIter->next) {
1632         pe__bundle_replica_t *replica = gIter->data;
1633         gboolean print_ip, print_child, print_ctnr, print_remote;
1634 
1635         CRM_ASSERT(replica);
1636 
1637         if (pcmk__rsc_filtered_by_node(replica->container, only_node)) {
1638             continue;
1639         }
1640 
1641         print_ip = replica->ip != NULL &&
1642                    !replica->ip->fns->is_filtered(replica->ip, only_rsc, print_everything);
1643         print_child = replica->child != NULL &&
1644                       !replica->child->fns->is_filtered(replica->child, only_rsc, print_everything);
1645         print_ctnr = !replica->container->fns->is_filtered(replica->container, only_rsc, print_everything);
1646         print_remote = replica->remote != NULL &&
1647                        !replica->remote->fns->is_filtered(replica->remote, only_rsc, print_everything);
1648 
1649         if (pcmk_is_set(show_opts, pcmk_show_implicit_rscs) ||
1650             (print_everything == FALSE && (print_ip || print_child || print_ctnr || print_remote))) {
1651             /* The text output messages used below require pe_print_implicit to
1652              * be set to do anything.
1653              */
1654             uint32_t new_show_opts = show_opts | pcmk_show_implicit_rscs;
1655 
1656             PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
1657                                      (bundle_data->nreplicas > 1)? " set" : "",
1658                                      rsc->id, bundle_data->image,
1659                                      pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
1660                                      desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
1661                                      get_unmanaged_str(rsc));
1662 
1663             if (pcmk__list_of_multiple(bundle_data->replicas)) {
1664                 out->begin_list(out, NULL, NULL, "Replica[%d]", replica->offset);
1665             }
1666 
1667             if (print_ip) {
1668                 out->message(out, crm_map_element_name(replica->ip->xml),
1669                              new_show_opts, replica->ip, only_node, only_rsc);
1670             }
1671 
1672             if (print_child) {
1673                 out->message(out, crm_map_element_name(replica->child->xml),
1674                              new_show_opts, replica->child, only_node, only_rsc);
1675             }
1676 
1677             if (print_ctnr) {
1678                 out->message(out, crm_map_element_name(replica->container->xml),
1679                              new_show_opts, replica->container, only_node, only_rsc);
1680             }
1681 
1682             if (print_remote) {
1683                 out->message(out, crm_map_element_name(replica->remote->xml),
1684                              new_show_opts, replica->remote, only_node, only_rsc);
1685             }
1686 
1687             if (pcmk__list_of_multiple(bundle_data->replicas)) {
1688                 out->end_list(out);
1689             }
1690         } else if (print_everything == FALSE && !(print_ip || print_child || print_ctnr || print_remote)) {
1691             continue;
1692         } else {
1693             PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
1694                                      (bundle_data->nreplicas > 1)? " set" : "",
1695                                      rsc->id, bundle_data->image,
1696                                      pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
1697                                      desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
1698                                      get_unmanaged_str(rsc));
1699 
1700             pe__bundle_replica_output_html(out, replica, pe__current_node(replica->container),
1701                                            show_opts);
1702         }
1703     }
1704 
1705     PCMK__OUTPUT_LIST_FOOTER(out, rc);
1706     return rc;
1707 }
1708 
1709 static void
1710 pe__bundle_replica_output_text(pcmk__output_t *out, pe__bundle_replica_t *replica,
     /*  */
1711                                pcmk_node_t *node, uint32_t show_opts)
1712 {
1713     const pcmk_resource_t *rsc = replica->child;
1714 
1715     int offset = 0;
1716     char buffer[LINE_MAX];
1717 
1718     if(rsc == NULL) {
1719         rsc = replica->container;
1720     }
1721 
1722     if (replica->remote) {
1723         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1724                            rsc_printable_id(replica->remote));
1725     } else {
1726         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1727                            rsc_printable_id(replica->container));
1728     }
1729     if (replica->ipaddr) {
1730         offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
1731                            replica->ipaddr);
1732     }
1733 
1734     pe__common_output_text(out, rsc, buffer, node, show_opts);
1735 }
1736 
1737 PCMK__OUTPUT_ARGS("bundle", "uint32_t", "pcmk_resource_t *", "GList *",
     /*  */
1738                   "GList *")
1739 int
1740 pe__bundle_text(pcmk__output_t *out, va_list args)
1741 {
1742     uint32_t show_opts = va_arg(args, uint32_t);
1743     pcmk_resource_t *rsc = va_arg(args, pcmk_resource_t *);
1744     GList *only_node = va_arg(args, GList *);
1745     GList *only_rsc = va_arg(args, GList *);
1746 
1747     const char *desc = NULL;
1748     pe__bundle_variant_data_t *bundle_data = NULL;
1749     int rc = pcmk_rc_no_output;
1750     gboolean print_everything = TRUE;
1751 
1752     desc = pe__resource_description(rsc, show_opts);
1753     
1754     get_bundle_variant_data(bundle_data, rsc);
1755 
1756     CRM_ASSERT(rsc != NULL);
1757 
1758     if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
1759         return rc;
1760     }
1761 
1762     print_everything = pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches);
1763 
1764     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1765          gIter = gIter->next) {
1766         pe__bundle_replica_t *replica = gIter->data;
1767         gboolean print_ip, print_child, print_ctnr, print_remote;
1768 
1769         CRM_ASSERT(replica);
1770 
1771         if (pcmk__rsc_filtered_by_node(replica->container, only_node)) {
1772             continue;
1773         }
1774 
1775         print_ip = replica->ip != NULL &&
1776                    !replica->ip->fns->is_filtered(replica->ip, only_rsc, print_everything);
1777         print_child = replica->child != NULL &&
1778                       !replica->child->fns->is_filtered(replica->child, only_rsc, print_everything);
1779         print_ctnr = !replica->container->fns->is_filtered(replica->container, only_rsc, print_everything);
1780         print_remote = replica->remote != NULL &&
1781                        !replica->remote->fns->is_filtered(replica->remote, only_rsc, print_everything);
1782 
1783         if (pcmk_is_set(show_opts, pcmk_show_implicit_rscs) ||
1784             (print_everything == FALSE && (print_ip || print_child || print_ctnr || print_remote))) {
1785             /* The text output messages used below require pe_print_implicit to
1786              * be set to do anything.
1787              */
1788             uint32_t new_show_opts = show_opts | pcmk_show_implicit_rscs;
1789 
1790             PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
1791                                      (bundle_data->nreplicas > 1)? " set" : "",
1792                                      rsc->id, bundle_data->image,
1793                                      pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
1794                                      desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
1795                                      get_unmanaged_str(rsc));
1796 
1797             if (pcmk__list_of_multiple(bundle_data->replicas)) {
1798                 out->list_item(out, NULL, "Replica[%d]", replica->offset);
1799             }
1800 
1801             out->begin_list(out, NULL, NULL, NULL);
1802 
1803             if (print_ip) {
1804                 out->message(out, crm_map_element_name(replica->ip->xml),
1805                              new_show_opts, replica->ip, only_node, only_rsc);
1806             }
1807 
1808             if (print_child) {
1809                 out->message(out, crm_map_element_name(replica->child->xml),
1810                              new_show_opts, replica->child, only_node, only_rsc);
1811             }
1812 
1813             if (print_ctnr) {
1814                 out->message(out, crm_map_element_name(replica->container->xml),
1815                              new_show_opts, replica->container, only_node, only_rsc);
1816             }
1817 
1818             if (print_remote) {
1819                 out->message(out, crm_map_element_name(replica->remote->xml),
1820                              new_show_opts, replica->remote, only_node, only_rsc);
1821             }
1822 
1823             out->end_list(out);
1824         } else if (print_everything == FALSE && !(print_ip || print_child || print_ctnr || print_remote)) {
1825             continue;
1826         } else {
1827             PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
1828                                      (bundle_data->nreplicas > 1)? " set" : "",
1829                                      rsc->id, bundle_data->image,
1830                                      pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
1831                                      desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
1832                                      get_unmanaged_str(rsc));
1833 
1834             pe__bundle_replica_output_text(out, replica, pe__current_node(replica->container),
1835                                            show_opts);
1836         }
1837     }
1838 
1839     PCMK__OUTPUT_LIST_FOOTER(out, rc);
1840     return rc;
1841 }
1842 
1843 /*!
1844  * \internal
1845  * \deprecated This function will be removed in a future release
1846  */
1847 static void
1848 print_bundle_replica(pe__bundle_replica_t *replica, const char *pre_text,
     /*  */
1849                      long options, void *print_data)
1850 {
1851     pcmk_node_t *node = NULL;
1852     pcmk_resource_t *rsc = replica->child;
1853 
1854     int offset = 0;
1855     char buffer[LINE_MAX];
1856 
1857     if(rsc == NULL) {
1858         rsc = replica->container;
1859     }
1860 
1861     if (replica->remote) {
1862         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1863                            rsc_printable_id(replica->remote));
1864     } else {
1865         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1866                            rsc_printable_id(replica->container));
1867     }
1868     if (replica->ipaddr) {
1869         offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
1870                            replica->ipaddr);
1871     }
1872 
1873     node = pe__current_node(replica->container);
1874     common_print(rsc, pre_text, buffer, node, options, print_data);
1875 }
1876 
1877 /*!
1878  * \internal
1879  * \deprecated This function will be removed in a future release
1880  */
1881 void
1882 pe__print_bundle(pcmk_resource_t *rsc, const char *pre_text, long options,
     /*  */
1883                  void *print_data)
1884 {
1885     pe__bundle_variant_data_t *bundle_data = NULL;
1886     char *child_text = NULL;
1887     CRM_CHECK(rsc != NULL, return);
1888 
1889     if (options & pe_print_xml) {
1890         bundle_print_xml(rsc, pre_text, options, print_data);
1891         return;
1892     }
1893 
1894     get_bundle_variant_data(bundle_data, rsc);
1895 
1896     if (pre_text == NULL) {
1897         pre_text = " ";
1898     }
1899 
1900     status_print("%sContainer bundle%s: %s [%s]%s%s\n",
1901                  pre_text, ((bundle_data->nreplicas > 1)? " set" : ""),
1902                  rsc->id, bundle_data->image,
1903                  pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
1904                  pcmk_is_set(rsc->flags, pcmk_rsc_managed)? "" : " (unmanaged)");
1905     if (options & pe_print_html) {
1906         status_print("<br />\n<ul>\n");
1907     }
1908 
1909 
1910     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1911          gIter = gIter->next) {
1912         pe__bundle_replica_t *replica = gIter->data;
1913 
1914         CRM_ASSERT(replica);
1915         if (options & pe_print_html) {
1916             status_print("<li>");
1917         }
1918 
1919         if (pcmk_is_set(options, pe_print_implicit)) {
1920             child_text = crm_strdup_printf("     %s", pre_text);
1921             if (pcmk__list_of_multiple(bundle_data->replicas)) {
1922                 status_print("  %sReplica[%d]\n", pre_text, replica->offset);
1923             }
1924             if (options & pe_print_html) {
1925                 status_print("<br />\n<ul>\n");
1926             }
1927             print_rsc_in_list(replica->ip, child_text, options, print_data);
1928             print_rsc_in_list(replica->container, child_text, options, print_data);
1929             print_rsc_in_list(replica->remote, child_text, options, print_data);
1930             print_rsc_in_list(replica->child, child_text, options, print_data);
1931             if (options & pe_print_html) {
1932                 status_print("</ul>\n");
1933             }
1934         } else {
1935             child_text = crm_strdup_printf("%s  ", pre_text);
1936             print_bundle_replica(replica, child_text, options, print_data);
1937         }
1938         free(child_text);
1939 
1940         if (options & pe_print_html) {
1941             status_print("</li>\n");
1942         }
1943     }
1944     if (options & pe_print_html) {
1945         status_print("</ul>\n");
1946     }
1947 }
1948 
1949 static void
1950 free_bundle_replica(pe__bundle_replica_t *replica)
     /*  */
1951 {
1952     if (replica == NULL) {
1953         return;
1954     }
1955 
1956     if (replica->node) {
1957         free(replica->node);
1958         replica->node = NULL;
1959     }
1960 
1961     if (replica->ip) {
1962         free_xml(replica->ip->xml);
1963         replica->ip->xml = NULL;
1964         replica->ip->fns->free(replica->ip);
1965         replica->ip = NULL;
1966     }
1967     if (replica->container) {
1968         free_xml(replica->container->xml);
1969         replica->container->xml = NULL;
1970         replica->container->fns->free(replica->container);
1971         replica->container = NULL;
1972     }
1973     if (replica->remote) {
1974         free_xml(replica->remote->xml);
1975         replica->remote->xml = NULL;
1976         replica->remote->fns->free(replica->remote);
1977         replica->remote = NULL;
1978     }
1979     free(replica->ipaddr);
1980     free(replica);
1981 }
1982 
1983 void
1984 pe__free_bundle(pcmk_resource_t *rsc)
     /*  */
1985 {
1986     pe__bundle_variant_data_t *bundle_data = NULL;
1987     CRM_CHECK(rsc != NULL, return);
1988 
1989     get_bundle_variant_data(bundle_data, rsc);
1990     pe_rsc_trace(rsc, "Freeing %s", rsc->id);
1991 
1992     free(bundle_data->prefix);
1993     free(bundle_data->image);
1994     free(bundle_data->control_port);
1995     free(bundle_data->host_network);
1996     free(bundle_data->host_netmask);
1997     free(bundle_data->ip_range_start);
1998     free(bundle_data->container_network);
1999     free(bundle_data->launcher_options);
2000     free(bundle_data->container_command);
2001     g_free(bundle_data->container_host_options);
2002 
2003     g_list_free_full(bundle_data->replicas,
2004                      (GDestroyNotify) free_bundle_replica);
2005     g_list_free_full(bundle_data->mounts, (GDestroyNotify)mount_free);
2006     g_list_free_full(bundle_data->ports, (GDestroyNotify)port_free);
2007     g_list_free(rsc->children);
2008 
2009     if(bundle_data->child) {
2010         free_xml(bundle_data->child->xml);
2011         bundle_data->child->xml = NULL;
2012         bundle_data->child->fns->free(bundle_data->child);
2013     }
2014     common_free(rsc);
2015 }
2016 
2017 enum rsc_role_e
2018 pe__bundle_resource_state(const pcmk_resource_t *rsc, gboolean current)
     /*  */
2019 {
2020     enum rsc_role_e container_role = pcmk_role_unknown;
2021     return container_role;
2022 }
2023 
2024 /*!
2025  * \brief Get the number of configured replicas in a bundle
2026  *
2027  * \param[in] rsc  Bundle resource
2028  *
2029  * \return Number of configured replicas, or 0 on error
2030  */
2031 int
2032 pe_bundle_replicas(const pcmk_resource_t *rsc)
     /*  */
2033 {
2034     if ((rsc == NULL) || (rsc->variant != pcmk_rsc_variant_bundle)) {
2035         return 0;
2036     } else {
2037         pe__bundle_variant_data_t *bundle_data = NULL;
2038 
2039         get_bundle_variant_data(bundle_data, rsc);
2040         return bundle_data->nreplicas;
2041     }
2042 }
2043 
2044 void
2045 pe__count_bundle(pcmk_resource_t *rsc)
     /*  */
2046 {
2047     pe__bundle_variant_data_t *bundle_data = NULL;
2048 
2049     get_bundle_variant_data(bundle_data, rsc);
2050     for (GList *item = bundle_data->replicas; item != NULL; item = item->next) {
2051         pe__bundle_replica_t *replica = item->data;
2052 
2053         if (replica->ip) {
2054             replica->ip->fns->count(replica->ip);
2055         }
2056         if (replica->child) {
2057             replica->child->fns->count(replica->child);
2058         }
2059         if (replica->container) {
2060             replica->container->fns->count(replica->container);
2061         }
2062         if (replica->remote) {
2063             replica->remote->fns->count(replica->remote);
2064         }
2065     }
2066 }
2067 
2068 gboolean
2069 pe__bundle_is_filtered(const pcmk_resource_t *rsc, GList *only_rsc,
     /*  */
2070                        gboolean check_parent)
2071 {
2072     gboolean passes = FALSE;
2073     pe__bundle_variant_data_t *bundle_data = NULL;
2074 
2075     if (pcmk__str_in_list(rsc_printable_id(rsc), only_rsc, pcmk__str_star_matches)) {
2076         passes = TRUE;
2077     } else {
2078         get_bundle_variant_data(bundle_data, rsc);
2079 
2080         for (GList *gIter = bundle_data->replicas; gIter != NULL; gIter = gIter->next) {
2081             pe__bundle_replica_t *replica = gIter->data;
2082 
2083             if (replica->ip != NULL && !replica->ip->fns->is_filtered(replica->ip, only_rsc, FALSE)) {
2084                 passes = TRUE;
2085                 break;
2086             } else if (replica->child != NULL && !replica->child->fns->is_filtered(replica->child, only_rsc, FALSE)) {
2087                 passes = TRUE;
2088                 break;
2089             } else if (!replica->container->fns->is_filtered(replica->container, only_rsc, FALSE)) {
2090                 passes = TRUE;
2091                 break;
2092             } else if (replica->remote != NULL && !replica->remote->fns->is_filtered(replica->remote, only_rsc, FALSE)) {
2093                 passes = TRUE;
2094                 break;
2095             }
2096         }
2097     }
2098 
2099     return !passes;
2100 }
2101 
2102 /*!
2103  * \internal
2104  * \brief Get a list of a bundle's containers
2105  *
2106  * \param[in] bundle  Bundle resource
2107  *
2108  * \return Newly created list of \p bundle's containers
2109  * \note It is the caller's responsibility to free the result with
2110  *       g_list_free().
2111  */
2112 GList *
2113 pe__bundle_containers(const pcmk_resource_t *bundle)
     /*  */
2114 {
2115     GList *containers = NULL;
2116     const pe__bundle_variant_data_t *data = NULL;
2117 
2118     get_bundle_variant_data(data, bundle);
2119     for (GList *iter = data->replicas; iter != NULL; iter = iter->next) {
2120         pe__bundle_replica_t *replica = iter->data;
2121 
2122         containers = g_list_append(containers, replica->container);
2123     }
2124     return containers;
2125 }
2126 
2127 // Bundle implementation of pcmk_rsc_methods_t:active_node()
2128 pcmk_node_t *
2129 pe__bundle_active_node(const pcmk_resource_t *rsc, unsigned int *count_all,
     /*  */
2130                        unsigned int *count_clean)
2131 {
2132     pcmk_node_t *active = NULL;
2133     pcmk_node_t *node = NULL;
2134     pcmk_resource_t *container = NULL;
2135     GList *containers = NULL;
2136     GList *iter = NULL;
2137     GHashTable *nodes = NULL;
2138     const pe__bundle_variant_data_t *data = NULL;
2139 
2140     if (count_all != NULL) {
2141         *count_all = 0;
2142     }
2143     if (count_clean != NULL) {
2144         *count_clean = 0;
2145     }
2146     if (rsc == NULL) {
2147         return NULL;
2148     }
2149 
2150     /* For the purposes of this method, we only care about where the bundle's
2151      * containers are active, so build a list of active containers.
2152      */
2153     get_bundle_variant_data(data, rsc);
2154     for (iter = data->replicas; iter != NULL; iter = iter->next) {
2155         pe__bundle_replica_t *replica = iter->data;
2156 
2157         if (replica->container->running_on != NULL) {
2158             containers = g_list_append(containers, replica->container);
2159         }
2160     }
2161     if (containers == NULL) {
2162         return NULL;
2163     }
2164 
2165     /* If the bundle has only a single active container, just use that
2166      * container's method. If live migration is ever supported for bundle
2167      * containers, this will allow us to prefer the migration source when there
2168      * is only one container and it is migrating. For now, this just lets us
2169      * avoid creating the nodes table.
2170      */
2171     if (pcmk__list_of_1(containers)) {
2172         container = containers->data;
2173         node = container->fns->active_node(container, count_all, count_clean);
2174         g_list_free(containers);
2175         return node;
2176     }
2177 
2178     // Add all containers' active nodes to a hash table (for uniqueness)
2179     nodes = g_hash_table_new(NULL, NULL);
2180     for (iter = containers; iter != NULL; iter = iter->next) {
2181         container = iter->data;
2182 
2183         for (GList *node_iter = container->running_on; node_iter != NULL;
2184              node_iter = node_iter->next) {
2185             node = node_iter->data;
2186 
2187             // If insert returns true, we haven't counted this node yet
2188             if (g_hash_table_insert(nodes, (gpointer) node->details,
2189                                     (gpointer) node)
2190                 && !pe__count_active_node(rsc, node, &active, count_all,
2191                                           count_clean)) {
2192                 goto done;
2193             }
2194         }
2195     }
2196 
2197 done:
2198     g_list_free(containers);
2199     g_hash_table_destroy(nodes);
2200     return active;
2201 }
2202 
2203 /*!
2204  * \internal
2205  * \brief Get maximum bundle resource instances per node
2206  *
2207  * \param[in] rsc  Bundle resource to check
2208  *
2209  * \return Maximum number of \p rsc instances that can be active on one node
2210  */
2211 unsigned int
2212 pe__bundle_max_per_node(const pcmk_resource_t *rsc)
     /*  */
2213 {
2214     pe__bundle_variant_data_t *bundle_data = NULL;
2215 
2216     get_bundle_variant_data(bundle_data, rsc);
2217     CRM_ASSERT(bundle_data->nreplicas_per_host >= 0);
2218     return (unsigned int) bundle_data->nreplicas_per_host;
2219 }
/* */
root/lib/pengine/bundle.c

DEFINITIONS
