%% Generated by Sphinx.
\def\sphinxdocclass{report}
\documentclass[letterpaper,10pt,english]{sphinxmanual}
\ifdefined\pdfpxdimen
   \let\sphinxpxdimen\pdfpxdimen\else\newdimen\sphinxpxdimen
\fi \sphinxpxdimen=.75bp\relax

\PassOptionsToPackage{warn}{textcomp}

\catcode`^^^^00a0\active\protected\def^^^^00a0{\leavevmode\nobreak\ }
\usepackage{cmap}
\usepackage{fontspec}
\usepackage{amsmath,amssymb,amstext}
\usepackage{polyglossia}
\setmainlanguage{english}

\usepackage[Bjarne]{fncychap}
\usepackage{sphinx}

\usepackage{geometry}

% Include hyperref last.
\usepackage{hyperref}
% Fix anchor placement for figures with captions.
\usepackage{hypcap}% it must be loaded after hyperref.
% Set up styles of URL: it should be placed after hyperref.
\urlstyle{same}

\addto\captionsenglish{\renewcommand{\figurename}{Fig.}}
\addto\captionsenglish{\renewcommand{\tablename}{Table}}
\addto\captionsenglish{\renewcommand{\literalblockname}{Listing}}

\addto\captionsenglish{\renewcommand{\literalblockcontinuedname}{continued from previous page}}
\addto\captionsenglish{\renewcommand{\literalblockcontinuesname}{continues on next page}}

\def\pageautorefname{page}

\setcounter{tocdepth}{2}



\title{Pacemaker Explained}
\date{Dec 19, 2023}
\release{2.1.7}
\author{the Pacemaker project contributors}
\newcommand{\sphinxlogo}{\vbox{}}
\renewcommand{\releasename}{Release}
\makeindex

\begin{document}

\maketitle
\sphinxtableofcontents
\phantomsection\label{\detokenize{index::doc}}


\sphinxstyleemphasis{Configuring Pacemaker Clusters}


\chapter{Abstract}
\label{\detokenize{index:abstract}}
This document definitively explains Pacemaker’s features and capabilities,
particularly the XML syntax used in Pacemaker’s Cluster Information Base (CIB).


\chapter{Table of Contents}
\label{\detokenize{index:table-of-contents}}

\section{Introduction}
\label{\detokenize{intro:introduction}}\label{\detokenize{intro::doc}}

\subsection{The Scope of this Document}
\label{\detokenize{intro:the-scope-of-this-document}}
This document is intended to be an exhaustive reference for configuring
Pacemaker. To achieve this, it focuses on the XML syntax used to configure the
CIB.

For those that are allergic to XML, multiple higher-level front-ends
(both command-line and GUI) are available. These tools will not be covered
in this document, though the concepts explained here should make the
functionality of these tools more easily understood.

Users may be interested in other parts of the
\sphinxhref{https://www.clusterlabs.org/pacemaker/doc/}{Pacemaker documentation set},
such as \sphinxstyleemphasis{Clusters from Scratch}, a step-by-step guide to setting up an
example cluster, and \sphinxstyleemphasis{Pacemaker Administration}, a guide to maintaining a
cluster.


\subsection{What Is Pacemaker?}
\label{\detokenize{intro:what-is-pacemaker}}
Pacemaker is a high-availability \sphinxstyleemphasis{cluster resource manager} \textendash{} software that
runs on a set of hosts (a \sphinxstyleemphasis{cluster} of \sphinxstyleemphasis{nodes}) in order to preserve integrity
and minimize downtime of desired services (\sphinxstyleemphasis{resources}). %
\begin{footnote}[1]\sphinxAtStartFootnote
\sphinxstyleemphasis{Cluster} is sometimes used in other contexts to refer to hosts grouped
together for other purposes, such as high-performance computing (HPC),
but Pacemaker is not intended for those purposes.
%
\end{footnote} It is maintained
by the \sphinxhref{https://www.ClusterLabs.org/}{ClusterLabs} community.

Pacemaker’s key features include:
\begin{itemize}
\item {} 
Detection of and recovery from node- and service-level failures

\item {} 
Ability to ensure data integrity by fencing faulty nodes

\item {} 
Support for one or more nodes per cluster

\item {} 
Support for multiple resource interface standards (anything that can be
scripted can be clustered)

\item {} 
Support (but no requirement) for shared storage

\item {} 
Support for practically any redundancy configuration (active/passive, N+1,
etc.)

\item {} 
Automatically replicated configuration that can be updated from any node

\item {} 
Ability to specify cluster-wide relationships between services,
such as ordering, colocation, and anti-colocation

\item {} 
Support for advanced service types, such as \sphinxstyleemphasis{clones} (services that need to
be active on multiple nodes), \sphinxstyleemphasis{promotable clones} (clones that can run in
one of two roles), and containerized services

\item {} 
Unified, scriptable cluster management tools

\end{itemize}

\begin{sphinxadmonition}{note}{Note:}
\sphinxstylestrong{Fencing}

\sphinxstyleemphasis{Fencing}, also known as \sphinxstyleemphasis{STONITH} (an acronym for Shoot The Other Node In
The Head), is the ability to ensure that it is not possible for a node to be
running a service. This is accomplished via \sphinxstyleemphasis{fence devices} such as
intelligent power switches that cut power to the target, or intelligent
network switches that cut the target’s access to the local network.

Pacemaker represents fence devices as a special class of resource.

A cluster cannot safely recover from certain failure conditions, such as an
unresponsive node, without fencing.
\end{sphinxadmonition}


\subsubsection{Cluster Architecture}
\label{\detokenize{intro:cluster-architecture}}
At a high level, a cluster can be viewed as having these parts (which together
are often referred to as the \sphinxstyleemphasis{cluster stack}):
\begin{itemize}
\item {} 
\sphinxstylestrong{Resources:} These are the reason for the cluster’s being \textendash{} the services
that need to be kept highly available.

\item {} 
\sphinxstylestrong{Resource agents:} These are scripts or operating system components that
start, stop, and monitor resources, given a set of resource parameters.
These provide a uniform interface between Pacemaker and the managed
services.

\item {} 
\sphinxstylestrong{Fence agents:} These are scripts that execute node fencing actions,
given a target and fence device parameters.

\item {} 
\sphinxstylestrong{Cluster membership layer:} This component provides reliable messaging,
membership, and quorum information about the cluster. Currently, Pacemaker
supports \sphinxhref{http://www.corosync.org/}{Corosync} as this layer.

\item {} 
\sphinxstylestrong{Cluster resource manager:} Pacemaker provides the brain that processes
and reacts to events that occur in the cluster. These events may include
nodes joining or leaving the cluster; resource events caused by failures,
maintenance, or scheduled activities; and other administrative actions.
To achieve the desired availability, Pacemaker may start and stop resources
and fence nodes.

\item {} 
\sphinxstylestrong{Cluster tools:} These provide an interface for users to interact with the
cluster. Various command-line and graphical (GUI) interfaces are available.

\end{itemize}

Most managed services are not, themselves, cluster-aware. However, many popular
open-source cluster filesystems make use of a common \sphinxstyleemphasis{Distributed Lock
Manager} (DLM), which makes direct use of Corosync for its messaging and
membership capabilities and Pacemaker for the ability to fence nodes.

\noindent{\hspace*{\fill}\sphinxincludegraphics{{pcmk-stack}.png}\hspace*{\fill}}


\subsubsection{Pacemaker Architecture}
\label{\detokenize{intro:pacemaker-architecture}}
Pacemaker itself is composed of multiple daemons that work together:
\begin{itemize}
\item {} 
\sphinxcode{\sphinxupquote{pacemakerd}}

\item {} 
\sphinxcode{\sphinxupquote{pacemaker-attrd}}

\item {} 
\sphinxcode{\sphinxupquote{pacemaker-based}}

\item {} 
\sphinxcode{\sphinxupquote{pacemaker-controld}}

\item {} 
\sphinxcode{\sphinxupquote{pacemaker-execd}}

\item {} 
\sphinxcode{\sphinxupquote{pacemaker-fenced}}

\item {} 
\sphinxcode{\sphinxupquote{pacemaker-schedulerd}}

\end{itemize}

\noindent{\hspace*{\fill}\sphinxincludegraphics{{pcmk-internals}.png}\hspace*{\fill}}

Pacemaker’s main process (\sphinxcode{\sphinxupquote{pacemakerd}}) spawns all the other daemons, and
respawns them if they unexpectedly exit.

The \sphinxstyleemphasis{Cluster Information Base} (CIB) is an
\sphinxhref{https://en.wikipedia.org/wiki/XML}{XML} representation of the cluster’s
configuration and the state of all nodes and resources. The \sphinxstyleemphasis{CIB manager}
(\sphinxcode{\sphinxupquote{pacemaker-based}}) keeps the CIB synchronized across the cluster, and
handles requests to modify it.

The \sphinxstyleemphasis{attribute manager} (\sphinxcode{\sphinxupquote{pacemaker-attrd}}) maintains a database of
attributes for all nodes, keeps it synchronized across the cluster, and handles
requests to modify them. These attributes are usually recorded in the CIB.

Given a snapshot of the CIB as input, the \sphinxstyleemphasis{scheduler}
(\sphinxcode{\sphinxupquote{pacemaker-schedulerd}}) determines what actions are necessary to achieve the
desired state of the cluster.

The \sphinxstyleemphasis{local executor} (\sphinxcode{\sphinxupquote{pacemaker-execd}}) handles requests to execute
resource agents on the local cluster node, and returns the result.

The \sphinxstyleemphasis{fencer} (\sphinxcode{\sphinxupquote{pacemaker-fenced}}) handles requests to fence nodes. Given a
target node, the fencer decides which cluster node(s) should execute which
fencing device(s), and calls the necessary fencing agents (either directly, or
via requests to the fencer peers on other nodes), and returns the result.

The \sphinxstyleemphasis{controller} (\sphinxcode{\sphinxupquote{pacemaker-controld}}) is Pacemaker’s coordinator,
maintaining a consistent view of the cluster membership and orchestrating all
the other components.

Pacemaker centralizes cluster decision-making by electing one of the controller
instances as the \sphinxstyleemphasis{Designated Controller} (\sphinxstyleemphasis{DC}). Should the elected DC process
(or the node it is on) fail, a new one is quickly established. The DC responds
to cluster events by taking a current snapshot of the CIB, feeding it to the
scheduler, then asking the executors (either directly on the local node, or via
requests to controller peers on other nodes) and the fencer to execute any
necessary actions.

\begin{sphinxadmonition}{note}{Note:}
\sphinxstylestrong{Old daemon names}

The Pacemaker daemons were renamed in version 2.0. You may still find
references to the old names, especially in documentation targeted to
version 1.1.


\begin{savenotes}\sphinxattablestart
\centering
\begin{tabulary}{\linewidth}[t]{|T|T|}
\hline
\sphinxstyletheadfamily 
Old name
&\sphinxstyletheadfamily 
New name
\\
\hline
\sphinxcode{\sphinxupquote{attrd}}
&
\sphinxcode{\sphinxupquote{pacemaker-attrd}}
\\
\hline
\sphinxcode{\sphinxupquote{cib}}
&
\sphinxcode{\sphinxupquote{pacemaker-based}}
\\
\hline
\sphinxcode{\sphinxupquote{crmd}}
&
\sphinxcode{\sphinxupquote{pacemaker-controld}}
\\
\hline
\sphinxcode{\sphinxupquote{lrmd}}
&
\sphinxcode{\sphinxupquote{pacemaker-execd}}
\\
\hline
\sphinxcode{\sphinxupquote{stonithd}}
&
\sphinxcode{\sphinxupquote{pacemaker-fenced}}
\\
\hline
\sphinxcode{\sphinxupquote{pacemaker\_remoted}}
&
\sphinxcode{\sphinxupquote{pacemaker-remoted}}
\\
\hline
\end{tabulary}
\par
\sphinxattableend\end{savenotes}
\end{sphinxadmonition}


\subsubsection{Node Redundancy Designs}
\label{\detokenize{intro:node-redundancy-designs}}
Pacemaker supports practically any \sphinxhref{https://en.wikipedia.org/wiki/High-availability\_cluster\#Node\_configurations}{node redundancy configuration}
including \sphinxstyleemphasis{Active/Active}, \sphinxstyleemphasis{Active/Passive}, \sphinxstyleemphasis{N+1}, \sphinxstyleemphasis{N+M}, \sphinxstyleemphasis{N-to-1}, and
\sphinxstyleemphasis{N-to-N}.

Active/passive clusters with two (or more) nodes using Pacemaker and
\sphinxhref{https://en.wikipedia.org/wiki/Distributed\_Replicated\_Block\_Device}{DRBD} are
a cost-effective high-availability solution for many situations. One of the
nodes provides the desired services, and if it fails, the other node takes
over.

\noindent{\hspace*{\fill}\sphinxincludegraphics{{pcmk-active-passive}.png}\hspace*{\fill}}

Pacemaker also supports multiple nodes in a shared-failover design, reducing
hardware costs by allowing several active/passive clusters to be combined and
share a common backup node.

\noindent{\hspace*{\fill}\sphinxincludegraphics{{pcmk-shared-failover}.png}\hspace*{\fill}}

When shared storage is available, every node can potentially be used for
failover. Pacemaker can even run multiple copies of services to spread out the
workload. This is sometimes called N-to-N redundancy.

\noindent{\hspace*{\fill}\sphinxincludegraphics{{pcmk-active-active}.png}\hspace*{\fill}}


\section{Host-Local Configuration}
\label{\detokenize{local-options:host-local-configuration}}\label{\detokenize{local-options::doc}}
\index{XML element!configuration}\index{configuration!XML element}\ignorespaces 
\begin{sphinxadmonition}{note}{Note:}
Directory and file paths below may differ on your system depending on
your Pacemaker build settings. Check your Pacemaker configuration
file to find the correct paths.
\end{sphinxadmonition}

Pacemaker supports several host-local configuration options. These options can
be configured on each node in the main Pacemaker configuration file
(\sphinxcode{\sphinxupquote{/etc/sysconfig/pacemaker}}) in the format \sphinxcode{\sphinxupquote{\textless{}NAME\textgreater{}="\textless{}VALUE\textgreater{}"}}. They work by setting
environment variables when Pacemaker daemons start up.


\begin{savenotes}\sphinxatlongtablestart\begin{longtable}{|\X{2}{11}|\X{2}{11}|\X{2}{11}|\X{5}{11}|}
\caption{\sphinxstylestrong{Local Options}\strut}\label{\detokenize{local-options:id1}}\\*[\sphinxlongtablecapskipadjust]
\hline
\sphinxstyletheadfamily 
Name
&\sphinxstyletheadfamily 
Type
&\sphinxstyletheadfamily 
Default
&\sphinxstyletheadfamily 
Description
\\
\hline
\endfirsthead

\multicolumn{4}{c}%
{\makebox[0pt]{\sphinxtablecontinued{\tablename\ \thetable{} -- continued from previous page}}}\\
\hline
\sphinxstyletheadfamily 
Name
&\sphinxstyletheadfamily 
Type
&\sphinxstyletheadfamily 
Default
&\sphinxstyletheadfamily 
Description
\\
\hline
\endhead

\hline
\multicolumn{4}{r}{\makebox[0pt][r]{\sphinxtablecontinued{Continued on next page}}}\\
\endfoot

\endlastfoot
\phantomsection\label{\detokenize{local-options:pcmk-logfacility}}
\index{node option!PCMK\_logfacility}\index{PCMK\_logfacility!node option}\ignorespaces 
PCMK\_logfacility
&
{\hyperref[\detokenize{cluster-options:enumeration}]{\sphinxcrossref{\DUrole{std,std-ref}{enumeration}}}}
&
daemon
&
Enable logging via the system log or journal, using the specified log
facility. Messages sent here are of value to all Pacemaker
administrators. This can be disabled using \sphinxcode{\sphinxupquote{none}}, but that is not
recommended. Allowed values:
\begin{itemize}
\item {} 
\sphinxcode{\sphinxupquote{none}}

\item {} 
\sphinxcode{\sphinxupquote{daemon}}

\item {} 
\sphinxcode{\sphinxupquote{user}}

\item {} 
\sphinxcode{\sphinxupquote{local0}}

\item {} 
\sphinxcode{\sphinxupquote{local1}}

\item {} 
\sphinxcode{\sphinxupquote{local2}}

\item {} 
\sphinxcode{\sphinxupquote{local3}}

\item {} 
\sphinxcode{\sphinxupquote{local4}}

\item {} 
\sphinxcode{\sphinxupquote{local5}}

\item {} 
\sphinxcode{\sphinxupquote{local6}}

\item {} 
\sphinxcode{\sphinxupquote{local7}}

\end{itemize}
\\
\hline\phantomsection\label{\detokenize{local-options:pcmk-logpriority}}
\index{: node option!PCMK\_logpriority}\index{PCMK\_logpriority!: node option}\ignorespaces 
PCMK\_logpriority
&
{\hyperref[\detokenize{cluster-options:enumeration}]{\sphinxcrossref{\DUrole{std,std-ref}{enumeration}}}}
&
notice
&
Unless system logging is disabled using \sphinxcode{\sphinxupquote{PCMK\_logfacility=none}},
messages of the specified log severity and higher will be sent to the
system log. The default is appropriate for most installations. Allowed
values:
\begin{itemize}
\item {} 
\sphinxcode{\sphinxupquote{emerg}}

\item {} 
\sphinxcode{\sphinxupquote{alert}}

\item {} 
\sphinxcode{\sphinxupquote{crit}}

\item {} 
\sphinxcode{\sphinxupquote{error}}

\item {} 
\sphinxcode{\sphinxupquote{warning}}

\item {} 
\sphinxcode{\sphinxupquote{notice}}

\item {} 
\sphinxcode{\sphinxupquote{info}}

\item {} 
\sphinxcode{\sphinxupquote{debug}}

\end{itemize}
\\
\hline\phantomsection\label{\detokenize{local-options:pcmk-logfile}}
\index{: node option!PCMK\_logfile}\index{PCMK\_logfile!: node option}\ignorespaces 
PCMK\_logfile
&
{\hyperref[\detokenize{cluster-options:text}]{\sphinxcrossref{\DUrole{std,std-ref}{text}}}}
&
/var/log/pacemaker/pacemaker.log
&
Unless set to \sphinxcode{\sphinxupquote{none}}, more detailed log messages will be sent to the
specified file (in addition to the system log, if enabled). These
messages may have extended information, and will include messages of info
severity. This log is of more use to developers and advanced system
administrators, and when reporting problems.
\\
\hline\phantomsection\label{\detokenize{local-options:pcmk-logfile-mode}}
\index{: node option!PCMK\_logfile\_mode}\index{PCMK\_logfile\_mode!: node option}\ignorespaces 
PCMK\_logfile\_mode
&
{\hyperref[\detokenize{cluster-options:text}]{\sphinxcrossref{\DUrole{std,std-ref}{text}}}}
&
0660
&
Pacemaker will set the permissions on the detail log to this value (see
\sphinxcode{\sphinxupquote{chmod(1)}}).
\\
\hline\phantomsection\label{\detokenize{local-options:pcmk-debug}}
\index{: node option!PCMK\_debug}\index{PCMK\_debug!: node option}\ignorespaces 
PCMK\_debug
&
{\hyperref[\detokenize{cluster-options:enumeration}]{\sphinxcrossref{\DUrole{std,std-ref}{enumeration}}}}
&
no
&
Whether to send debug severity messages to the detail log. This may be
set for all subsystems (\sphinxcode{\sphinxupquote{yes}} or \sphinxcode{\sphinxupquote{no}}) or for specific (comma-
separated) subsystems. Allowed subsystems are:
\begin{itemize}
\item {} 
\sphinxcode{\sphinxupquote{pacemakerd}}

\item {} 
\sphinxcode{\sphinxupquote{pacemaker-attrd}}

\item {} 
\sphinxcode{\sphinxupquote{pacemaker-based}}

\item {} 
\sphinxcode{\sphinxupquote{pacemaker-controld}}

\item {} 
\sphinxcode{\sphinxupquote{pacemaker-execd}}

\item {} 
\sphinxcode{\sphinxupquote{pacemaker-fenced}}

\item {} 
\sphinxcode{\sphinxupquote{pacemaker-schedulerd}}

\end{itemize}

Example: \sphinxcode{\sphinxupquote{PCMK\_debug="pacemakerd,pacemaker-execd"}}
\\
\hline\phantomsection\label{\detokenize{local-options:pcmk-stderr}}
\index{: node option!PCMK\_stderr}\index{PCMK\_stderr!: node option}\ignorespaces 
PCMK\_stderr
&
{\hyperref[\detokenize{cluster-options:boolean}]{\sphinxcrossref{\DUrole{std,std-ref}{boolean}}}}
&
no
&
\sphinxstyleemphasis{Advanced Use Only:} Whether to send daemon log messages to stderr. This
would be useful only during troubleshooting, when starting Pacemaker
manually on the command line.

Setting this option in the configuration file is pointless, since the
file is not read when starting Pacemaker manually. However, it can be set
directly as an environment variable on the command line.
\\
\hline\phantomsection\label{\detokenize{local-options:pcmk-trace-functions}}
\index{: node option!PCMK\_trace\_functions}\index{PCMK\_trace\_functions!: node option}\ignorespaces 
PCMK\_trace\_functions
&
{\hyperref[\detokenize{cluster-options:text}]{\sphinxcrossref{\DUrole{std,std-ref}{text}}}}
&&
\sphinxstyleemphasis{Advanced Use Only:} Send debug and trace severity messages from these
(comma-separated) source code functions to the detail log.

Example:
\sphinxcode{\sphinxupquote{PCMK\_trace\_functions="func1,func2"}}
\\
\hline\phantomsection\label{\detokenize{local-options:pcmk-trace-files}}
\index{: node option!PCMK\_trace\_files}\index{PCMK\_trace\_files!: node option}\ignorespaces 
PCMK\_trace\_files
&
{\hyperref[\detokenize{cluster-options:text}]{\sphinxcrossref{\DUrole{std,std-ref}{text}}}}
&&
\sphinxstyleemphasis{Advanced Use Only:} Send debug and trace severity messages from all
functions in these (comma-separated) source file names to the detail log.

Example: \sphinxcode{\sphinxupquote{PCMK\_trace\_files="file1.c,file2.c"}}
\\
\hline\phantomsection\label{\detokenize{local-options:pcmk-trace-formats}}
\index{: node option!PCMK\_trace\_formats}\index{PCMK\_trace\_formats!: node option}\ignorespaces 
PCMK\_trace\_formats
&
{\hyperref[\detokenize{cluster-options:text}]{\sphinxcrossref{\DUrole{std,std-ref}{text}}}}
&&
\sphinxstyleemphasis{Advanced Use Only:} Send trace severity messages that are generated by
these (comma-separated) format strings in the source code to the detail
log.

Example: \sphinxcode{\sphinxupquote{PCMK\_trace\_formats="Error: \%s (\%d)"}}
\\
\hline\phantomsection\label{\detokenize{local-options:pcmk-trace-tags}}
\index{: node option!PCMK\_trace\_tags}\index{PCMK\_trace\_tags!: node option}\ignorespaces 
PCMK\_trace\_tags
&
{\hyperref[\detokenize{cluster-options:text}]{\sphinxcrossref{\DUrole{std,std-ref}{text}}}}
&&
\sphinxstyleemphasis{Advanced Use Only:} Send debug and trace severity messages related to
these (comma-separated) resource IDs to the detail log.

Example: \sphinxcode{\sphinxupquote{PCMK\_trace\_tags="client-ip,dbfs"}}
\\
\hline\phantomsection\label{\detokenize{local-options:pcmk-blackbox}}
\index{: node option!PCMK\_blackbox}\index{PCMK\_blackbox!: node option}\ignorespaces 
PCMK\_blackbox
&
{\hyperref[\detokenize{cluster-options:enumeration}]{\sphinxcrossref{\DUrole{std,std-ref}{enumeration}}}}
&
no
&
\sphinxstyleemphasis{Advanced Use Only:} Enable blackbox logging globally (\sphinxcode{\sphinxupquote{yes}} or \sphinxcode{\sphinxupquote{no}})
or by subsystem. A blackbox contains a rolling buffer of all logs (of all
severities). Blackboxes are stored under \sphinxcode{\sphinxupquote{/var/lib/pacemaker/blackbox}} by default,
by default, and their contents can be viewed using the \sphinxcode{\sphinxupquote{qb-blackbox(8)}}
command.

The blackbox recorder can be enabled at start using this variable, or at
runtime by sending a Pacemaker subsystem daemon process a \sphinxcode{\sphinxupquote{SIGUSR1}} or
\sphinxcode{\sphinxupquote{SIGTRAP}} signal, and disabled by sending \sphinxcode{\sphinxupquote{SIGUSR2}} (see
\sphinxcode{\sphinxupquote{kill(1)}}). The blackbox will be written after a crash, assertion
failure, or \sphinxcode{\sphinxupquote{SIGTRAP}} signal.

See {\hyperref[\detokenize{local-options:pcmk-debug}]{\sphinxcrossref{\DUrole{std,std-ref}{PCMK\_debug}}}} for allowed subsystems.

Example:
\sphinxcode{\sphinxupquote{PCMK\_blackbox="pacemakerd,pacemaker-execd"}}
\\
\hline\phantomsection\label{\detokenize{local-options:pcmk-trace-blackbox}}
\index{: node option!PCMK\_trace\_blackbox}\index{PCMK\_trace\_blackbox!: node option}\ignorespaces 
PCMK\_trace\_blackbox
&
{\hyperref[\detokenize{cluster-options:enumeration}]{\sphinxcrossref{\DUrole{std,std-ref}{enumeration}}}}
&&
\sphinxstyleemphasis{Advanced Use Only:} Write a blackbox whenever the message at the
specified function and line is logged. Multiple entries may be comma-
separated.

Example: \sphinxcode{\sphinxupquote{PCMK\_trace\_blackbox="remote.c:144,remote.c:149"}}
\\
\hline\phantomsection\label{\detokenize{local-options:pcmk-node-start-state}}
\index{: node option!PCMK\_node\_start\_state}\index{PCMK\_node\_start\_state!: node option}\ignorespaces 
PCMK\_node\_start\_state
&
{\hyperref[\detokenize{cluster-options:enumeration}]{\sphinxcrossref{\DUrole{std,std-ref}{enumeration}}}}
&
default
&
By default, the local host will join the cluster in an online or standby
state when Pacemaker first starts depending on whether it was previously
put into standby mode. If this variable is set to \sphinxcode{\sphinxupquote{standby}} or
\sphinxcode{\sphinxupquote{online}}, it will force the local host to join in the specified state.
\\
\hline\phantomsection\label{\detokenize{local-options:pcmk-node-action-limit}}
\index{: node option!PCMK\_node\_action\_limit}\index{PCMK\_node\_action\_limit!: node option}\ignorespaces 
PCMK\_node\_action\_limit
&
{\hyperref[\detokenize{cluster-options:nonnegative-integer}]{\sphinxcrossref{\DUrole{std,std-ref}{nonnegative integer}}}}
&&
Specify the maximum number of jobs that can be scheduled on this node. If
set, this overrides the \sphinxcode{\sphinxupquote{node-action-limit}} cluster property for this
node.
\\
\hline\phantomsection\label{\detokenize{local-options:pcmk-shutdown-delay}}
\index{: node option!PCMK\_shutdown\_delay}\index{PCMK\_shutdown\_delay!: node option}\ignorespaces 
PCMK\_shutdown\_delay
&
{\hyperref[\detokenize{cluster-options:timeout}]{\sphinxcrossref{\DUrole{std,std-ref}{timeout}}}}
&&
Specify a delay before shutting down \sphinxcode{\sphinxupquote{pacemakerd}} after shutting down
all other Pacemaker daemons.
\\
\hline\phantomsection\label{\detokenize{local-options:pcmk-fail-fast}}
\index{: node option!PCMK\_fail\_fast}\index{PCMK\_fail\_fast!: node option}\ignorespaces 
PCMK\_fail\_fast
&
{\hyperref[\detokenize{cluster-options:boolean}]{\sphinxcrossref{\DUrole{std,std-ref}{boolean}}}}
&
no
&
By default, if a Pacemaker subsystem crashes, the main \sphinxcode{\sphinxupquote{pacemakerd}}
process will attempt to restart it. If this variable is set to \sphinxcode{\sphinxupquote{yes}},
\sphinxcode{\sphinxupquote{pacemakerd}} will panic the local host instead.
\\
\hline\phantomsection\label{\detokenize{local-options:pcmk-panic-action}}
\index{: node option!PCMK\_panic\_action}\index{PCMK\_panic\_action!: node option}\ignorespaces 
PCMK\_panic\_action
&
{\hyperref[\detokenize{cluster-options:enumeration}]{\sphinxcrossref{\DUrole{std,std-ref}{enumeration}}}}
&
reboot
&
Pacemaker will panic the local host under certain conditions. By default,
this means rebooting the host. This variable can change that behavior: if
\sphinxcode{\sphinxupquote{crash}}, trigger a kernel crash (useful if you want a kernel dump to
investigate); if \sphinxcode{\sphinxupquote{sync-reboot}} or \sphinxcode{\sphinxupquote{sync-crash}}, synchronize
filesystems before rebooting the host or triggering a kernel crash. The
sync values are more likely to preserve log messages, but with the risk
that the host may be left active if the synchronization hangs.
\\
\hline\phantomsection\label{\detokenize{local-options:pcmk-authkey-location}}
\index{: node option!PCMK\_authkey\_location}\index{PCMK\_authkey\_location!: node option}\ignorespaces 
PCMK\_authkey\_location
&
{\hyperref[\detokenize{cluster-options:text}]{\sphinxcrossref{\DUrole{std,std-ref}{text}}}}
&
/etc/pacemaker/authkey
&
Use the contents of this file as the authorization key to use with
Pacemaker Remote connections. This file must be readable by Pacemaker
daemons (that is, it must allow read permissions to either the
\sphinxcode{\sphinxupquote{hacluster}} user or the \sphinxcode{\sphinxupquote{haclient}} group), and its contents
must be identical on all nodes.
\\
\hline\phantomsection\label{\detokenize{local-options:pcmk-remote-address}}
\index{: node option!PCMK\_remote\_address}\index{PCMK\_remote\_address!: node option}\ignorespaces 
PCMK\_remote\_address
&
{\hyperref[\detokenize{cluster-options:text}]{\sphinxcrossref{\DUrole{std,std-ref}{text}}}}
&&
By default, if the Pacemaker Remote service is run on the local node, it
will listen for connections on all IP addresses. This may be set to one
address to listen on instead, as a resolvable hostname or as a numeric
IPv4 or IPv6 address. When resolving names or listening on all addresses,
IPv6 will be preferred if available. When listening on an IPv6 address,
IPv4 clients will be supported via IPv4-mapped IPv6 addresses.

Example: \sphinxcode{\sphinxupquote{PCMK\_remote\_address="192.0.2.1"}}
\\
\hline\phantomsection\label{\detokenize{local-options:pcmk-remote-port}}
\index{: node option!PCMK\_remote\_port}\index{PCMK\_remote\_port!: node option}\ignorespaces 
PCMK\_remote\_port
&
{\hyperref[\detokenize{cluster-options:port}]{\sphinxcrossref{\DUrole{std,std-ref}{port}}}}
&
3121
&
Use this TCP port number for Pacemaker Remote node connections. This
value must be the same on all nodes.
\\
\hline\phantomsection\label{\detokenize{local-options:pcmk-remote-pid1}}
\index{: node option!PCMK\_remote\_pid1}\index{PCMK\_remote\_pid1!: node option}\ignorespaces 
PCMK\_remote\_pid1
&
{\hyperref[\detokenize{cluster-options:enumeration}]{\sphinxcrossref{\DUrole{std,std-ref}{enumeration}}}}
&
default
&
\sphinxstyleemphasis{Advanced Use Only:} When a bundle resource’s \sphinxcode{\sphinxupquote{run-command}} option is
left to default, Pacemaker Remote runs as PID 1 in the bundle’s
containers. When it does so, it loads environment variables from the
container’s \sphinxcode{\sphinxupquote{/etc/pacemaker/pcmk-init.env}} and performs the PID 1 responsibility of
reaping dead subprocesses.

This option controls whether those actions are performed when Pacemaker
Remote is not running as PID 1. It is intended primarily for developer
testing but can be useful when \sphinxcode{\sphinxupquote{run-command}} is set to a separate,
custom PID 1 process that launches Pacemaker Remote.
\begin{itemize}
\item {} 
\sphinxcode{\sphinxupquote{full}}: Pacemaker Remote loads environment variables from
\sphinxcode{\sphinxupquote{/etc/pacemaker/pcmk-init.env}} and reaps dead subprocesses.

\item {} 
\sphinxcode{\sphinxupquote{vars}}: Pacemaker Remote loads environment variables from
\sphinxcode{\sphinxupquote{/etc/pacemaker/pcmk-init.env}} but does not reap dead subprocesses.

\item {} 
\sphinxcode{\sphinxupquote{default}}: Pacemaker Remote performs neither action.

\end{itemize}

If Pacemaker Remote is running as PID 1, this option is ignored, and the
behavior is the same as for \sphinxcode{\sphinxupquote{full}}.
\\
\hline\phantomsection\label{\detokenize{local-options:pcmk-tls-priorities}}
\index{: node option!PCMK\_tls\_priorities}\index{PCMK\_tls\_priorities!: node option}\ignorespaces 
PCMK\_tls\_priorities
&
{\hyperref[\detokenize{cluster-options:text}]{\sphinxcrossref{\DUrole{std,std-ref}{text}}}}
&
NORMAL
&
\sphinxstyleemphasis{Advanced Use Only:} These GnuTLS cipher priorities will be used for TLS
connections (whether for Pacemaker Remote connections or remote CIB
access, when enabled). See:
\begin{quote}

\sphinxurl{https://gnutls.org/manual/html\_node/Priority-Strings.html}
\end{quote}

Pacemaker will append \sphinxcode{\sphinxupquote{":+ANON-DH"}} for remote CIB access and
\sphinxcode{\sphinxupquote{":+DHE-PSK:+PSK"}} for Pacemaker Remote connections, as they are
required for the respective functionality.

Example:
\sphinxcode{\sphinxupquote{PCMK\_tls\_priorities="SECURE128:+SECURE192"}}
\\
\hline\phantomsection\label{\detokenize{local-options:pcmk-dh-min-bits}}
\index{: node option!PCMK\_dh\_min\_bits}\index{PCMK\_dh\_min\_bits!: node option}\ignorespaces 
PCMK\_dh\_min\_bits
&
{\hyperref[\detokenize{cluster-options:nonnegative-integer}]{\sphinxcrossref{\DUrole{std,std-ref}{nonnegative integer}}}}
&
0 (no minimum)
&
\sphinxstyleemphasis{Advanced Use Only:} Set a lower bound on the bit length of the prime
number generated for Diffie-Hellman parameters needed by TLS connections.
The default is no minimum.

The server (Pacemaker Remote daemon, or CIB manager configured to accept
remote clients) will use this value to provide a floor for the value
recommended by the GnuTLS library. The library will only accept a limited
number of specific values, which vary by library version, so setting
these is recommended only when required for compatibility with specific
client versions.

Clients (connecting cluster nodes or remote CIB commands) will require
that the server use a prime of at least this size. This is recommended
only when the value must be lowered in order for the client’s GnuTLS
library to accept a connection to an older server.
\\
\hline\phantomsection\label{\detokenize{local-options:pcmk-dh-max-bits}}
\index{: node option!PCMK\_dh\_max\_bits}\index{PCMK\_dh\_max\_bits!: node option}\ignorespaces 
PCMK\_dh\_max\_bits
&
{\hyperref[\detokenize{cluster-options:nonnegative-integer}]{\sphinxcrossref{\DUrole{std,std-ref}{nonnegative integer}}}}
&
0 (no maximum)
&
\sphinxstyleemphasis{Advanced Use Only:} Set an upper bound on the bit length of the prime
number generated for Diffie-Hellman parameters needed by TLS connections.
The default is no maximum.

The server (Pacemaker Remote daemon, or CIB manager configured to accept
remote clients) will use this value to provide a ceiling for the value
recommended by the GnuTLS library. The library will only accept a limited
number of specific values, which vary by library version, so setting
these is recommended only when required for compatibility with specific
client versions.

Clients do not use \sphinxcode{\sphinxupquote{PCMK\_dh\_max\_bits}}.
\\
\hline\phantomsection\label{\detokenize{local-options:pcmk-ipc-type}}
\index{: node option!PCMK\_ipc\_type}\index{PCMK\_ipc\_type!: node option}\ignorespaces 
PCMK\_ipc\_type
&
{\hyperref[\detokenize{cluster-options:enumeration}]{\sphinxcrossref{\DUrole{std,std-ref}{enumeration}}}}
&
shared-mem
&
\sphinxstyleemphasis{Advanced Use Only:} Force use of a particular IPC method. Allowed values:
\begin{itemize}
\item {} 
\sphinxcode{\sphinxupquote{shared-mem}}

\item {} 
\sphinxcode{\sphinxupquote{socket}}

\item {} 
\sphinxcode{\sphinxupquote{posix}}

\item {} 
\sphinxcode{\sphinxupquote{sysv}}

\end{itemize}
\\
\hline\phantomsection\label{\detokenize{local-options:pcmk-ipc-buffer}}
\index{: node option!PCMK\_ipc\_buffer}\index{PCMK\_ipc\_buffer!: node option}\ignorespaces 
PCMK\_ipc\_buffer
&
{\hyperref[\detokenize{cluster-options:nonnegative-integer}]{\sphinxcrossref{\DUrole{std,std-ref}{nonnegative integer}}}}
&
131072
&
\sphinxstyleemphasis{Advanced Use Only:} Specify an IPC buffer size in bytes. This can be
useful when connecting to large clusters that result in messages
exceeding the default size (which will also result in log messages
referencing this variable).
\\
\hline\phantomsection\label{\detokenize{local-options:pcmk-cluster-type}}
\index{: node option!PCMK\_cluster\_type}\index{PCMK\_cluster\_type!: node option}\ignorespaces 
PCMK\_cluster\_type
&
{\hyperref[\detokenize{cluster-options:enumeration}]{\sphinxcrossref{\DUrole{std,std-ref}{enumeration}}}}
&
corosync
&
\sphinxstyleemphasis{Advanced Use Only:} Specify the cluster layer to be used. If unset,
Pacemaker will detect and use a supported cluster layer, if available.
Currently, \sphinxcode{\sphinxupquote{"corosync"}} is the only supported cluster layer. If
multiple layers are supported in the future, this will allow overriding
Pacemaker’s automatic detection to select a specific one.
\\
\hline\phantomsection\label{\detokenize{local-options:pcmk-schema-directory}}
\index{: node option!PCMK\_schema\_directory}\index{PCMK\_schema\_directory!: node option}\ignorespaces 
PCMK\_schema\_directory
&
{\hyperref[\detokenize{cluster-options:text}]{\sphinxcrossref{\DUrole{std,std-ref}{text}}}}
&
/usr/share/pacemaker
&
\sphinxstyleemphasis{Advanced Use Only:} Specify an alternate location for RNG schemas and
XSL transforms.
\\
\hline\phantomsection\label{\detokenize{local-options:pcmk-valgrind-enabled}}
\index{: node option!PCMK\_valgrind\_enabled}\index{PCMK\_valgrind\_enabled!: node option}\ignorespaces 
PCMK\_valgrind\_enabled
&
{\hyperref[\detokenize{cluster-options:enumeration}]{\sphinxcrossref{\DUrole{std,std-ref}{enumeration}}}}
&
no
&
\sphinxstyleemphasis{Advanced Use Only:} Whether subsystem daemons should be run under
\sphinxcode{\sphinxupquote{valgrind}}. Allowed values are the same as for \sphinxcode{\sphinxupquote{PCMK\_debug}}.
\\
\hline\phantomsection\label{\detokenize{local-options:pcmk-callgrind-enabled}}
\index{: node option!PCMK\_callgrind\_enabled}\index{PCMK\_callgrind\_enabled!: node option}\ignorespaces 
PCMK\_callgrind\_enabled
&
{\hyperref[\detokenize{cluster-options:enumeration}]{\sphinxcrossref{\DUrole{std,std-ref}{enumeration}}}}
&
no
&
\sphinxstyleemphasis{Advanced Use Only:} Whether subsystem daemons should be run under
\sphinxcode{\sphinxupquote{valgrind}} with the \sphinxcode{\sphinxupquote{callgrind}} tool enabled. Allowed values are the
same as for \sphinxcode{\sphinxupquote{PCMK\_debug}}.
\\
\hline\phantomsection\label{\detokenize{local-options:valgrind-opts}}
\index{: node option!VALGRIND\_OPTS}\index{VALGRIND\_OPTS!: node option}\ignorespaces 
VALGRIND\_OPTS
&
{\hyperref[\detokenize{cluster-options:text}]{\sphinxcrossref{\DUrole{std,std-ref}{text}}}}
&&
\sphinxstyleemphasis{Advanced Use Only:} Pass these options to valgrind, when enabled (see
\sphinxcode{\sphinxupquote{valgrind(1)}}). \sphinxcode{\sphinxupquote{"-{-}vgdb=no"}} should usually be specified because
\sphinxcode{\sphinxupquote{pacemaker-execd}} can lower privileges when executing commands, which
would otherwise leave a bunch of unremovable files in \sphinxcode{\sphinxupquote{/tmp}}.
\\
\hline
\end{longtable}\sphinxatlongtableend\end{savenotes}


\section{Cluster-Wide Configuration}
\label{\detokenize{cluster-options:cluster-wide-configuration}}\label{\detokenize{cluster-options::doc}}
\index{XML element!cib}\index{cib!XML element}\index{XML element!configuration}\index{configuration!XML element}\ignorespaces 

\subsection{Configuration Layout}
\label{\detokenize{cluster-options:configuration-layout}}\label{\detokenize{cluster-options:index-0}}
The cluster is defined by the Cluster Information Base (CIB), which uses XML
notation. The simplest CIB, an empty one, looks like this:

\begin{sphinxShadowBox}
\sphinxstyletopictitle{An empty configuration}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}cib} \PYG{n+na}{crm\PYGZus{}feature\PYGZus{}set=}\PYG{l+s}{\PYGZdq{}3.6.0\PYGZdq{}} \PYG{n+na}{validate\PYGZhy{}with=}\PYG{l+s}{\PYGZdq{}pacemaker\PYGZhy{}3.5\PYGZdq{}} \PYG{n+na}{epoch=}\PYG{l+s}{\PYGZdq{}1\PYGZdq{}} \PYG{n+na}{num\PYGZus{}updates=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{admin\PYGZus{}epoch=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}configuration}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}crm\PYGZus{}config}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nodes}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}resources}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}constraints}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}/configuration\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}status}\PYG{n+nt}{/\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/cib\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

The empty configuration above contains the major sections that make up a CIB:
\begin{itemize}
\item {} 
\sphinxcode{\sphinxupquote{cib}}: The entire CIB is enclosed with a \sphinxcode{\sphinxupquote{cib}} element. Certain
fundamental settings are defined as attributes of this element.
\begin{itemize}
\item {} 
\sphinxcode{\sphinxupquote{configuration}}: This section \textendash{} the primary focus of this document \textendash{}
contains traditional configuration information such as what resources the
cluster serves and the relationships among them.
\begin{itemize}
\item {} 
\sphinxcode{\sphinxupquote{crm\_config}}: cluster-wide configuration options

\item {} 
\sphinxcode{\sphinxupquote{nodes}}: the machines that host the cluster

\item {} 
\sphinxcode{\sphinxupquote{resources}}: the services run by the cluster

\item {} 
\sphinxcode{\sphinxupquote{constraints}}: indications of how resources should be placed

\end{itemize}

\item {} 
\sphinxcode{\sphinxupquote{status}}: This section contains the history of each resource on each
node. Based on this data, the cluster can construct the complete current
state of the cluster. The authoritative source for this section is the
local executor (pacemaker-execd process) on each cluster node, and the
cluster will occasionally repopulate the entire section. For this reason,
it is never written to disk, and administrators are advised against
modifying it in any way.

\end{itemize}

\end{itemize}

In this document, configuration settings will be described as properties or
options based on how they are defined in the CIB:
\begin{itemize}
\item {} 
Properties are XML attributes of an XML element.

\item {} 
Options are name-value pairs expressed as \sphinxcode{\sphinxupquote{nvpair}} child elements of an XML
element.

\end{itemize}

Normally, you will use command-line tools that abstract the XML, so the
distinction will be unimportant; both properties and options are cluster
settings you can tweak.


\subsection{Configuration Value Types}
\label{\detokenize{cluster-options:configuration-value-types}}
Throughout this document, configuration values will be designated as having one
of the following types:


\begin{savenotes}\sphinxatlongtablestart\begin{longtable}{|\X{1}{4}|\X{3}{4}|}
\caption{\sphinxstylestrong{Configuration Value Types}\strut}\label{\detokenize{cluster-options:id2}}\\*[\sphinxlongtablecapskipadjust]
\hline
\sphinxstyletheadfamily 
Type
&\sphinxstyletheadfamily 
Description
\\
\hline
\endfirsthead

\multicolumn{2}{c}%
{\makebox[0pt]{\sphinxtablecontinued{\tablename\ \thetable{} -- continued from previous page}}}\\
\hline
\sphinxstyletheadfamily 
Type
&\sphinxstyletheadfamily 
Description
\\
\hline
\endhead

\hline
\multicolumn{2}{r}{\makebox[0pt][r]{\sphinxtablecontinued{Continued on next page}}}\\
\endfoot

\endlastfoot
\phantomsection\label{\detokenize{cluster-options:boolean}}
\index{type!boolean}\index{boolean!type}\ignorespaces 
boolean
&
Case-insensitive text value where \sphinxcode{\sphinxupquote{1}}, \sphinxcode{\sphinxupquote{yes}}, \sphinxcode{\sphinxupquote{y}}, \sphinxcode{\sphinxupquote{on}},
and \sphinxcode{\sphinxupquote{true}} evaluate as true and \sphinxcode{\sphinxupquote{0}}, \sphinxcode{\sphinxupquote{no}}, \sphinxcode{\sphinxupquote{n}}, \sphinxcode{\sphinxupquote{off}},
\sphinxcode{\sphinxupquote{false}}, and unset evaluate as false
\\
\hline\phantomsection\label{\detokenize{cluster-options:date-time}}
\index{type!date/time}\index{date/time!type}\ignorespaces 
date/time
&
Textual timestamp like \sphinxcode{\sphinxupquote{Sat Dec 21 11:47:45 2013}}
\\
\hline\phantomsection\label{\detokenize{cluster-options:duration}}
\index{type!duration}\index{duration!type}\ignorespaces 
duration
&
A time duration, specified either like a {\hyperref[\detokenize{cluster-options:timeout}]{\sphinxcrossref{\DUrole{std,std-ref}{timeout}}}} or an
\sphinxhref{https://en.wikipedia.org/wiki/ISO\_8601\#Durations}{ISO 8601 duration}.
A duration may be up to approximately 49 days but is intended for much
smaller time periods.
\\
\hline\phantomsection\label{\detokenize{cluster-options:enumeration}}
\index{type!enumeration}\index{enumeration!type}\ignorespaces 
enumeration
&
Text that must be one of a set of defined values (which will be listed
in the description)
\\
\hline\phantomsection\label{\detokenize{cluster-options:integer}}
\index{type!integer}\index{integer!type}\ignorespaces 
integer
&
32-bit signed integer value (-2,147,483,648 to 2,147,483,647)
\\
\hline\phantomsection\label{\detokenize{cluster-options:nonnegative-integer}}
\index{type!nonnegative integer}\index{nonnegative integer!type}\ignorespaces 
nonnegative integer
&
32-bit nonnegative integer value (0 to 2,147,483,647)
\\
\hline\phantomsection\label{\detokenize{cluster-options:port}}
\index{type!port}\index{port!type}\ignorespaces 
port
&
Integer TCP port number (0 to 65535)
\\
\hline\phantomsection\label{\detokenize{cluster-options:score}}
\index{type!score}\index{score!type}\ignorespaces 
score
&
A Pacemaker score can be an integer between -1,000,000 and 1,000,000, or
a string alias: \sphinxcode{\sphinxupquote{INFINITY}} or \sphinxcode{\sphinxupquote{+INFINITY}} is equivalent to
1,000,000, \sphinxcode{\sphinxupquote{-INFINITY}} is equivalent to -1,000,000, and \sphinxcode{\sphinxupquote{red}},
\sphinxcode{\sphinxupquote{yellow}}, and \sphinxcode{\sphinxupquote{green}} are equivalent to integers as described in
{\hyperref[\detokenize{nodes:node-health}]{\sphinxcrossref{\DUrole{std,std-ref}{Tracking Node Health}}}}.
\\
\hline\phantomsection\label{\detokenize{cluster-options:text}}
\index{type!text}\index{text!type}\ignorespaces 
text
&
A text string
\\
\hline\phantomsection\label{\detokenize{cluster-options:timeout}}
\index{type!timeout}\index{timeout!type}\ignorespaces 
timeout
&
A time duration, specified as a bare number (in which case it is
considered to be in seconds) or a number with a unit (\sphinxcode{\sphinxupquote{ms}} or \sphinxcode{\sphinxupquote{msec}}
for milliseconds, \sphinxcode{\sphinxupquote{us}} or \sphinxcode{\sphinxupquote{usec}} for microseconds, \sphinxcode{\sphinxupquote{s}} or \sphinxcode{\sphinxupquote{sec}}
for seconds, \sphinxcode{\sphinxupquote{m}} or \sphinxcode{\sphinxupquote{min}} for minutes, \sphinxcode{\sphinxupquote{h}} or \sphinxcode{\sphinxupquote{hr}} for hours)
optionally with whitespace before and/or after the number.
\\
\hline\phantomsection\label{\detokenize{cluster-options:version}}
\index{type!version}\index{version!type}\ignorespaces 
version
&
Version number (any combination of alphanumeric characters, dots, and
dashes, starting with a number).
\\
\hline
\end{longtable}\sphinxatlongtableend\end{savenotes}


\subsubsection{Scores}
\label{\detokenize{cluster-options:scores}}
Scores are integral to how Pacemaker works. Practically everything from moving
a resource to deciding which resource to stop in a degraded cluster is achieved
by manipulating scores in some way.

Scores are calculated per resource and node. Any node with a negative score for
a resource can’t run that resource. The cluster places a resource on the node
with the highest score for it.

Score addition and subtraction follow these rules:
\begin{itemize}
\item {} 
Any value (including \sphinxcode{\sphinxupquote{INFINITY}}) - \sphinxcode{\sphinxupquote{INFINITY}} = \sphinxcode{\sphinxupquote{-INFINITY}}

\item {} 
\sphinxcode{\sphinxupquote{INFINITY}} + any value other than \sphinxcode{\sphinxupquote{-INFINITY}} = \sphinxcode{\sphinxupquote{INFINITY}}

\end{itemize}

\begin{sphinxadmonition}{note}{Note:}
What if you want to use a score higher than 1,000,000? Typically this possibility
arises when someone wants to base the score on some external metric that might
go above 1,000,000.

The short answer is you can’t.

The long answer is it is sometimes possible work around this limitation
creatively. You may be able to set the score to some computed value based on
the external metric rather than use the metric directly. For nodes, you can
store the metric as a node attribute, and query the attribute when computing
the score (possibly as part of a custom resource agent).
\end{sphinxadmonition}


\subsection{CIB Properties}
\label{\detokenize{cluster-options:cib-properties}}
Certain settings are defined by CIB properties (that is, attributes of the
\sphinxcode{\sphinxupquote{cib}} tag) rather than with the rest of the cluster configuration in the
\sphinxcode{\sphinxupquote{configuration}} section.

The reason is simply a matter of parsing. These options are used by the
configuration database which is, by design, mostly ignorant of the content it
holds. So the decision was made to place them in an easy-to-find location.


\begin{savenotes}\sphinxatlongtablestart\begin{longtable}{|\X{2}{11}|\X{2}{11}|\X{2}{11}|\X{5}{11}|}
\caption{\sphinxstylestrong{CIB Properties}\strut}\label{\detokenize{cluster-options:id3}}\\*[\sphinxlongtablecapskipadjust]
\hline
\sphinxstyletheadfamily 
Name
&\sphinxstyletheadfamily 
Type
&\sphinxstyletheadfamily 
Default
&\sphinxstyletheadfamily 
Description
\\
\hline
\endfirsthead

\multicolumn{4}{c}%
{\makebox[0pt]{\sphinxtablecontinued{\tablename\ \thetable{} -- continued from previous page}}}\\
\hline
\sphinxstyletheadfamily 
Name
&\sphinxstyletheadfamily 
Type
&\sphinxstyletheadfamily 
Default
&\sphinxstyletheadfamily 
Description
\\
\hline
\endhead

\hline
\multicolumn{4}{r}{\makebox[0pt][r]{\sphinxtablecontinued{Continued on next page}}}\\
\endfoot

\endlastfoot
\phantomsection\label{\detokenize{cluster-options:admin-epoch}}
\index{admin\_epoch!cib}\index{cib!admin\_epoch}\ignorespaces 
admin\_epoch
&
{\hyperref[\detokenize{cluster-options:nonnegative-integer}]{\sphinxcrossref{\DUrole{std,std-ref}{nonnegative integer}}}}
&
0
&
When a node joins the cluster, the cluster asks the node with the
highest (\sphinxcode{\sphinxupquote{admin\_epoch}}, \sphinxcode{\sphinxupquote{epoch}}, \sphinxcode{\sphinxupquote{num\_updates}}) tuple to replace
the configuration on all the nodes \textendash{} which makes setting them correctly
very important. \sphinxcode{\sphinxupquote{admin\_epoch}} is never modified by the cluster; you
can use this to make the configurations on any inactive nodes obsolete.
\\
\hline\phantomsection\label{\detokenize{cluster-options:epoch}}
\index{epoch!cib}\index{cib!epoch}\ignorespaces 
epoch
&
{\hyperref[\detokenize{cluster-options:nonnegative-integer}]{\sphinxcrossref{\DUrole{std,std-ref}{nonnegative integer}}}}
&
0
&
The cluster increments this every time the CIB’s configuration section
is updated.
\\
\hline\phantomsection\label{\detokenize{cluster-options:num-updates}}
\index{num\_updates!cib}\index{cib!num\_updates}\ignorespaces 
num\_updates
&
{\hyperref[\detokenize{cluster-options:nonnegative-integer}]{\sphinxcrossref{\DUrole{std,std-ref}{nonnegative integer}}}}
&
0
&
The cluster increments this every time the CIB’s configuration or status
sections are updated, and resets it to 0 when epoch changes.
\\
\hline\phantomsection\label{\detokenize{cluster-options:validate-with}}
\index{validate-with!cib}\index{cib!validate-with}\ignorespaces 
validate-with
&
{\hyperref[\detokenize{cluster-options:enumeration}]{\sphinxcrossref{\DUrole{std,std-ref}{enumeration}}}}
&&
Determines the type of XML validation that will be done on the
configuration. Allowed values are \sphinxcode{\sphinxupquote{none}} (in which case the cluster
will not require that updates conform to expected syntax) and the base
names of schema files installed on the local machine (for example,
“pacemaker-3.9”)
\\
\hline\phantomsection\label{\detokenize{cluster-options:remote-tls-port}}
\index{remote-tls-port!cib}\index{cib!remote-tls-port}\ignorespaces 
remote-tls-port
&
{\hyperref[\detokenize{cluster-options:port}]{\sphinxcrossref{\DUrole{std,std-ref}{port}}}}
&&
If set, the CIB manager will listen for anonymously encrypted remote
connections on this port, to allow CIB administration from hosts not in
the cluster. No key is used, so this should be used only on a protected
network where man-in-the-middle attacks can be avoided.
\\
\hline\phantomsection\label{\detokenize{cluster-options:remote-clear-port}}
\index{remote-clear-port!cib}\index{cib!remote-clear-port}\ignorespaces 
remote-clear-port
&
{\hyperref[\detokenize{cluster-options:port}]{\sphinxcrossref{\DUrole{std,std-ref}{port}}}}
&&
If set to a TCP port number, the CIB manager will listen for remote
connections on this port, to allow for CIB administration from hosts not
in the cluster. No encryption is used, so this should be used only on a
protected network.
\\
\hline\phantomsection\label{\detokenize{cluster-options:cib-last-written}}
\index{cib-last-written!cib}\index{cib!cib-last-written}\ignorespaces 
cib-last-written
&
{\hyperref[\detokenize{cluster-options:date-time}]{\sphinxcrossref{\DUrole{std,std-ref}{date/time}}}}
&&
Indicates when the configuration was last written to disk. Maintained by
the cluster; for informational purposes only.
\\
\hline\phantomsection\label{\detokenize{cluster-options:have-quorum}}
\index{have-quorum!cib}\index{cib!have-quorum}\ignorespaces 
have-quorum
&
{\hyperref[\detokenize{cluster-options:boolean}]{\sphinxcrossref{\DUrole{std,std-ref}{boolean}}}}
&&
Indicates whether the cluster has quorum. If false, the cluster’s
response is determined by \sphinxcode{\sphinxupquote{no-quorum-policy}} (see below). Maintained
by the cluster.
\\
\hline\phantomsection\label{\detokenize{cluster-options:dc-uuid}}
\index{dc-uuid!cib}\index{cib!dc-uuid}\ignorespaces 
dc-uuid
&
{\hyperref[\detokenize{cluster-options:text}]{\sphinxcrossref{\DUrole{std,std-ref}{text}}}}
&&
Node ID of the cluster’s current designated controller (DC). Used and
maintained by the cluster.
\\
\hline
\end{longtable}\sphinxatlongtableend\end{savenotes}


\subsection{Cluster Options}
\label{\detokenize{cluster-options:cluster-options}}\label{\detokenize{cluster-options:id1}}
Cluster options, as you might expect, control how the cluster behaves when
confronted with various situations.

They are grouped into sets within the \sphinxcode{\sphinxupquote{crm\_config}} section. In advanced
configurations, there may be more than one set. (This will be described later
in the chapter on {\hyperref[\detokenize{rules:rules}]{\sphinxcrossref{\DUrole{std,std-ref}{Rules}}}} where we will show how to have the cluster use
different sets of options during working hours than during weekends.) For now,
we will describe the simple case where each option is present at most once.

You can obtain an up-to-date list of cluster options, including their default
values, by running the \sphinxcode{\sphinxupquote{man pacemaker-schedulerd}} and
\sphinxcode{\sphinxupquote{man pacemaker-controld}} commands.


\begin{savenotes}\sphinxatlongtablestart\begin{longtable}{|\X{2}{11}|\X{2}{11}|\X{2}{11}|\X{5}{11}|}
\caption{\sphinxstylestrong{Cluster Options}\strut}\label{\detokenize{cluster-options:id4}}\\*[\sphinxlongtablecapskipadjust]
\hline
\sphinxstyletheadfamily 
Name
&\sphinxstyletheadfamily 
Type
&\sphinxstyletheadfamily 
Default
&\sphinxstyletheadfamily 
Description
\\
\hline
\endfirsthead

\multicolumn{4}{c}%
{\makebox[0pt]{\sphinxtablecontinued{\tablename\ \thetable{} -- continued from previous page}}}\\
\hline
\sphinxstyletheadfamily 
Name
&\sphinxstyletheadfamily 
Type
&\sphinxstyletheadfamily 
Default
&\sphinxstyletheadfamily 
Description
\\
\hline
\endhead

\hline
\multicolumn{4}{r}{\makebox[0pt][r]{\sphinxtablecontinued{Continued on next page}}}\\
\endfoot

\endlastfoot
\phantomsection\label{\detokenize{cluster-options:cluster-name}}
\index{cluster option!cluster-name}\index{cluster-name!cluster option}\ignorespaces 
cluster-name
&
{\hyperref[\detokenize{cluster-options:text}]{\sphinxcrossref{\DUrole{std,std-ref}{text}}}}
&&
An (optional) name for the cluster as a whole. This is mostly for users’
convenience for use as desired in administration, but can be used in the
Pacemaker configuration in {\hyperref[\detokenize{rules:rules}]{\sphinxcrossref{\DUrole{std,std-ref}{Rules}}}} (as the \sphinxcode{\sphinxupquote{\#cluster-name}}
{\hyperref[\detokenize{rules:node-attribute-expressions-special}]{\sphinxcrossref{\DUrole{std,std-ref}{node attribute}}}}). It may also
be used by higher-level tools when displaying cluster information, and
by certain resource agents (for example, the \sphinxcode{\sphinxupquote{ocf:heartbeat:GFS2}}
agent stores the cluster name in filesystem meta-data).
\\
\hline\phantomsection\label{\detokenize{cluster-options:dc-version}}
\index{cluster option!dc-version}\index{dc-version!cluster option}\ignorespaces 
dc-version
&
{\hyperref[\detokenize{cluster-options:version}]{\sphinxcrossref{\DUrole{std,std-ref}{version}}}}
&
\sphinxstyleemphasis{detected}
&
Version of Pacemaker on the cluster’s designated controller (DC).
Maintained by the cluster, and intended for diagnostic purposes.
\\
\hline\phantomsection\label{\detokenize{cluster-options:cluster-infrastructure}}
\index{cluster option!cluster-infrastructure}\index{cluster-infrastructure!cluster option}\ignorespaces 
cluster-infrastructure
&
{\hyperref[\detokenize{cluster-options:text}]{\sphinxcrossref{\DUrole{std,std-ref}{text}}}}
&
\sphinxstyleemphasis{detected}
&
The messaging layer with which Pacemaker is currently running.
Maintained by the cluster, and intended for informational and diagnostic
purposes.
\\
\hline\phantomsection\label{\detokenize{cluster-options:no-quorum-policy}}
\index{cluster option!no-quorum-policy}\index{no-quorum-policy!cluster option}\ignorespaces 
no-quorum-policy
&
{\hyperref[\detokenize{cluster-options:enumeration}]{\sphinxcrossref{\DUrole{std,std-ref}{enumeration}}}}
&
stop
&
What to do when the cluster does not have quorum. Allowed values:
\begin{itemize}
\item {} 
\sphinxcode{\sphinxupquote{ignore:}} continue all resource management

\item {} 
\sphinxcode{\sphinxupquote{freeze:}} continue resource management, but don’t recover resources
from nodes not in the affected partition

\item {} 
\sphinxcode{\sphinxupquote{stop:}} stop all resources in the affected cluster partition

\item {} 
\sphinxcode{\sphinxupquote{demote:}} demote promotable resources and stop all other resources
in the affected cluster partition \sphinxstyleemphasis{(since 2.0.5)}

\item {} 
\sphinxcode{\sphinxupquote{suicide:}} fence all nodes in the affected cluster partition

\end{itemize}
\\
\hline\phantomsection\label{\detokenize{cluster-options:batch-limit}}
\index{cluster option!batch-limit}\index{batch-limit!cluster option}\ignorespaces 
batch-limit
&
{\hyperref[\detokenize{cluster-options:integer}]{\sphinxcrossref{\DUrole{std,std-ref}{integer}}}}
&
0
&
The maximum number of actions that the cluster may execute in parallel
across all nodes. The ideal value will depend on the speed and load
of your network and cluster nodes. If zero, the cluster will impose a
dynamically calculated limit only when any node has high load. If -1,
the cluster will not impose any limit.
\\
\hline\phantomsection\label{\detokenize{cluster-options:migration-limit}}
\index{cluster option!migration-limit}\index{migration-limit!cluster option}\ignorespaces 
migration-limit
&
{\hyperref[\detokenize{cluster-options:integer}]{\sphinxcrossref{\DUrole{std,std-ref}{integer}}}}
&
-1
&
The number of {\hyperref[\detokenize{operations:live-migration}]{\sphinxcrossref{\DUrole{std,std-ref}{live migration}}}} actions that the
cluster is allowed to execute in parallel on a node. A value of -1 means
unlimited.
\\
\hline\phantomsection\label{\detokenize{cluster-options:symmetric-cluster}}
\index{cluster option!symmetric-cluster}\index{symmetric-cluster!cluster option}\ignorespaces 
symmetric-cluster
&
{\hyperref[\detokenize{cluster-options:boolean}]{\sphinxcrossref{\DUrole{std,std-ref}{boolean}}}}
&
true
&
If true, resources can run on any node by default. If false, a resource
is allowed to run on a node only if a
{\hyperref[\detokenize{constraints:location-constraint}]{\sphinxcrossref{\DUrole{std,std-ref}{location constraint}}}} enables it.
\\
\hline\phantomsection\label{\detokenize{cluster-options:stop-all-resources}}
\index{cluster option!stop-all-resources}\index{stop-all-resources!cluster option}\ignorespaces 
stop-all-resources
&
{\hyperref[\detokenize{cluster-options:boolean}]{\sphinxcrossref{\DUrole{std,std-ref}{boolean}}}}
&
false
&
Whether all resources should be disallowed from running (can be useful
during maintenance or troubleshooting)
\\
\hline\phantomsection\label{\detokenize{cluster-options:stop-orphan-resources}}
\index{cluster option!stop-orphan-resources}\index{stop-orphan-resources!cluster option}\ignorespaces 
stop-orphan-resources
&
{\hyperref[\detokenize{cluster-options:boolean}]{\sphinxcrossref{\DUrole{std,std-ref}{boolean}}}}
&
true
&
Whether resources that have been deleted from the configuration should
be stopped. This value takes precedence over
{\hyperref[\detokenize{resources:is-managed}]{\sphinxcrossref{\DUrole{std,std-ref}{is-managed}}}} (that is, even unmanaged resources will
be stopped when orphaned if this value is \sphinxcode{\sphinxupquote{true}}).
\\
\hline\phantomsection\label{\detokenize{cluster-options:stop-orphan-actions}}
\index{cluster option!stop-orphan-actions}\index{stop-orphan-actions!cluster option}\ignorespaces 
stop-orphan-actions
&
{\hyperref[\detokenize{cluster-options:boolean}]{\sphinxcrossref{\DUrole{std,std-ref}{boolean}}}}
&
true
&
Whether recurring {\hyperref[\detokenize{operations:operation}]{\sphinxcrossref{\DUrole{std,std-ref}{operations}}}} that have been deleted
from the configuration should be cancelled
\\
\hline\phantomsection\label{\detokenize{cluster-options:start-failure-is-fatal}}
\index{cluster option!start-failure-is-fatal}\index{start-failure-is-fatal!cluster option}\ignorespaces 
start-failure-is-fatal
&
{\hyperref[\detokenize{cluster-options:boolean}]{\sphinxcrossref{\DUrole{std,std-ref}{boolean}}}}
&
true
&
Whether a failure to start a resource on a particular node prevents
further start attempts on that node. If \sphinxcode{\sphinxupquote{false}}, the cluster will
decide whether the node is still eligible based on the resource’s
current failure count and \sphinxcode{\sphinxupquote{migration-threshold}}.
\\
\hline\phantomsection\label{\detokenize{cluster-options:enable-startup-probes}}
\index{cluster option!enable-startup-probes}\index{enable-startup-probes!cluster option}\ignorespaces 
enable-startup-probes
&
{\hyperref[\detokenize{cluster-options:boolean}]{\sphinxcrossref{\DUrole{std,std-ref}{boolean}}}}
&
true
&
Whether the cluster should check the pre-existing state of resources
when the cluster starts
\\
\hline\phantomsection\label{\detokenize{cluster-options:maintenance-mode}}
\index{cluster option!maintenance-mode}\index{maintenance-mode!cluster option}\ignorespaces 
maintenance-mode
&
{\hyperref[\detokenize{cluster-options:boolean}]{\sphinxcrossref{\DUrole{std,std-ref}{boolean}}}}
&
false
&
If true, the cluster will not start or stop any resource in the cluster,
and any recurring operations (expect those specifying \sphinxcode{\sphinxupquote{role}} as
\sphinxcode{\sphinxupquote{Stopped}}) will be paused. If true, this overrides the
{\hyperref[\detokenize{nodes:node-maintenance}]{\sphinxcrossref{\DUrole{std,std-ref}{maintenance}}}} node attribute,
{\hyperref[\detokenize{resources:is-managed}]{\sphinxcrossref{\DUrole{std,std-ref}{is-managed}}}} and {\hyperref[\detokenize{resources:rsc-maintenance}]{\sphinxcrossref{\DUrole{std,std-ref}{maintenance}}}}
resource meta-attributes, and {\hyperref[\detokenize{operations:op-enabled}]{\sphinxcrossref{\DUrole{std,std-ref}{enabled}}}} operation
meta-attribute.
\\
\hline\phantomsection\label{\detokenize{cluster-options:stonith-enabled}}
\index{cluster option!stonith-enabled}\index{stonith-enabled!cluster option}\ignorespaces 
stonith-enabled
&
{\hyperref[\detokenize{cluster-options:boolean}]{\sphinxcrossref{\DUrole{std,std-ref}{boolean}}}}
&
true
&
Whether the cluster is allowed to fence nodes (for example, failed nodes
and nodes with resources that can’t be stopped).

If true, at least one fence device must be configured before resources
are allowed to run.

If false, unresponsive nodes are immediately assumed to be running no
resources, and resource recovery on online nodes starts without any
further protection (which can mean \sphinxstyleemphasis{data loss} if the unresponsive node
still accesses shared storage, for example). See also the
{\hyperref[\detokenize{resources:requires}]{\sphinxcrossref{\DUrole{std,std-ref}{requires}}}} resource meta-attribute.
\\
\hline\phantomsection\label{\detokenize{cluster-options:stonith-action}}
\index{cluster option!stonith-action}\index{stonith-action!cluster option}\ignorespaces 
stonith-action
&
{\hyperref[\detokenize{cluster-options:enumeration}]{\sphinxcrossref{\DUrole{std,std-ref}{enumeration}}}}
&
reboot
&
Action the cluster should send to the fence agent when a node must be
fenced. Allowed values are \sphinxcode{\sphinxupquote{reboot}}, \sphinxcode{\sphinxupquote{off}}, and (for legacy agents
only) \sphinxcode{\sphinxupquote{poweroff}}.
\\
\hline\phantomsection\label{\detokenize{cluster-options:stonith-timeout}}
\index{cluster option!stonith-timeout}\index{stonith-timeout!cluster option}\ignorespaces 
stonith-timeout
&
{\hyperref[\detokenize{cluster-options:duration}]{\sphinxcrossref{\DUrole{std,std-ref}{duration}}}}
&
60s
&
How long to wait for \sphinxcode{\sphinxupquote{on}}, \sphinxcode{\sphinxupquote{off}}, and \sphinxcode{\sphinxupquote{reboot}} fence actions to
complete by default.
\\
\hline\phantomsection\label{\detokenize{cluster-options:stonith-max-attempts}}
\index{cluster option!stonith-max-attempts}\index{stonith-max-attempts!cluster option}\ignorespaces 
stonith-max-attempts
&
{\hyperref[\detokenize{cluster-options:score}]{\sphinxcrossref{\DUrole{std,std-ref}{score}}}}
&
10
&
How many times fencing can fail for a target before the cluster will no
longer immediately re-attempt it. Any value below 1 will be ignored, and
the default will be used instead.
\\
\hline\phantomsection\label{\detokenize{cluster-options:stonith-watchdog-timeout}}
\index{cluster option!stonith-watchdog-timeout}\index{stonith-watchdog-timeout!cluster option}\ignorespaces 
stonith-watchdog-timeout
&
{\hyperref[\detokenize{cluster-options:timeout}]{\sphinxcrossref{\DUrole{std,std-ref}{timeout}}}}
&
0
&
If nonzero, and the cluster detects \sphinxcode{\sphinxupquote{have-watchdog}} as \sphinxcode{\sphinxupquote{true}}, then
watchdog-based self-fencing will be performed via SBD when fencing is
required, without requiring a fencing resource explicitly configured.

If this is set to a positive value, unseen nodes are assumed to
self-fence within this much time.

\sphinxstylestrong{Warning:} It must be ensured that this value is larger than the
\sphinxcode{\sphinxupquote{SBD\_WATCHDOG\_TIMEOUT}} environment variable on all nodes. Pacemaker
verifies the settings individually on all nodes and prevents startup or
shuts down if configured wrongly on the fly. It is strongly recommended
that \sphinxcode{\sphinxupquote{SBD\_WATCHDOG\_TIMEOUT}} be set to the same value on all nodes.

If this is set to a negative value, and \sphinxcode{\sphinxupquote{SBD\_WATCHDOG\_TIMEOUT}} is set,
twice that value will be used.

\sphinxstylestrong{Warning:} In this case, it is essential (and currently not verified
by pacemaker) that \sphinxcode{\sphinxupquote{SBD\_WATCHDOG\_TIMEOUT}} is set to the same value on
all nodes.
\\
\hline\phantomsection\label{\detokenize{cluster-options:concurrent-fencing}}
\index{cluster option!concurrent-fencing}\index{concurrent-fencing!cluster option}\ignorespaces 
concurrent-fencing
&
{\hyperref[\detokenize{cluster-options:boolean}]{\sphinxcrossref{\DUrole{std,std-ref}{boolean}}}}
&
false
&
Whether the cluster is allowed to initiate multiple fence actions
concurrently. Fence actions initiated externally, such as via the
\sphinxcode{\sphinxupquote{stonith\_admin}} tool or an application such as DLM, or by the fencer
itself such as recurring device monitors and \sphinxcode{\sphinxupquote{status}} and \sphinxcode{\sphinxupquote{list}}
commands, are not limited by this option.
\\
\hline\phantomsection\label{\detokenize{cluster-options:fence-reaction}}
\index{cluster option!fence-reaction}\index{fence-reaction!cluster option}\ignorespaces 
fence-reaction
&
{\hyperref[\detokenize{cluster-options:enumeration}]{\sphinxcrossref{\DUrole{std,std-ref}{enumeration}}}}
&
stop
&
How should a cluster node react if notified of its own fencing? A
cluster node may receive notification of its own fencing if fencing is
misconfigured, or if fabric fencing is in use that doesn’t cut cluster
communication. Allowed values are \sphinxcode{\sphinxupquote{stop}} to attempt to immediately
stop Pacemaker and stay stopped, or \sphinxcode{\sphinxupquote{panic}} to attempt to immediately
reboot the local node, falling back to stop on failure. The default is
likely to be changed to \sphinxcode{\sphinxupquote{panic}} in a future release. \sphinxstyleemphasis{(since 2.0.3)}
\\
\hline\phantomsection\label{\detokenize{cluster-options:priority-fencing-delay}}
\index{cluster option!priority-fencing-delay}\index{priority-fencing-delay!cluster option}\ignorespaces 
priority-fencing-delay
&
{\hyperref[\detokenize{cluster-options:duration}]{\sphinxcrossref{\DUrole{std,std-ref}{duration}}}}
&
0
&
Apply this delay to any fencing targeting the lost nodes with the
highest total resource priority in case we don’t have the majority of
the nodes in our cluster partition, so that the more significant nodes
potentially win any fencing match (especially meaningful in a
split-brain of a 2-node cluster). A promoted resource instance takes the
resource’s priority plus 1 if the resource’s priority is not 0. Any
static or random delays introduced by \sphinxcode{\sphinxupquote{pcmk\_delay\_base}} and
\sphinxcode{\sphinxupquote{pcmk\_delay\_max}} configured for the corresponding fencing resources
will be added to this delay. This delay should be significantly greater
than (safely twice) the maximum delay from those parameters. \sphinxstyleemphasis{(since
2.0.4)}
\\
\hline\phantomsection\label{\detokenize{cluster-options:node-pending-timeout}}
\index{cluster option!node-pending-timeout}\index{node-pending-timeout!cluster option}\ignorespaces 
node-pending-timeout
&
{\hyperref[\detokenize{cluster-options:duration}]{\sphinxcrossref{\DUrole{std,std-ref}{duration}}}}
&
0
&
Fence nodes that do not join the controller process group within this
much time after joining the cluster, to allow the cluster to continue
managing resources. A value of 0 means never fence pending nodes. Setting the value to 2h means fence nodes after 2 hours.
\sphinxstyleemphasis{(since 2.1.7)}
\\
\hline\phantomsection\label{\detokenize{cluster-options:cluster-delay}}
\index{cluster option!cluster-delay}\index{cluster-delay!cluster option}\ignorespaces 
cluster-delay
&
{\hyperref[\detokenize{cluster-options:duration}]{\sphinxcrossref{\DUrole{std,std-ref}{duration}}}}
&
60s
&
If the DC requires an action to be executed on another node, it will
consider the action failed if it does not get a response from the other
node within this time (beyond the action’s own timeout). The ideal value
will depend on the speed and load of your network and cluster nodes.
\\
\hline\phantomsection\label{\detokenize{cluster-options:dc-deadtime}}
\index{cluster option!dc-deadtime}\index{dc-deadtime!cluster option}\ignorespaces 
dc-deadtime
&
{\hyperref[\detokenize{cluster-options:duration}]{\sphinxcrossref{\DUrole{std,std-ref}{duration}}}}
&
20s
&
How long to wait for a response from other nodes when electing a DC. The
ideal value will depend on the speed and load of your network and
cluster nodes.
\\
\hline\phantomsection\label{\detokenize{cluster-options:cluster-ipc-limit}}
\index{cluster option!cluster-ipc-limit}\index{cluster-ipc-limit!cluster option}\ignorespaces 
cluster-ipc-limit
&
{\hyperref[\detokenize{cluster-options:nonnegative-integer}]{\sphinxcrossref{\DUrole{std,std-ref}{nonnegative integer}}}}
&
500
&
The maximum IPC message backlog before one cluster daemon will
disconnect another. This is of use in large clusters, for which a good
value is the number of resources in the cluster multiplied by the number
of nodes. The default of 500 is also the minimum. Raise this if you see
“Evicting client” log messages for cluster daemon process IDs.
\\
\hline\phantomsection\label{\detokenize{cluster-options:pe-error-series-max}}
\index{cluster option!pe-error-series-max}\index{pe-error-series-max!cluster option}\ignorespaces 
pe-error-series-max
&
{\hyperref[\detokenize{cluster-options:integer}]{\sphinxcrossref{\DUrole{std,std-ref}{integer}}}}
&
-1
&
The number of scheduler inputs resulting in errors to save. These inputs
can be helpful during troubleshooting and when reporting issues. A
negative value means save all inputs, and 0 means save none.
\\
\hline\phantomsection\label{\detokenize{cluster-options:pe-warn-series-max}}
\index{cluster option!pe-warn-series-max}\index{pe-warn-series-max!cluster option}\ignorespaces 
pe-warn-series-max
&
{\hyperref[\detokenize{cluster-options:integer}]{\sphinxcrossref{\DUrole{std,std-ref}{integer}}}}
&
5000
&
The number of scheduler inputs resulting in warnings to save. These
inputs can be helpful during troubleshooting and when reporting issues.
A negative value means save all inputs, and 0 means save none.
\\
\hline\phantomsection\label{\detokenize{cluster-options:pe-input-series-max}}
\index{cluster option!pe-input-series-max}\index{pe-input-series-max!cluster option}\ignorespaces 
pe-input-series-max
&
{\hyperref[\detokenize{cluster-options:integer}]{\sphinxcrossref{\DUrole{std,std-ref}{integer}}}}
&
4000
&
The number of “normal” scheduler inputs to save. These inputs can be
helpful during troubleshooting and when reporting issues. A negative
value means save all inputs, and 0 means save none.
\\
\hline\phantomsection\label{\detokenize{cluster-options:enable-acl}}
\index{cluster option!enable-acl}\index{enable-acl!cluster option}\ignorespaces 
enable-acl
&
{\hyperref[\detokenize{cluster-options:boolean}]{\sphinxcrossref{\DUrole{std,std-ref}{boolean}}}}
&
false
&
Whether {\hyperref[\detokenize{acls:acl}]{\sphinxcrossref{\DUrole{std,std-ref}{access control lists}}}} should be used to authorize
CIB modifications
\\
\hline\phantomsection\label{\detokenize{cluster-options:placement-strategy}}
\index{cluster option!placement-strategy}\index{placement-strategy!cluster option}\ignorespaces 
placement-strategy
&
{\hyperref[\detokenize{cluster-options:enumeration}]{\sphinxcrossref{\DUrole{std,std-ref}{enumeration}}}}
&
default
&
How the cluster should assign resources to nodes (see
{\hyperref[\detokenize{utilization:utilization}]{\sphinxcrossref{\DUrole{std,std-ref}{Utilization and Placement Strategy}}}}). Allowed values are \sphinxcode{\sphinxupquote{default}}, \sphinxcode{\sphinxupquote{utilization}},
\sphinxcode{\sphinxupquote{balanced}}, and \sphinxcode{\sphinxupquote{minimal}}.
\\
\hline\phantomsection\label{\detokenize{cluster-options:node-health-strategy}}
\index{cluster option!node-health-strategy}\index{node-health-strategy!cluster option}\ignorespaces 
node-health-strategy
&
{\hyperref[\detokenize{cluster-options:enumeration}]{\sphinxcrossref{\DUrole{std,std-ref}{enumeration}}}}
&
none
&
How the cluster should react to {\hyperref[\detokenize{nodes:node-health}]{\sphinxcrossref{\DUrole{std,std-ref}{node health}}}}
attributes. Allowed values are \sphinxcode{\sphinxupquote{none}}, \sphinxcode{\sphinxupquote{migrate-on-red}},
\sphinxcode{\sphinxupquote{only-green}}, \sphinxcode{\sphinxupquote{progressive}}, and \sphinxcode{\sphinxupquote{custom}}.
\\
\hline\phantomsection\label{\detokenize{cluster-options:node-health-base}}
\index{cluster option!node-health-base}\index{node-health-base!cluster option}\ignorespaces 
node-health-base
&
{\hyperref[\detokenize{cluster-options:score}]{\sphinxcrossref{\DUrole{std,std-ref}{score}}}}
&
0
&
The base health score assigned to a node. Only used when
\sphinxcode{\sphinxupquote{node-health-strategy}} is \sphinxcode{\sphinxupquote{progressive}}.
\\
\hline\phantomsection\label{\detokenize{cluster-options:node-health-green}}
\index{cluster option!node-health-green}\index{node-health-green!cluster option}\ignorespaces 
node-health-green
&
{\hyperref[\detokenize{cluster-options:score}]{\sphinxcrossref{\DUrole{std,std-ref}{score}}}}
&
0
&
The score to use for a node health attribute whose value is \sphinxcode{\sphinxupquote{green}}.
Only used when \sphinxcode{\sphinxupquote{node-health-strategy}} is \sphinxcode{\sphinxupquote{progressive}} or
\sphinxcode{\sphinxupquote{custom}}.
\\
\hline\phantomsection\label{\detokenize{cluster-options:node-health-yellow}}
\index{cluster option!node-health-yellow}\index{node-health-yellow!cluster option}\ignorespaces 
node-health-yellow
&
{\hyperref[\detokenize{cluster-options:score}]{\sphinxcrossref{\DUrole{std,std-ref}{score}}}}
&
0
&
The score to use for a node health attribute whose value is \sphinxcode{\sphinxupquote{yellow}}.
Only used when \sphinxcode{\sphinxupquote{node-health-strategy}} is \sphinxcode{\sphinxupquote{progressive}} or
\sphinxcode{\sphinxupquote{custom}}.
\\
\hline\phantomsection\label{\detokenize{cluster-options:node-health-red}}
\index{cluster option!node-health-red}\index{node-health-red!cluster option}\ignorespaces 
node-health-red
&
{\hyperref[\detokenize{cluster-options:score}]{\sphinxcrossref{\DUrole{std,std-ref}{score}}}}
&
0
&
The score to use for a node health attribute whose value is \sphinxcode{\sphinxupquote{red}}.
Only used when \sphinxcode{\sphinxupquote{node-health-strategy}} is \sphinxcode{\sphinxupquote{progressive}} or
\sphinxcode{\sphinxupquote{custom}}.
\\
\hline\phantomsection\label{\detokenize{cluster-options:cluster-recheck-interval}}
\index{cluster option!cluster-recheck-interval}\index{cluster-recheck-interval!cluster option}\ignorespaces 
cluster-recheck-interval
&
{\hyperref[\detokenize{cluster-options:duration}]{\sphinxcrossref{\DUrole{std,std-ref}{duration}}}}
&
15min
&
Pacemaker is primarily event-driven, and looks ahead to know when to
recheck the cluster for failure timeouts and most time-based rules
\sphinxstyleemphasis{(since 2.0.3)}. However, it will also recheck the cluster after this
amount of inactivity. This has two goals: rules with \sphinxcode{\sphinxupquote{date\_spec}} are
only guaranteed to be checked this often, and it also serves as a
fail-safe for some kinds of scheduler bugs. A value of 0 disables this
polling.
\\
\hline\phantomsection\label{\detokenize{cluster-options:shutdown-lock}}
\index{cluster option!shutdown-lock}\index{shutdown-lock!cluster option}\ignorespaces 
shutdown-lock
&
{\hyperref[\detokenize{cluster-options:boolean}]{\sphinxcrossref{\DUrole{std,std-ref}{boolean}}}}
&
false
&
The default of false allows active resources to be recovered elsewhere
when their node is cleanly shut down, which is what the vast majority of
users will want. However, some users prefer to make resources highly
available only for failures, with no recovery for clean shutdowns. If
this option is true, resources active on a node when it is cleanly shut
down are kept “locked” to that node (not allowed to run elsewhere) until
they start again on that node after it rejoins (or for at most
\sphinxcode{\sphinxupquote{shutdown-lock-limit}}, if set). Stonith resources and Pacemaker Remote
connections are never locked. Clone and bundle instances and the
promoted role of promotable clones are currently never locked, though
support could be added in a future release. Locks may be manually
cleared using the \sphinxcode{\sphinxupquote{-{-}refresh}} option of \sphinxcode{\sphinxupquote{crm\_resource}} (both the
resource and node must be specified; this works with remote nodes if
their connection resource’s \sphinxcode{\sphinxupquote{target-role}} is set to \sphinxcode{\sphinxupquote{Stopped}}, but
not if Pacemaker Remote is stopped on the remote node without disabling
the connection resource). \sphinxstyleemphasis{(since 2.0.4)}
\\
\hline\phantomsection\label{\detokenize{cluster-options:shutdown-lock-limit}}
\index{cluster option!shutdown-lock-limit}\index{shutdown-lock-limit!cluster option}\ignorespaces 
shutdown-lock-limit
&
{\hyperref[\detokenize{cluster-options:duration}]{\sphinxcrossref{\DUrole{std,std-ref}{duration}}}}
&
0
&
If \sphinxcode{\sphinxupquote{shutdown-lock}} is true, and this is set to a nonzero time
duration, locked resources will be allowed to start after this much time
has passed since the node shutdown was initiated, even if the node has
not rejoined. (This works with remote nodes only if their connection
resource’s \sphinxcode{\sphinxupquote{target-role}} is set to \sphinxcode{\sphinxupquote{Stopped}}.) \sphinxstyleemphasis{(since 2.0.4)}
\\
\hline\phantomsection\label{\detokenize{cluster-options:remove-after-stop}}
\index{cluster option!remove-after-stop}\index{remove-after-stop!cluster option}\ignorespaces 
remove-after-stop
&
{\hyperref[\detokenize{cluster-options:boolean}]{\sphinxcrossref{\DUrole{std,std-ref}{boolean}}}}
&
false
&
\sphinxstyleemphasis{Deprecated} Whether the cluster should remove resources from
Pacemaker’s executor after they are stopped. Values other than the
default are, at best, poorly tested and potentially dangerous.  This
option is deprecated and will be removed in a future release.
\\
\hline\phantomsection\label{\detokenize{cluster-options:startup-fencing}}
\index{cluster option!startup-fencing}\index{startup-fencing!cluster option}\ignorespaces 
startup-fencing
&
{\hyperref[\detokenize{cluster-options:boolean}]{\sphinxcrossref{\DUrole{std,std-ref}{boolean}}}}
&
true
&
\sphinxstyleemphasis{Advanced Use Only:} Whether the cluster should fence unseen nodes at
start-up. Setting this to false is unsafe, because the unseen nodes
could be active and running resources but unreachable. \sphinxcode{\sphinxupquote{dc-deadtime}}
acts as a grace period before this fencing, since a DC must be elected
to schedule fencing.
\\
\hline\phantomsection\label{\detokenize{cluster-options:election-timeout}}
\index{cluster option!election-timeout}\index{election-timeout!cluster option}\ignorespaces 
election-timeout
&
{\hyperref[\detokenize{cluster-options:duration}]{\sphinxcrossref{\DUrole{std,std-ref}{duration}}}}
&
2min
&
\sphinxstyleemphasis{Advanced Use Only:} If a winner is not declared within this much time
of starting an election, the node that initiated the election will
declare itself the winner.
\\
\hline\phantomsection\label{\detokenize{cluster-options:shutdown-escalation}}
\index{cluster option!shutdown-escalation}\index{shutdown-escalation!cluster option}\ignorespaces 
shutdown-escalation
&
{\hyperref[\detokenize{cluster-options:duration}]{\sphinxcrossref{\DUrole{std,std-ref}{duration}}}}
&
20min
&
\sphinxstyleemphasis{Advanced Use Only:} The controller will exit immediately if a shutdown
does not complete within this much time.
\\
\hline\phantomsection\label{\detokenize{cluster-options:join-integration-timeout}}
\index{cluster option!join-integration-timeout}\index{join-integration-timeout!cluster option}\ignorespaces 
join-integration-timeout
&
{\hyperref[\detokenize{cluster-options:duration}]{\sphinxcrossref{\DUrole{std,std-ref}{duration}}}}
&
3min
&
\sphinxstyleemphasis{Advanced Use Only:} If you need to adjust this value, it probably
indicates the presence of a bug.
\\
\hline\phantomsection\label{\detokenize{cluster-options:join-finalization-timeout}}
\index{cluster option!join-finalization-timeout}\index{join-finalization-timeout!cluster option}\ignorespaces 
join-finalization-timeout
&
{\hyperref[\detokenize{cluster-options:duration}]{\sphinxcrossref{\DUrole{std,std-ref}{duration}}}}
&
30min
&
\sphinxstyleemphasis{Advanced Use Only:} If you need to adjust this value, it probably
indicates the presence of a bug.
\\
\hline\phantomsection\label{\detokenize{cluster-options:transition-delay}}
\index{cluster option!transition-delay}\index{transition-delay!cluster option}\ignorespaces 
transition-delay
&
{\hyperref[\detokenize{cluster-options:duration}]{\sphinxcrossref{\DUrole{std,std-ref}{duration}}}}
&
0s
&
\sphinxstyleemphasis{Advanced Use Only:} Delay cluster recovery for the configured interval
to allow for additional or related events to occur. This can be useful
if your configuration is sensitive to the order in which ping updates
arrive. Enabling this option will slow down cluster recovery under all
conditions.
\\
\hline
\end{longtable}\sphinxatlongtableend\end{savenotes}


\section{Cluster Nodes}
\label{\detokenize{nodes:cluster-nodes}}\label{\detokenize{nodes::doc}}

\subsection{Defining a Cluster Node}
\label{\detokenize{nodes:defining-a-cluster-node}}
Each cluster node will have an entry in the \sphinxcode{\sphinxupquote{nodes}} section containing at
least an ID and a name. A cluster node’s ID is defined by the cluster layer
(Corosync).

\begin{sphinxShadowBox}
\sphinxstyletopictitle{\sphinxstylestrong{Example Corosync cluster node entry}}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}node} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}101\PYGZdq{}} \PYG{n+na}{uname=}\PYG{l+s}{\PYGZdq{}pcmk\PYGZhy{}1\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

In normal circumstances, the admin should let the cluster populate this
information automatically from the cluster layer.


\subsubsection{Where Pacemaker Gets the Node Name}
\label{\detokenize{nodes:where-pacemaker-gets-the-node-name}}\label{\detokenize{nodes:node-name}}
The name that Pacemaker uses for a node in the configuration does not have to
be the same as its local hostname. Pacemaker uses the following for a Corosync
node’s name, in order of most preferred first:
\begin{itemize}
\item {} 
The value of \sphinxcode{\sphinxupquote{name}} in the \sphinxcode{\sphinxupquote{nodelist}} section of \sphinxcode{\sphinxupquote{corosync.conf}}

\item {} 
The value of \sphinxcode{\sphinxupquote{ring0\_addr}} in the \sphinxcode{\sphinxupquote{nodelist}} section of \sphinxcode{\sphinxupquote{corosync.conf}}

\item {} 
The local hostname (value of \sphinxcode{\sphinxupquote{uname -n}})

\end{itemize}

If the cluster is running, the \sphinxcode{\sphinxupquote{crm\_node -n}} command will display the local
node’s name as used by the cluster.

If a Corosync \sphinxcode{\sphinxupquote{nodelist}} is used, \sphinxcode{\sphinxupquote{crm\_node -{-}name-for-id}} with a Corosync
node ID will display the name used by the node with the given Corosync
\sphinxcode{\sphinxupquote{nodeid}}, for example:

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
crm\PYGZus{}node \PYGZhy{}\PYGZhy{}name\PYGZhy{}for\PYGZhy{}id 2
\end{sphinxVerbatim}

\index{node!attribute}\index{node attribute}\ignorespaces 

\subsection{Node Attributes}
\label{\detokenize{nodes:node-attributes}}\label{\detokenize{nodes:index-0}}\label{\detokenize{nodes:id1}}
Pacemaker allows node-specific values to be specified using \sphinxstyleemphasis{node attributes}.
A node attribute has a name, and may have a distinct value for each node.

Node attributes come in two types, \sphinxstyleemphasis{permanent} and \sphinxstyleemphasis{transient}. Permanent node
attributes are kept within the \sphinxcode{\sphinxupquote{node}} entry, and keep their values even if
the cluster restarts on a node. Transient node attributes are kept in the CIB’s
\sphinxcode{\sphinxupquote{status}} section, and go away when the cluster stops on the node.

While certain node attributes have specific meanings to the cluster, they are
mainly intended to allow administrators and resource agents to track any
information desired.

For example, an administrator might choose to define node attributes for how
much RAM and disk space each node has, which OS each uses, or which server room
rack each node is in.

Users can configure {\hyperref[\detokenize{rules:rules}]{\sphinxcrossref{\DUrole{std,std-ref}{Rules}}}} that use node attributes to affect where
resources are placed.


\subsubsection{Setting and querying node attributes}
\label{\detokenize{nodes:setting-and-querying-node-attributes}}
Node attributes can be set and queried using the \sphinxcode{\sphinxupquote{crm\_attribute}} and
\sphinxcode{\sphinxupquote{attrd\_updater}} commands, so that the user does not have to deal with XML
configuration directly.

Here is an example command to set a permanent node attribute, and the XML
configuration that would be generated:

\begin{sphinxShadowBox}
\sphinxstyletopictitle{\sphinxstylestrong{Result of using crm\_attribute to specify which kernel pcmk-1 is running}}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYGZsh{} crm\PYGZus{}attribute \PYGZhy{}\PYGZhy{}type nodes \PYGZhy{}\PYGZhy{}node pcmk\PYGZhy{}1 \PYGZhy{}\PYGZhy{}name kernel \PYGZhy{}\PYGZhy{}update \PYGZdl{}(uname \PYGZhy{}r)
\end{sphinxVerbatim}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}node} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}1\PYGZdq{}} \PYG{n+na}{uname=}\PYG{l+s}{\PYGZdq{}pcmk\PYGZhy{}1\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}instance\PYGZus{}attributes} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}nodes\PYGZhy{}1\PYGZhy{}attributes\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
     \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}nodes\PYGZhy{}1\PYGZhy{}kernel\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}kernel\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}3.10.0\PYGZhy{}862.14.4.el7.x86\PYGZus{}64\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}/instance\PYGZus{}attributes\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/node\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

To read back the value that was just set:

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYGZsh{} crm\PYGZus{}attribute \PYGZhy{}\PYGZhy{}type nodes \PYGZhy{}\PYGZhy{}node pcmk\PYGZhy{}1 \PYGZhy{}\PYGZhy{}name kernel \PYGZhy{}\PYGZhy{}query
scope=nodes  name=kernel value=3.10.0\PYGZhy{}862.14.4.el7.x86\PYGZus{}64
\end{sphinxVerbatim}

The \sphinxcode{\sphinxupquote{-{-}type nodes}} indicates that this is a permanent node attribute;
\sphinxcode{\sphinxupquote{-{-}type status}} would indicate a transient node attribute.


\subsubsection{Special node attributes}
\label{\detokenize{nodes:special-node-attributes}}\label{\detokenize{nodes:id2}}
Certain node attributes have special meaning to the cluster.

Node attribute names beginning with \sphinxcode{\sphinxupquote{\#}} are considered reserved for these
special attributes. Some special attributes do not start with \sphinxcode{\sphinxupquote{\#}}, for
historical reasons.

Certain special attributes are set automatically by the cluster, should never
be modified directly, and can be used only within {\hyperref[\detokenize{rules:rules}]{\sphinxcrossref{\DUrole{std,std-ref}{Rules}}}}; these are
listed under
{\hyperref[\detokenize{rules:node-attribute-expressions-special}]{\sphinxcrossref{\DUrole{std,std-ref}{built-in node attributes}}}}.

For true/false values, the cluster considers a value of “1”, “y”, “yes”, “on”,
or “true” (case-insensitively) to be true, “0”, “n”, “no”, “off”, “false”, or
unset to be false, and anything else to be an error.


\begin{savenotes}\sphinxatlongtablestart\begin{longtable}{|\X{1}{3}|\X{2}{3}|}
\caption{\sphinxstylestrong{Node attributes with special significance}\strut}\label{\detokenize{nodes:id3}}\\*[\sphinxlongtablecapskipadjust]
\hline
\sphinxstyletheadfamily 
Name
&\sphinxstyletheadfamily 
Description
\\
\hline
\endfirsthead

\multicolumn{2}{c}%
{\makebox[0pt]{\sphinxtablecontinued{\tablename\ \thetable{} -- continued from previous page}}}\\
\hline
\sphinxstyletheadfamily 
Name
&\sphinxstyletheadfamily 
Description
\\
\hline
\endhead

\hline
\multicolumn{2}{r}{\makebox[0pt][r]{\sphinxtablecontinued{Continued on next page}}}\\
\endfoot

\endlastfoot

fail-count-*
&
\index{node attribute!fail-count}\index{fail-count!node attribute}\ignorespaces 
Attributes whose names start with
\sphinxcode{\sphinxupquote{fail-count-}} are managed by the cluster
to track how many times particular resource
operations have failed on this node. These
should be queried and cleared via the
\sphinxcode{\sphinxupquote{crm\_failcount}} or
\sphinxcode{\sphinxupquote{crm\_resource -{-}cleanup}} commands rather
than directly.
\\
\hline
last-failure-*
&
\index{node attribute!last-failure}\index{last-failure!node attribute}\ignorespaces 
Attributes whose names start with
\sphinxcode{\sphinxupquote{last-failure-}} are managed by the cluster
to track when particular resource operations
have most recently failed on this node.
These should be cleared via the
\sphinxcode{\sphinxupquote{crm\_failcount}} or
\sphinxcode{\sphinxupquote{crm\_resource -{-}cleanup}} commands rather
than directly.
\\
\hline
maintenance
&\phantomsection\label{\detokenize{nodes:node-maintenance}}
\index{node attribute!maintenance}\index{maintenance!node attribute}\ignorespaces 
If true, the cluster will not start or stop any
resources on this node. Any resources active on the
node become unmanaged, and any recurring operations
for those resources (except those specifying
\sphinxcode{\sphinxupquote{role}} as \sphinxcode{\sphinxupquote{Stopped}}) will be paused. The
{\hyperref[\detokenize{cluster-options:maintenance-mode}]{\sphinxcrossref{\DUrole{std,std-ref}{maintenance-mode}}}} cluster
option, if true, overrides this. If this attribute
is true, it overrides the
{\hyperref[\detokenize{resources:is-managed}]{\sphinxcrossref{\DUrole{std,std-ref}{is-managed}}}} and
{\hyperref[\detokenize{resources:rsc-maintenance}]{\sphinxcrossref{\DUrole{std,std-ref}{maintenance}}}}
meta-attributes of affected resources and
{\hyperref[\detokenize{operations:op-enabled}]{\sphinxcrossref{\DUrole{std,std-ref}{enabled}}}} meta-attribute for
affected recurring actions. Pacemaker should not be
restarted on a node that is in single-node
maintenance mode.
\\
\hline
probe\_complete
&
\index{node attribute!probe\_complete}\index{probe\_complete!node attribute}\ignorespaces 
This is managed by the cluster to detect
when nodes need to be reprobed, and should
never be used directly.
\\
\hline
resource-discovery-enabled
&
\index{node attribute!resource-discovery-enabled}\index{resource-discovery-enabled!node attribute}\ignorespaces 
If the node is a remote node, fencing is enabled,
and this attribute is explicitly set to false
(unset means true in this case), resource discovery
(probes) will not be done on this node. This is
highly discouraged; the \sphinxcode{\sphinxupquote{resource-discovery}}
location constraint property is preferred for this
purpose.
\\
\hline
shutdown
&
\index{node attribute!shutdown}\index{shutdown!node attribute}\ignorespaces 
This is managed by the cluster to orchestrate the
shutdown of a node, and should never be used
directly.
\\
\hline
site-name
&
\index{node attribute!site-name}\index{site-name!node attribute}\ignorespaces 
If set, this will be used as the value of the
\sphinxcode{\sphinxupquote{\#site-name}} node attribute used in rules. (If
not set, the value of the \sphinxcode{\sphinxupquote{cluster-name}} cluster
option will be used as \sphinxcode{\sphinxupquote{\#site-name}} instead.)
\\
\hline
standby
&
\index{node attribute!standby}\index{standby!node attribute}\ignorespaces 
If true, the node is in standby mode. This is
typically set and queried via the \sphinxcode{\sphinxupquote{crm\_standby}}
command rather than directly.
\\
\hline
terminate
&
\index{node attribute!terminate}\index{terminate!node attribute}\ignorespaces 
If the value is true or begins with any nonzero
number, the node will be fenced. This is typically
set by tools rather than directly.
\\
\hline
\#digests-*
&
\index{node attribute!\#digests}\index{\#digests!node attribute}\ignorespaces 
Attributes whose names start with \sphinxcode{\sphinxupquote{\#digests-}} are
managed by the cluster to detect when
{\hyperref[\detokenize{fencing:unfencing}]{\sphinxcrossref{\DUrole{std,std-ref}{Unfencing}}}} needs to be redone, and should
never be used directly.
\\
\hline
\#node-unfenced
&
\index{node attribute!\#node-unfenced}\index{\#node-unfenced!node attribute}\ignorespaces 
When the node was last unfenced (as seconds since
the epoch). This is managed by the cluster and
should never be used directly.
\\
\hline
\end{longtable}\sphinxatlongtableend\end{savenotes}

\index{node!health}\ignorespaces 

\subsection{Tracking Node Health}
\label{\detokenize{nodes:tracking-node-health}}\label{\detokenize{nodes:node-health}}\label{\detokenize{nodes:index-12}}
A node may be functioning adequately as far as cluster membership is concerned,
and yet be “unhealthy” in some respect that makes it an undesirable location
for resources. For example, a disk drive may be reporting SMART errors, or the
CPU may be highly loaded.

Pacemaker offers a way to automatically move resources off unhealthy nodes.

\index{node attribute!health}\ignorespaces 

\subsubsection{Node Health Attributes}
\label{\detokenize{nodes:node-health-attributes}}\label{\detokenize{nodes:index-13}}
Pacemaker will treat any node attribute whose name starts with \sphinxcode{\sphinxupquote{\#health}} as
an indicator of node health. Node health attributes may have one of the
following values:


\begin{savenotes}\sphinxattablestart
\centering
\sphinxcapstartof{table}
\sphinxcaption{\sphinxstylestrong{Allowed Values for Node Health Attributes}}\label{\detokenize{nodes:id4}}
\sphinxaftercaption
\begin{tabular}[t]{|\X{1}{5}|\X{4}{5}|}
\hline
\sphinxstyletheadfamily 
Value
&\sphinxstyletheadfamily 
Intended significance
\\
\hline
\sphinxcode{\sphinxupquote{red}}
&
\index{red!node health attribute value}\index{node attribute!health (red)}\ignorespaces 
This indicator is unhealthy
\\
\hline
\sphinxcode{\sphinxupquote{yellow}}
&
\index{yellow!node health attribute value}\index{node attribute!health (yellow)}\ignorespaces 
This indicator is becoming unhealthy
\\
\hline
\sphinxcode{\sphinxupquote{green}}
&
\index{green!node health attribute value}\index{node attribute!health (green)}\ignorespaces 
This indicator is healthy
\\
\hline
\sphinxstyleemphasis{integer}
&
\index{score!node health attribute value}\index{node attribute!health (score)}\ignorespaces 
A numeric score to apply to all resources on this node (0 or
positive is healthy, negative is unhealthy)
\\
\hline
\end{tabular}
\par
\sphinxattableend\end{savenotes}

\index{cluster option!node-health-strategy}\index{node-health-strategy!cluster option}\ignorespaces 

\subsubsection{Node Health Strategy}
\label{\detokenize{nodes:node-health-strategy}}\label{\detokenize{nodes:index-18}}
Pacemaker assigns a node health score to each node, as the sum of the values of
all its node health attributes. This score will be used as a location
constraint applied to this node for all resources.

The \sphinxcode{\sphinxupquote{node-health-strategy}} cluster option controls how Pacemaker responds to
changes in node health attributes, and how it translates \sphinxcode{\sphinxupquote{red}}, \sphinxcode{\sphinxupquote{yellow}},
and \sphinxcode{\sphinxupquote{green}} to scores.

Allowed values are:


\begin{savenotes}\sphinxattablestart
\centering
\sphinxcapstartof{table}
\sphinxcaption{\sphinxstylestrong{Node Health Strategies}}\label{\detokenize{nodes:id5}}
\sphinxaftercaption
\begin{tabular}[t]{|\X{1}{5}|\X{4}{5}|}
\hline
\sphinxstyletheadfamily 
Value
&\sphinxstyletheadfamily 
Effect
\\
\hline
none
&
\index{node-health-strategy!none}\index{none!node-health-strategy value}\ignorespaces 
Do not track node health attributes at all.
\\
\hline
migrate-on-red
&
\index{node-health-strategy!migrate-on-red}\index{migrate-on-red!node-health-strategy value}\ignorespaces 
Assign the value of \sphinxcode{\sphinxupquote{-INFINITY}} to \sphinxcode{\sphinxupquote{red}}, and 0 to
\sphinxcode{\sphinxupquote{yellow}} and \sphinxcode{\sphinxupquote{green}}. This will cause all resources
to move off the node if any attribute is \sphinxcode{\sphinxupquote{red}}.
\\
\hline
only-green
&
\index{node-health-strategy!only-green}\index{only-green!node-health-strategy value}\ignorespaces 
Assign the value of \sphinxcode{\sphinxupquote{-INFINITY}} to \sphinxcode{\sphinxupquote{red}} and
\sphinxcode{\sphinxupquote{yellow}}, and 0 to \sphinxcode{\sphinxupquote{green}}. This will cause all
resources to move off the node if any attribute is
\sphinxcode{\sphinxupquote{red}} or \sphinxcode{\sphinxupquote{yellow}}.
\\
\hline
progressive
&
\index{node-health-strategy!progressive}\index{progressive!node-health-strategy value}\ignorespaces 
Assign the value of the \sphinxcode{\sphinxupquote{node-health-red}} cluster
option to \sphinxcode{\sphinxupquote{red}}, the value of \sphinxcode{\sphinxupquote{node-health-yellow}}
to \sphinxcode{\sphinxupquote{yellow}}, and the value of \sphinxcode{\sphinxupquote{node-health-green}} to
\sphinxcode{\sphinxupquote{green}}. Each node is additionally assigned a score of
\sphinxcode{\sphinxupquote{node-health-base}} (this allows resources to start
even if some attributes are \sphinxcode{\sphinxupquote{yellow}}). This strategy
gives the administrator finer control over how important
each value is.
\\
\hline
custom
&
\index{node-health-strategy!custom}\index{custom!node-health-strategy value}\ignorespaces 
Track node health attributes using the same values as
\sphinxcode{\sphinxupquote{progressive}} for \sphinxcode{\sphinxupquote{red}}, \sphinxcode{\sphinxupquote{yellow}}, and \sphinxcode{\sphinxupquote{green}},
but do not take them into account. The administrator is
expected to implement a policy by defining {\hyperref[\detokenize{rules:rules}]{\sphinxcrossref{\DUrole{std,std-ref}{Rules}}}}
referencing node health attributes.
\\
\hline
\end{tabular}
\par
\sphinxattableend\end{savenotes}


\subsubsection{Exempting a Resource from Health Restrictions}
\label{\detokenize{nodes:exempting-a-resource-from-health-restrictions}}
If you want a resource to be able to run on a node even if its health score
would otherwise prevent it, set the resource’s \sphinxcode{\sphinxupquote{allow-unhealthy-nodes}}
meta-attribute to \sphinxcode{\sphinxupquote{true}} \sphinxstyleemphasis{(available since 2.1.3)}.

This is particularly useful for node health agents, to allow them to detect
when the node becomes healthy again. If you configure a health agent without
this setting, then the health agent will be banned from an unhealthy node,
and you will have to investigate and clear the health attribute manually once
it is healthy to allow resources on the node again.

If you want the meta-attribute to apply to a clone, it must be set on the clone
itself, not on the resource being cloned.


\subsubsection{Configuring Node Health Agents}
\label{\detokenize{nodes:configuring-node-health-agents}}
Since Pacemaker calculates node health based on node attributes, any method
that sets node attributes may be used to measure node health. The most common
are resource agents and custom daemons.

Pacemaker provides examples that can be used directly or as a basis for custom
code. The \sphinxcode{\sphinxupquote{ocf:pacemaker:HealthCPU}}, \sphinxcode{\sphinxupquote{ocf:pacemaker:HealthIOWait}}, and
\sphinxcode{\sphinxupquote{ocf:pacemaker:HealthSMART}} resource agents set node health attributes based
on CPU and disk status.

To take advantage of this feature, add the resource to your cluster (generally
as a cloned resource with a recurring monitor action, to continually check the
health of all nodes). For example:

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Example HealthIOWait resource configuration}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}clone} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}resHealthIOWait\PYGZhy{}clone\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}primitive} \PYG{n+na}{class=}\PYG{l+s}{\PYGZdq{}ocf\PYGZdq{}} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}HealthIOWait\PYGZdq{}} \PYG{n+na}{provider=}\PYG{l+s}{\PYGZdq{}pacemaker\PYGZdq{}} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}HealthIOWait\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}instance\PYGZus{}attributes} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}resHealthIOWait\PYGZhy{}instance\PYGZus{}attributes\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}resHealthIOWait\PYGZhy{}instance\PYGZus{}attributes\PYGZhy{}red\PYGZus{}limit\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}red\PYGZus{}limit\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}30\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}resHealthIOWait\PYGZhy{}instance\PYGZus{}attributes\PYGZhy{}yellow\PYGZus{}limit\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}yellow\PYGZus{}limit\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}10\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/instance\PYGZus{}attributes\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}operations}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}op} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}resHealthIOWait\PYGZhy{}monitor\PYGZhy{}interval\PYGZhy{}5\PYGZdq{}} \PYG{n+na}{interval=}\PYG{l+s}{\PYGZdq{}5\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}monitor\PYGZdq{}} \PYG{n+na}{timeout=}\PYG{l+s}{\PYGZdq{}5\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}op} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}resHealthIOWait\PYGZhy{}start\PYGZhy{}interval\PYGZhy{}0s\PYGZdq{}} \PYG{n+na}{interval=}\PYG{l+s}{\PYGZdq{}0s\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}start\PYGZdq{}} \PYG{n+na}{timeout=}\PYG{l+s}{\PYGZdq{}10s\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}op} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}resHealthIOWait\PYGZhy{}stop\PYGZhy{}interval\PYGZhy{}0s\PYGZdq{}} \PYG{n+na}{interval=}\PYG{l+s}{\PYGZdq{}0s\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}stop\PYGZdq{}} \PYG{n+na}{timeout=}\PYG{l+s}{\PYGZdq{}10s\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/operations\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}/primitive\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/clone\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

The resource agents use \sphinxcode{\sphinxupquote{attrd\_updater}} to set proper status for each node
running this resource, as a node attribute whose name starts with \sphinxcode{\sphinxupquote{\#health}}
(for \sphinxcode{\sphinxupquote{HealthIOWait}}, the node attribute is named \sphinxcode{\sphinxupquote{\#health-iowait}}).

When a node is no longer faulty, you can force the cluster to make it available
to take resources without waiting for the next monitor, by setting the node
health attribute to green. For example:

\begin{sphinxShadowBox}
\sphinxstyletopictitle{\sphinxstylestrong{Force node1 to be marked as healthy}}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYGZsh{} attrd\PYGZus{}updater \PYGZhy{}\PYGZhy{}name \PYGZdq{}\PYGZsh{}health\PYGZhy{}iowait\PYGZdq{} \PYGZhy{}\PYGZhy{}update \PYGZdq{}green\PYGZdq{} \PYGZhy{}\PYGZhy{}node \PYGZdq{}node1\PYGZdq{}
\end{sphinxVerbatim}
\end{sphinxShadowBox}


\section{Cluster Resources}
\label{\detokenize{resources:cluster-resources}}\label{\detokenize{resources:resource}}\label{\detokenize{resources::doc}}

\subsection{What is a Cluster Resource?}
\label{\detokenize{resources:what-is-a-cluster-resource}}\label{\detokenize{resources:s-resource-primitive}}
\index{resource}\ignorespaces 
A \sphinxstyleemphasis{resource} is a service managed by Pacemaker. The simplest type of resource,
a \sphinxstyleemphasis{primitive}, is described in this chapter. More complex forms, such as groups
and clones, are described in later chapters.

Every primitive has a \sphinxstyleemphasis{resource agent} that provides Pacemaker a standardized
interface for managing the service. This allows Pacemaker to be agnostic about
the services it manages. Pacemaker doesn’t need to understand how the service
works because it relies on the resource agent to do the right thing when asked.

Every resource has a \sphinxstyleemphasis{class} specifying the standard that its resource agent
follows, and a \sphinxstyleemphasis{type} identifying the specific service being managed.

\phantomsection\label{\detokenize{resources:s-resource-supported}}
\index{resource!class}\ignorespaces 

\subsection{Resource Classes}
\label{\detokenize{resources:resource-classes}}\label{\detokenize{resources:index-1}}
Pacemaker supports several classes, or standards, of resource agents:
\begin{itemize}
\item {} 
OCF

\item {} 
LSB

\item {} 
Systemd

\item {} 
Service

\item {} 
Fencing

\item {} 
Nagios \sphinxstyleemphasis{(deprecated since 2.1.6)}

\item {} 
Upstart \sphinxstyleemphasis{(deprecated since 2.1.0)}

\end{itemize}

\index{resource!OCF}\index{OCF!resources}\index{Open Cluster Framework!resources}\ignorespaces 

\subsubsection{Open Cluster Framework}
\label{\detokenize{resources:open-cluster-framework}}\label{\detokenize{resources:index-2}}
The Open Cluster Framework (OCF) Resource Agent API is a ClusterLabs
standard for managing services. It is the most preferred since it is
specifically designed for use in a Pacemaker cluster.

OCF agents are scripts that support a variety of actions including \sphinxcode{\sphinxupquote{start}},
\sphinxcode{\sphinxupquote{stop}}, and \sphinxcode{\sphinxupquote{monitor}}. They may accept parameters, making them more
flexible than other classes. The number and purpose of parameters is left to
the agent, which advertises them via the \sphinxcode{\sphinxupquote{meta-data}} action.

Unlike other classes, OCF agents have a \sphinxstyleemphasis{provider} as well as a class and type.

For more information, see the “Resource Agents” chapter of \sphinxstyleemphasis{Pacemaker
Administration} and the \sphinxhref{https://github.com/ClusterLabs/OCF-spec/tree/main/ra}{OCF standard}.

\phantomsection\label{\detokenize{resources:s-resource-supported-systemd}}
\index{Resource!Systemd}\index{Systemd!resources}\ignorespaces 

\subsubsection{Systemd}
\label{\detokenize{resources:systemd}}\label{\detokenize{resources:index-3}}
Most Linux distributions use \sphinxhref{http://www.freedesktop.org/wiki/Software/systemd}{Systemd} for system initialization
and service management. \sphinxstyleemphasis{Unit files} specify how to manage services and are
usually provided by the distribution.

Pacemaker can manage systemd services. Simply create a resource with
\sphinxcode{\sphinxupquote{systemd}} as the resource class and the unit file name as the resource type.
Do \sphinxstyleemphasis{not} run \sphinxcode{\sphinxupquote{systemctl enable}} on the unit.

\begin{sphinxadmonition}{important}{Important:}
Make sure that any systemd services to be controlled by the cluster are
\sphinxstyleemphasis{not} enabled to start at boot.
\end{sphinxadmonition}

\index{resource!LSB}\index{LSB!resources}\index{Linux Standard Base!resources}\ignorespaces 

\subsubsection{Linux Standard Base}
\label{\detokenize{resources:linux-standard-base}}\label{\detokenize{resources:index-4}}
\sphinxstyleemphasis{LSB} resource agents, also known as \sphinxhref{https://en.wikipedia.org/wiki/Init\#SysV-styleinitscripts}{SysV-style}, are scripts that
provide start, stop, and status actions for a service.

They are provided by some operating system distributions. If a full path is not
given, they are assumed to be located in a directory specified when your
Pacemaker software was built (usually \sphinxcode{\sphinxupquote{/etc/init.d}}).

In order to be used with Pacemaker, they must conform to the \sphinxhref{http://refspecs.linux-foundation.org/LSB\_5.0.0/LSB-Core-generic/LSB-Core-generic/iniscrptact.html}{LSB specification}
as it relates to init scripts.

\begin{sphinxadmonition}{warning}{Warning:}
Some LSB scripts do not fully comply with the standard. For details on how
to check whether your script is LSB-compatible, see the “Resource Agents”
chapter of \sphinxtitleref{Pacemaker Administration}. Common problems include:
\begin{itemize}
\item {} 
Not implementing the \sphinxcode{\sphinxupquote{status}} action

\item {} 
Not observing the correct exit status codes

\item {} 
Starting a started resource returns an error

\item {} 
Stopping a stopped resource returns an error

\end{itemize}
\end{sphinxadmonition}

\begin{sphinxadmonition}{important}{Important:}
Make sure the host is \sphinxstyleemphasis{not} configured to start any LSB services at boot
that will be controlled by the cluster.
\end{sphinxadmonition}

\index{Resource!System Services}\index{System Service!resources}\ignorespaces 

\subsubsection{System Services}
\label{\detokenize{resources:system-services}}\label{\detokenize{resources:index-5}}
Since there are various types of system services (\sphinxcode{\sphinxupquote{systemd}},
\sphinxcode{\sphinxupquote{upstart}}, and \sphinxcode{\sphinxupquote{lsb}}), Pacemaker supports a special \sphinxcode{\sphinxupquote{service}} alias which
intelligently figures out which one applies to a given cluster node.

This is particularly useful when the cluster contains a mix of
\sphinxcode{\sphinxupquote{systemd}}, \sphinxcode{\sphinxupquote{upstart}}, and \sphinxcode{\sphinxupquote{lsb}}.

In order, Pacemaker will try to find the named service as:
\begin{itemize}
\item {} 
an LSB init script

\item {} 
a Systemd unit file

\item {} 
an Upstart job

\end{itemize}

\index{Resource!STONITH}\index{STONITH!resources}\ignorespaces 

\subsubsection{STONITH}
\label{\detokenize{resources:stonith}}\label{\detokenize{resources:index-6}}
The \sphinxcode{\sphinxupquote{stonith}} class is used for managing fencing devices, discussed later in
{\hyperref[\detokenize{fencing:fencing}]{\sphinxcrossref{\DUrole{std,std-ref}{Fencing}}}}.

\index{Resource!Nagios Plugins}\index{Nagios Plugins!resources}\ignorespaces 

\subsubsection{Nagios Plugins}
\label{\detokenize{resources:nagios-plugins}}\label{\detokenize{resources:index-7}}
Nagios Plugins are a way to monitor services. Pacemaker can use these as
resources, to react to a change in the service’s status.

To use plugins as resources, Pacemaker must have been built with support, and
OCF-style meta-data for the plugins must be installed on nodes that can run
them. Meta-data for several common plugins is provided by the
\sphinxhref{https://github.com/ClusterLabs/nagios-agents-metadata}{nagios-agents-metadata}
project.

The supported parameters for such a resource are same as the long options of
the plugin.

Start and monitor actions for plugin resources are implemented as invoking the
plugin. A plugin result of “OK” (0) is treated as success, a result of “WARN”
(1) is treated as a successful but degraded service, and any other result is
considered a failure.

A plugin resource is not going to change its status after recovery by
restarting the plugin, so using them alone does not make sense with \sphinxcode{\sphinxupquote{on-fail}}
set (or left to default) to \sphinxcode{\sphinxupquote{restart}}. Another value could make sense, for
example, if you want to fence or standby nodes that cannot reach some external
service.

A more common use case for plugin resources is to configure them with a
\sphinxcode{\sphinxupquote{container}} meta-attribute set to the name of another resource that actually
makes the service available, such as a virtual machine or container.

With \sphinxcode{\sphinxupquote{container}} set, the plugin resource will automatically be colocated
with the containing resource and ordered after it, and the containing resource
will be considered failed if the plugin resource fails. This allows monitoring
of a service inside a virtual machine or container, with recovery of the
virtual machine or container if the service fails.

\begin{sphinxadmonition}{warning}{Warning:}
Nagios support is deprecated in Pacemaker. Support will be dropped entirely
at the next major release of Pacemaker.

For monitoring a service inside a virtual machine or container, the
recommended alternative is to configure the virtual machine as a guest node
or the container as a {\hyperref[\detokenize{collective:s-resource-bundle}]{\sphinxcrossref{\DUrole{std,std-ref}{bundle}}}}. For other use
cases, or when the virtual machine or container image cannot be modified,
the recommended alternative is to write a custom OCF agent for the service
(which may even call the Nagios plugin as part of its status action).
\end{sphinxadmonition}

\index{Resource!Upstart}\index{Upstart!resources}\ignorespaces 

\subsubsection{Upstart}
\label{\detokenize{resources:upstart}}\label{\detokenize{resources:index-8}}
Some Linux distributions previously used \sphinxhref{https://upstart.ubuntu.com/}{Upstart} for system initialization and service
management. Pacemaker is able to manage services using Upstart if the local
system supports them and support was enabled when your Pacemaker software was
built.

The \sphinxstyleemphasis{jobs} that specify how services are managed are usually provided by the
operating system distribution.

\begin{sphinxadmonition}{important}{Important:}
Make sure the host is \sphinxstyleemphasis{not} configured to start any Upstart services at boot
that will be controlled by the cluster.
\end{sphinxadmonition}

\begin{sphinxadmonition}{warning}{Warning:}
Upstart support is deprecated in Pacemaker. Upstart is no longer actively
maintained, and test platforms for it are no longer readily usable. Support
will be dropped entirely at the next major release of Pacemaker.
\end{sphinxadmonition}


\subsection{Resource Properties}
\label{\detokenize{resources:resource-properties}}\label{\detokenize{resources:primitive-resource}}
These values tell the cluster which resource agent to use for the resource,
where to find that resource agent and what standards it conforms to.


\begin{savenotes}\sphinxattablestart
\centering
\sphinxcapstartof{table}
\sphinxcaption{\sphinxstylestrong{Properties of a Primitive Resource}}\label{\detokenize{resources:id4}}
\sphinxaftercaption
\begin{tabular}[t]{|\X{1}{5}|\X{4}{5}|}
\hline
\sphinxstyletheadfamily 
Field
&\sphinxstyletheadfamily 
Description
\\
\hline
id
&
\index{id!resource}\index{resource!property, id}\ignorespaces 
Your name for the resource
\\
\hline
class
&
\index{class!resource}\index{resource!property, class}\ignorespaces 
The standard the resource agent conforms to. Allowed values:
\sphinxcode{\sphinxupquote{lsb}}, \sphinxcode{\sphinxupquote{ocf}}, \sphinxcode{\sphinxupquote{service}}, \sphinxcode{\sphinxupquote{stonith}}, \sphinxcode{\sphinxupquote{systemd}},
\sphinxcode{\sphinxupquote{nagios}} \sphinxstyleemphasis{(deprecated since 2.1.6)}, and \sphinxcode{\sphinxupquote{upstart}}
\sphinxstyleemphasis{(deprecated since 2.1.0)}
\\
\hline
description
&
\index{description!resource}\index{resource!property, description}\ignorespaces 
A description of the Resource Agent, intended for local use.
E.g. \sphinxcode{\sphinxupquote{IP address for website}}
\\
\hline
type
&
\index{type!resource}\index{resource!property, type}\ignorespaces 
The name of the Resource Agent you wish to use. E.g.
\sphinxcode{\sphinxupquote{IPaddr}} or \sphinxcode{\sphinxupquote{Filesystem}}
\\
\hline
provider
&
\index{provider!resource}\index{resource!property, provider}\ignorespaces 
The OCF spec allows multiple vendors to supply the same resource
agent. To use the OCF resource agents supplied by the Heartbeat
project, you would specify \sphinxcode{\sphinxupquote{heartbeat}} here.
\\
\hline
\end{tabular}
\par
\sphinxattableend\end{savenotes}

The XML definition of a resource can be queried with the \sphinxstylestrong{crm\_resource} tool.
For example:

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYGZsh{} crm\PYGZus{}resource \PYGZhy{}\PYGZhy{}resource Email \PYGZhy{}\PYGZhy{}query\PYGZhy{}xml
\end{sphinxVerbatim}

might produce:

\begin{sphinxShadowBox}
\sphinxstyletopictitle{A system resource definition}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}primitive} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}Email\PYGZdq{}} \PYG{n+na}{class=}\PYG{l+s}{\PYGZdq{}service\PYGZdq{}} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}exim\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

\begin{sphinxadmonition}{note}{Note:}
One of the main drawbacks to system services (LSB, systemd or
Upstart) resources is that they do not allow any parameters!
\end{sphinxadmonition}

\begin{sphinxShadowBox}
\sphinxstyletopictitle{An OCF resource definition}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}primitive} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}Public\PYGZhy{}IP\PYGZdq{}} \PYG{n+na}{class=}\PYG{l+s}{\PYGZdq{}ocf\PYGZdq{}} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}IPaddr\PYGZdq{}} \PYG{n+na}{provider=}\PYG{l+s}{\PYGZdq{}heartbeat\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}instance\PYGZus{}attributes} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}Public\PYGZhy{}IP\PYGZhy{}params\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}Public\PYGZhy{}IP\PYGZhy{}ip\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}ip\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}192.0.2.2\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}/instance\PYGZus{}attributes\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/primitive\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}


\subsection{Resource Options}
\label{\detokenize{resources:resource-options}}\label{\detokenize{resources:id3}}
Resources have two types of options: \sphinxstyleemphasis{meta-attributes} and \sphinxstyleemphasis{instance attributes}.
Meta-attributes apply to any type of resource, while instance attributes
are specific to each resource agent.


\subsubsection{Resource Meta-Attributes}
\label{\detokenize{resources:resource-meta-attributes}}
Meta-attributes are used by the cluster to decide how a resource should
behave and can be easily set using the \sphinxcode{\sphinxupquote{-{-}meta}} option of the
\sphinxstylestrong{crm\_resource} command.


\begin{savenotes}\sphinxatlongtablestart\begin{longtable}{|\X{2}{7}|\X{2}{7}|\X{3}{7}|}
\caption{\sphinxstylestrong{Meta-attributes of a Primitive Resource}\strut}\label{\detokenize{resources:id5}}\\*[\sphinxlongtablecapskipadjust]
\hline
\sphinxstyletheadfamily 
Field
&\sphinxstyletheadfamily 
Default
&\sphinxstyletheadfamily 
Description
\\
\hline
\endfirsthead

\multicolumn{3}{c}%
{\makebox[0pt]{\sphinxtablecontinued{\tablename\ \thetable{} -- continued from previous page}}}\\
\hline
\sphinxstyletheadfamily 
Field
&\sphinxstyletheadfamily 
Default
&\sphinxstyletheadfamily 
Description
\\
\hline
\endhead

\hline
\multicolumn{3}{r}{\makebox[0pt][r]{\sphinxtablecontinued{Continued on next page}}}\\
\endfoot

\endlastfoot

priority
&
0
&
\index{priority!resource option}\index{resource!option, priority}\ignorespaces 
If not all resources can be active, the cluster
will stop lower priority resources in order to
keep higher priority ones active.
\\
\hline
critical
&
true
&
\index{critical!resource option}\index{resource!option, critical}\ignorespaces 
Use this value as the default for \sphinxcode{\sphinxupquote{influence}} in
all {\hyperref[\detokenize{constraints:s-resource-colocation}]{\sphinxcrossref{\DUrole{std,std-ref}{colocation constraints}}}} involving this resource,
as well as the implicit colocation constraints
created if this resource is in a
{\hyperref[\detokenize{collective:group-resources}]{\sphinxcrossref{\DUrole{std,std-ref}{group}}}}. For details, see
{\hyperref[\detokenize{constraints:s-coloc-influence}]{\sphinxcrossref{\DUrole{std,std-ref}{Colocation Influence}}}}. \sphinxstyleemphasis{(since 2.1.0)}
\\
\hline
target-role
&
Started
&
\index{target-role!resource option}\index{resource!option, target-role}\ignorespaces 
What state should the cluster attempt to keep this
resource in? Allowed values:
\begin{itemize}
\item {} 
\sphinxcode{\sphinxupquote{Stopped:}} Force the resource to be stopped

\item {} 
\sphinxcode{\sphinxupquote{Started:}} Allow the resource to be started
(and in the case of
{\hyperref[\detokenize{collective:s-resource-promotable}]{\sphinxcrossref{\DUrole{std,std-ref}{promotable}}}} clone
resources, promoted if appropriate)

\item {} 
\sphinxcode{\sphinxupquote{Unpromoted:}} Allow the resource to be started,
but only in the unpromoted role if the resource is
{\hyperref[\detokenize{collective:s-resource-promotable}]{\sphinxcrossref{\DUrole{std,std-ref}{promotable}}}}

\item {} 
\sphinxcode{\sphinxupquote{Promoted:}} Equivalent to \sphinxcode{\sphinxupquote{Started}}

\end{itemize}
\\
\hline
is-managed
&
TRUE
&\phantomsection\label{\detokenize{resources:is-managed}}
\index{is-managed!resource option}\index{resource!option, is-managed}\ignorespaces 
If false, the cluster will not start or stop the
resource on any node. Recurring actions for the
resource are unaffected. Maintenance mode overrides
this setting. Allowed values: \sphinxcode{\sphinxupquote{true}}, \sphinxcode{\sphinxupquote{false}}
\\
\hline
maintenance
&
FALSE
&\phantomsection\label{\detokenize{resources:rsc-maintenance}}
\index{maintenance!resource option}\index{resource!option, maintenance}\ignorespaces 
If true, the cluster will not start or stop the
resource on any node, and will pause any recurring
monitors (except those specifying \sphinxcode{\sphinxupquote{role}} as
\sphinxcode{\sphinxupquote{Stopped}}). If true, the
{\hyperref[\detokenize{cluster-options:maintenance-mode}]{\sphinxcrossref{\DUrole{std,std-ref}{maintenance-mode}}}} cluster
option or {\hyperref[\detokenize{nodes:node-maintenance}]{\sphinxcrossref{\DUrole{std,std-ref}{maintenance}}}}
node attribute override this. Allowed values:
\sphinxcode{\sphinxupquote{true}}, \sphinxcode{\sphinxupquote{false}}
\\
\hline
resource-stickiness
&
1 for individual clone
instances, 0 for all
other resources
&\phantomsection\label{\detokenize{resources:resource-stickiness}}
\index{resource-stickiness!resource option}\index{resource!option, resource-stickiness}\ignorespaces 
A score that will be added to the current node when
a resource is already active. This allows running
resources to stay where they are, even if they
would be placed elsewhere if they were being
started from a stopped state.
\\
\hline
requires
&
\sphinxcode{\sphinxupquote{quorum}} for resources
with a \sphinxcode{\sphinxupquote{class}} of \sphinxcode{\sphinxupquote{stonith}},
otherwise \sphinxcode{\sphinxupquote{unfencing}} if
unfencing is active in the
cluster, otherwise \sphinxcode{\sphinxupquote{fencing}}
if \sphinxcode{\sphinxupquote{stonith-enabled}} is true,
otherwise \sphinxcode{\sphinxupquote{quorum}}
&\phantomsection\label{\detokenize{resources:requires}}
\index{requires!resource option}\index{resource!option, requires}\ignorespaces 
Conditions under which the resource can be
started. Allowed values:
\begin{itemize}
\item {} 
\sphinxcode{\sphinxupquote{nothing:}} can always be started

\item {} 
\sphinxcode{\sphinxupquote{quorum:}} The cluster can only start this
resource if a majority of the configured nodes
are active

\item {} 
\sphinxcode{\sphinxupquote{fencing:}} The cluster can only start this
resource if a majority of the configured nodes
are active \sphinxstyleemphasis{and} any failed or unknown nodes
have been {\hyperref[\detokenize{fencing:fencing}]{\sphinxcrossref{\DUrole{std,std-ref}{fenced}}}}

\item {} 
\sphinxcode{\sphinxupquote{unfencing:}} The cluster can only start this
resource if a majority of the configured nodes
are active \sphinxstyleemphasis{and} any failed or unknown nodes have
been fenced \sphinxstyleemphasis{and} only on nodes that have been
{\hyperref[\detokenize{fencing:unfencing}]{\sphinxcrossref{\DUrole{std,std-ref}{unfenced}}}}

\end{itemize}
\\
\hline
migration-threshold
&
INFINITY
&
\index{migration-threshold!resource option}\index{resource!option, migration-threshold}\ignorespaces 
How many failures may occur for this resource on
a node, before this node is marked ineligible to
host this resource. A value of 0 indicates that this
feature is disabled (the node will never be marked
ineligible); by constrast, the cluster treats
INFINITY (the default) as a very large but finite
number. This option has an effect only if the
failed operation specifies \sphinxcode{\sphinxupquote{on-fail}} as
\sphinxcode{\sphinxupquote{restart}} (the default), and additionally for
failed \sphinxcode{\sphinxupquote{start}} operations, if the cluster
property \sphinxcode{\sphinxupquote{start-failure-is-fatal}} is \sphinxcode{\sphinxupquote{false}}.
\\
\hline
failure-timeout
&
0
&
\index{failure-timeout!resource option}\index{resource!option, failure-timeout}\ignorespaces 
How many seconds to wait before acting as if the
failure had not occurred, and potentially allowing
the resource back to the node on which it failed.
A value of 0 indicates that this feature is
disabled.
\\
\hline
multiple-active
&
stop\_start
&
\index{multiple-active!resource option}\index{resource!option, multiple-active}\ignorespaces 
What should the cluster do if it ever finds the
resource active on more than one node? Allowed
values:
\begin{itemize}
\item {} 
\sphinxcode{\sphinxupquote{block}}: mark the resource as unmanaged

\item {} 
\sphinxcode{\sphinxupquote{stop\_only}}: stop all active instances and
leave them that way

\item {} 
\sphinxcode{\sphinxupquote{stop\_start}}: stop all active instances and
start the resource in one location only

\item {} 
\sphinxcode{\sphinxupquote{stop\_unexpected}}: stop all active instances
except where the resource should be active (this
should be used only when extra instances are not
expected to disrupt existing instances, and the
resource agent’s monitor of an existing instance
is capable of detecting any problems that could be
caused; note that any resources ordered after this
will still need to be restarted) \sphinxstyleemphasis{(since 2.1.3)}

\end{itemize}
\\
\hline
allow-migrate
&
TRUE for ocf:pacemaker:remote
resources, FALSE otherwise
&
Whether the cluster should try to “live migrate”
this resource when it needs to be moved (see
{\hyperref[\detokenize{operations:live-migration}]{\sphinxcrossref{\DUrole{std,std-ref}{Migrating Resources}}}})
\\
\hline
allow-unhealthy-nodes
&
FALSE
&
Whether the resource should be able to run on a node
even if the node’s health score would otherwise
prevent it (see {\hyperref[\detokenize{nodes:node-health}]{\sphinxcrossref{\DUrole{std,std-ref}{Tracking Node Health}}}}) \sphinxstyleemphasis{(since 2.1.3)}
\\
\hline
container-attribute-target
&&
Specific to bundle resources; see
{\hyperref[\detokenize{collective:s-bundle-attributes}]{\sphinxcrossref{\DUrole{std,std-ref}{Bundle Node Attributes}}}}
\\
\hline
remote-node
&&
The name of the Pacemaker Remote guest node this
resource is associated with, if any. If
specified, this both enables the resource as a
guest node and defines the unique name used to
identify the guest node. The guest must be
configured to run the Pacemaker Remote daemon
when it is started. \sphinxstylestrong{WARNING:} This value
cannot overlap with any resource or node IDs.
\\
\hline
remote-port
&
3121
&
If \sphinxcode{\sphinxupquote{remote-node}} is specified, the port on the
guest used for its Pacemaker Remote connection.
The Pacemaker Remote daemon on the guest must
be configured to listen on this port.
\\
\hline
remote-addr
&
value of \sphinxcode{\sphinxupquote{remote-node}}
&
If \sphinxcode{\sphinxupquote{remote-node}} is specified, the IP
address or hostname used to connect to the
guest via Pacemaker Remote. The Pacemaker Remote
daemon on the guest must be configured to accept
connections on this address.
\\
\hline
remote-connect-timeout
&
60s
&
If \sphinxcode{\sphinxupquote{remote-node}} is specified, how long before
a pending guest connection will time out.
\\
\hline
\end{longtable}\sphinxatlongtableend\end{savenotes}

As an example of setting resource options, if you performed the following
commands on an LSB Email resource:

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYGZsh{} crm\PYGZus{}resource \PYGZhy{}\PYGZhy{}meta \PYGZhy{}\PYGZhy{}resource Email \PYGZhy{}\PYGZhy{}set\PYGZhy{}parameter priority \PYGZhy{}\PYGZhy{}parameter\PYGZhy{}value 100
\PYGZsh{} crm\PYGZus{}resource \PYGZhy{}m \PYGZhy{}r Email \PYGZhy{}p multiple\PYGZhy{}active \PYGZhy{}v block
\end{sphinxVerbatim}

the resulting resource definition might be:

\begin{sphinxShadowBox}
\sphinxstyletopictitle{An LSB resource with cluster options}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}primitive} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}Email\PYGZdq{}} \PYG{n+na}{class=}\PYG{l+s}{\PYGZdq{}lsb\PYGZdq{}} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}exim\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}meta\PYGZus{}attributes} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}Email\PYGZhy{}meta\PYGZus{}attributes\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}Email\PYGZhy{}meta\PYGZus{}attributes\PYGZhy{}priority\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}priority\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}100\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}Email\PYGZhy{}meta\PYGZus{}attributes\PYGZhy{}multiple\PYGZhy{}active\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}multiple\PYGZhy{}active\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}block\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}/meta\PYGZus{}attributes\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/primitive\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

In addition to the cluster-defined meta-attributes described above, you may
also configure arbitrary meta-attributes of your own choosing. Most commonly,
this would be done for use in {\hyperref[\detokenize{rules:rules}]{\sphinxcrossref{\DUrole{std,std-ref}{rules}}}}. For example, an IT department
might define a custom meta-attribute to indicate which company department each
resource is intended for. To reduce the chance of name collisions with
cluster-defined meta-attributes added in the future, it is recommended to use
a unique, organization-specific prefix for such attributes.


\subsubsection{Setting Global Defaults for Resource Meta-Attributes}
\label{\detokenize{resources:setting-global-defaults-for-resource-meta-attributes}}\label{\detokenize{resources:s-resource-defaults}}
To set a default value for a resource option, add it to the
\sphinxcode{\sphinxupquote{rsc\_defaults}} section with \sphinxcode{\sphinxupquote{crm\_attribute}}. For example,

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYGZsh{} crm\PYGZus{}attribute \PYGZhy{}\PYGZhy{}type rsc\PYGZus{}defaults \PYGZhy{}\PYGZhy{}name is\PYGZhy{}managed \PYGZhy{}\PYGZhy{}update false
\end{sphinxVerbatim}

would prevent the cluster from starting or stopping any of the
resources in the configuration (unless of course the individual
resources were specifically enabled by having their \sphinxcode{\sphinxupquote{is-managed}} set to
\sphinxcode{\sphinxupquote{true}}).


\subsubsection{Resource Instance Attributes}
\label{\detokenize{resources:resource-instance-attributes}}
The resource agents of some resource classes (lsb, systemd and upstart \sphinxstyleemphasis{not} among them)
can be given parameters which determine how they behave and which instance
of a service they control.

If your resource agent supports parameters, you can add them with the
\sphinxcode{\sphinxupquote{crm\_resource}} command. For example,

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYGZsh{} crm\PYGZus{}resource \PYGZhy{}\PYGZhy{}resource Public\PYGZhy{}IP \PYGZhy{}\PYGZhy{}set\PYGZhy{}parameter ip \PYGZhy{}\PYGZhy{}parameter\PYGZhy{}value 192.0.2.2
\end{sphinxVerbatim}

would create an entry in the resource like this:

\begin{sphinxShadowBox}
\sphinxstyletopictitle{An example OCF resource with instance attributes}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}primitive} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}Public\PYGZhy{}IP\PYGZdq{}} \PYG{n+na}{class=}\PYG{l+s}{\PYGZdq{}ocf\PYGZdq{}} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}IPaddr\PYGZdq{}} \PYG{n+na}{provider=}\PYG{l+s}{\PYGZdq{}heartbeat\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}instance\PYGZus{}attributes} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}params\PYGZhy{}public\PYGZhy{}ip\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}public\PYGZhy{}ip\PYGZhy{}addr\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}ip\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}192.0.2.2\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}/instance\PYGZus{}attributes\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/primitive\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

For an OCF resource, the result would be an environment variable
called \sphinxcode{\sphinxupquote{OCF\_RESKEY\_ip}} with a value of \sphinxcode{\sphinxupquote{192.0.2.2}}.

The list of instance attributes supported by an OCF resource agent can be
found by calling the resource agent with the \sphinxcode{\sphinxupquote{meta-data}} command.
The output contains an XML description of all the supported
attributes, their purpose and default values.

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Displaying the metadata for the Dummy resource agent template}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYGZsh{} export OCF\PYGZus{}ROOT=/usr/lib/ocf
\PYGZsh{} \PYGZdl{}OCF\PYGZus{}ROOT/resource.d/pacemaker/Dummy meta\PYGZhy{}data
\end{sphinxVerbatim}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{c+cp}{\PYGZlt{}?xml version=\PYGZdq{}1.0\PYGZdq{}?\PYGZgt{}}
\PYG{c+cp}{\PYGZlt{}!DOCTYPE resource\PYGZhy{}agent SYSTEM \PYGZdq{}ra\PYGZhy{}api\PYGZhy{}1.dtd\PYGZdq{}\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}resource\PYGZhy{}agent} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}Dummy\PYGZdq{}} \PYG{n+na}{version=}\PYG{l+s}{\PYGZdq{}2.0\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}version}\PYG{n+nt}{\PYGZgt{}}1.1\PYG{n+nt}{\PYGZlt{}/version\PYGZgt{}}

\PYG{n+nt}{\PYGZlt{}longdesc} \PYG{n+na}{lang=}\PYG{l+s}{\PYGZdq{}en\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
This is a dummy OCF resource agent. It does absolutely nothing except keep track
of whether it is running or not, and can be configured so that actions fail or
take a long time. Its purpose is primarily for testing, and to serve as a
template for resource agent writers.
\PYG{n+nt}{\PYGZlt{}/longdesc\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}shortdesc} \PYG{n+na}{lang=}\PYG{l+s}{\PYGZdq{}en\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}Example stateless resource agent\PYG{n+nt}{\PYGZlt{}/shortdesc\PYGZgt{}}

\PYG{n+nt}{\PYGZlt{}parameters}\PYG{n+nt}{\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}parameter} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}state\PYGZdq{}} \PYG{n+na}{unique\PYGZhy{}group=}\PYG{l+s}{\PYGZdq{}state\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}longdesc} \PYG{n+na}{lang=}\PYG{l+s}{\PYGZdq{}en\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
Location to store the resource state in.
\PYG{n+nt}{\PYGZlt{}/longdesc\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}shortdesc} \PYG{n+na}{lang=}\PYG{l+s}{\PYGZdq{}en\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}State file\PYG{n+nt}{\PYGZlt{}/shortdesc\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}content} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}string\PYGZdq{}} \PYG{n+na}{default=}\PYG{l+s}{\PYGZdq{}/var/run/Dummy\PYGZhy{}RESOURCE\PYGZus{}ID.state\PYGZdq{}} \PYG{n+nt}{/\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/parameter\PYGZgt{}}

\PYG{n+nt}{\PYGZlt{}parameter} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}passwd\PYGZdq{}} \PYG{n+na}{reloadable=}\PYG{l+s}{\PYGZdq{}1\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}longdesc} \PYG{n+na}{lang=}\PYG{l+s}{\PYGZdq{}en\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
Fake password field
\PYG{n+nt}{\PYGZlt{}/longdesc\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}shortdesc} \PYG{n+na}{lang=}\PYG{l+s}{\PYGZdq{}en\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}Password\PYG{n+nt}{\PYGZlt{}/shortdesc\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}content} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}string\PYGZdq{}} \PYG{n+na}{default=}\PYG{l+s}{\PYGZdq{}\PYGZdq{}} \PYG{n+nt}{/\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/parameter\PYGZgt{}}

\PYG{n+nt}{\PYGZlt{}parameter} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}fake\PYGZdq{}} \PYG{n+na}{reloadable=}\PYG{l+s}{\PYGZdq{}1\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}longdesc} \PYG{n+na}{lang=}\PYG{l+s}{\PYGZdq{}en\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
Fake attribute that can be changed to cause a reload
\PYG{n+nt}{\PYGZlt{}/longdesc\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}shortdesc} \PYG{n+na}{lang=}\PYG{l+s}{\PYGZdq{}en\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}Fake attribute that can be changed to cause a reload\PYG{n+nt}{\PYGZlt{}/shortdesc\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}content} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}string\PYGZdq{}} \PYG{n+na}{default=}\PYG{l+s}{\PYGZdq{}dummy\PYGZdq{}} \PYG{n+nt}{/\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/parameter\PYGZgt{}}

\PYG{n+nt}{\PYGZlt{}parameter} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}op\PYGZus{}sleep\PYGZdq{}} \PYG{n+na}{reloadable=}\PYG{l+s}{\PYGZdq{}1\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}longdesc} \PYG{n+na}{lang=}\PYG{l+s}{\PYGZdq{}en\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
Number of seconds to sleep during operations.  This can be used to test how
the cluster reacts to operation timeouts.
\PYG{n+nt}{\PYGZlt{}/longdesc\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}shortdesc} \PYG{n+na}{lang=}\PYG{l+s}{\PYGZdq{}en\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}Operation sleep duration in seconds.\PYG{n+nt}{\PYGZlt{}/shortdesc\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}content} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}string\PYGZdq{}} \PYG{n+na}{default=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+nt}{/\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/parameter\PYGZgt{}}

\PYG{n+nt}{\PYGZlt{}parameter} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}fail\PYGZus{}start\PYGZus{}on\PYGZdq{}} \PYG{n+na}{reloadable=}\PYG{l+s}{\PYGZdq{}1\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}longdesc} \PYG{n+na}{lang=}\PYG{l+s}{\PYGZdq{}en\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
Start, migrate\PYGZus{}from, and reload\PYGZhy{}agent actions will return failure if running on
the host specified here, but the resource will run successfully anyway (future
monitor calls will find it running). This can be used to test on\PYGZhy{}fail=ignore.
\PYG{n+nt}{\PYGZlt{}/longdesc\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}shortdesc} \PYG{n+na}{lang=}\PYG{l+s}{\PYGZdq{}en\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}Report bogus start failure on specified host\PYG{n+nt}{\PYGZlt{}/shortdesc\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}content} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}string\PYGZdq{}} \PYG{n+na}{default=}\PYG{l+s}{\PYGZdq{}\PYGZdq{}} \PYG{n+nt}{/\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/parameter\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}parameter} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}envfile\PYGZdq{}} \PYG{n+na}{reloadable=}\PYG{l+s}{\PYGZdq{}1\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}longdesc} \PYG{n+na}{lang=}\PYG{l+s}{\PYGZdq{}en\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
If this is set, the environment will be dumped to this file for every call.
\PYG{n+nt}{\PYGZlt{}/longdesc\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}shortdesc} \PYG{n+na}{lang=}\PYG{l+s}{\PYGZdq{}en\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}Environment dump file\PYG{n+nt}{\PYGZlt{}/shortdesc\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}content} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}string\PYGZdq{}} \PYG{n+na}{default=}\PYG{l+s}{\PYGZdq{}\PYGZdq{}} \PYG{n+nt}{/\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/parameter\PYGZgt{}}

\PYG{n+nt}{\PYGZlt{}/parameters\PYGZgt{}}

\PYG{n+nt}{\PYGZlt{}actions}\PYG{n+nt}{\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}action} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}start\PYGZdq{}}        \PYG{n+na}{timeout=}\PYG{l+s}{\PYGZdq{}20s\PYGZdq{}} \PYG{n+nt}{/\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}action} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}stop\PYGZdq{}}         \PYG{n+na}{timeout=}\PYG{l+s}{\PYGZdq{}20s\PYGZdq{}} \PYG{n+nt}{/\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}action} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}monitor\PYGZdq{}}      \PYG{n+na}{timeout=}\PYG{l+s}{\PYGZdq{}20s\PYGZdq{}} \PYG{n+na}{interval=}\PYG{l+s}{\PYGZdq{}10s\PYGZdq{}} \PYG{n+na}{depth=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}action} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}reload\PYGZdq{}}       \PYG{n+na}{timeout=}\PYG{l+s}{\PYGZdq{}20s\PYGZdq{}} \PYG{n+nt}{/\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}action} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}reload\PYGZhy{}agent\PYGZdq{}} \PYG{n+na}{timeout=}\PYG{l+s}{\PYGZdq{}20s\PYGZdq{}} \PYG{n+nt}{/\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}action} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}migrate\PYGZus{}to\PYGZdq{}}   \PYG{n+na}{timeout=}\PYG{l+s}{\PYGZdq{}20s\PYGZdq{}} \PYG{n+nt}{/\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}action} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}migrate\PYGZus{}from\PYGZdq{}} \PYG{n+na}{timeout=}\PYG{l+s}{\PYGZdq{}20s\PYGZdq{}} \PYG{n+nt}{/\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}action} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}validate\PYGZhy{}all\PYGZdq{}} \PYG{n+na}{timeout=}\PYG{l+s}{\PYGZdq{}20s\PYGZdq{}} \PYG{n+nt}{/\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}action} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}meta\PYGZhy{}data\PYGZdq{}}    \PYG{n+na}{timeout=}\PYG{l+s}{\PYGZdq{}5s\PYGZdq{}} \PYG{n+nt}{/\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/actions\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/resource\PYGZhy{}agent\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

\index{resource!action}\index{resource!operation}\ignorespaces 

\section{Resource Operations}
\label{\detokenize{operations:resource-operations}}\label{\detokenize{operations:operation}}\label{\detokenize{operations:index-0}}\label{\detokenize{operations::doc}}
\sphinxstyleemphasis{Operations} are actions the cluster can perform on a resource by calling the
resource agent. Resource agents must support certain common operations such as
start, stop, and monitor, and may implement any others.

Operations may be explicitly configured for two purposes: to override defaults
for options (such as timeout) that the cluster will use whenever it initiates
the operation, and to run an operation on a recurring basis (for example, to
monitor the resource for failure).

\begin{sphinxShadowBox}
\sphinxstyletopictitle{An OCF resource with a non-default start timeout}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}primitive} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}Public\PYGZhy{}IP\PYGZdq{}} \PYG{n+na}{class=}\PYG{l+s}{\PYGZdq{}ocf\PYGZdq{}} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}IPaddr\PYGZdq{}} \PYG{n+na}{provider=}\PYG{l+s}{\PYGZdq{}heartbeat\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}operations}\PYG{n+nt}{\PYGZgt{}}
     \PYG{n+nt}{\PYGZlt{}op} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}Public\PYGZhy{}IP\PYGZhy{}start\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}start\PYGZdq{}} \PYG{n+na}{timeout=}\PYG{l+s}{\PYGZdq{}60s\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}/operations\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}instance\PYGZus{}attributes} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}params\PYGZhy{}public\PYGZhy{}ip\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
     \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}public\PYGZhy{}ip\PYGZhy{}addr\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}ip\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}192.0.2.2\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}/instance\PYGZus{}attributes\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/primitive\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

Pacemaker identifies operations by a combination of name and interval, so this
combination must be unique for each resource. That is, you should not configure
two operations for the same resource with the same name and interval.


\subsection{Operation Properties}
\label{\detokenize{operations:operation-properties}}\label{\detokenize{operations:id1}}
Operation properties may be specified directly in the \sphinxcode{\sphinxupquote{op}} element as
XML attributes, or in a separate \sphinxcode{\sphinxupquote{meta\_attributes}} block as \sphinxcode{\sphinxupquote{nvpair}} elements.
XML attributes take precedence over \sphinxcode{\sphinxupquote{nvpair}} elements if both are specified.


\begin{savenotes}\sphinxatlongtablestart\begin{longtable}{|\X{1}{6}|\X{2}{6}|\X{3}{6}|}
\caption{\sphinxstylestrong{Properties of an Operation}\strut}\label{\detokenize{operations:id6}}\\*[\sphinxlongtablecapskipadjust]
\hline
\sphinxstyletheadfamily 
Field
&\sphinxstyletheadfamily 
Default
&\sphinxstyletheadfamily 
Description
\\
\hline
\endfirsthead

\multicolumn{3}{c}%
{\makebox[0pt]{\sphinxtablecontinued{\tablename\ \thetable{} -- continued from previous page}}}\\
\hline
\sphinxstyletheadfamily 
Field
&\sphinxstyletheadfamily 
Default
&\sphinxstyletheadfamily 
Description
\\
\hline
\endhead

\hline
\multicolumn{3}{r}{\makebox[0pt][r]{\sphinxtablecontinued{Continued on next page}}}\\
\endfoot

\endlastfoot

id
&&
\index{id!action property}\index{action!property, id}\ignorespaces 
A unique name for the operation.
\\
\hline
name
&&
\index{name!action property}\index{action!property, name}\ignorespaces 
The action to perform. This can be any action
supported by the agent; common values include
\sphinxcode{\sphinxupquote{monitor}}, \sphinxcode{\sphinxupquote{start}}, and \sphinxcode{\sphinxupquote{stop}}.
\\
\hline
interval
&
0
&
\index{interval!action property}\index{action!property, interval}\ignorespaces 
How frequently (in seconds) to perform the
operation. A value of 0 means “when needed”.
A positive value defines a \sphinxstyleemphasis{recurring action},
which is typically used with
{\hyperref[\detokenize{operations:s-resource-monitoring}]{\sphinxcrossref{\DUrole{std,std-ref}{monitor}}}}.
\\
\hline
timeout
&&
\index{timeout!action property}\index{action!property, timeout}\ignorespaces 
How long to wait before declaring the action
has failed
\\
\hline
on-fail
&
Varies by action:
\begin{itemize}
\item {} 
\sphinxcode{\sphinxupquote{stop}}: \sphinxcode{\sphinxupquote{fence}} if
\sphinxcode{\sphinxupquote{stonith-enabled}} is true
or \sphinxcode{\sphinxupquote{block}} otherwise

\item {} 
\sphinxcode{\sphinxupquote{demote}}: \sphinxcode{\sphinxupquote{on-fail}} of the
\sphinxcode{\sphinxupquote{monitor}} action with
\sphinxcode{\sphinxupquote{role}} set to \sphinxcode{\sphinxupquote{Promoted}},
if present, enabled, and
configured to a value other
than \sphinxcode{\sphinxupquote{demote}}, or \sphinxcode{\sphinxupquote{restart}}
otherwise

\item {} 
all other actions: \sphinxcode{\sphinxupquote{restart}}

\end{itemize}
&
\index{on-fail!action property}\index{action!property, on-fail}\ignorespaces 
The action to take if this action ever fails.
Allowed values:
\begin{itemize}
\item {} 
\sphinxcode{\sphinxupquote{ignore:}} Pretend the resource did not fail.

\item {} 
\sphinxcode{\sphinxupquote{block:}} Don’t perform any further operations
on the resource.

\item {} 
\sphinxcode{\sphinxupquote{stop:}} Stop the resource and do not start
it elsewhere.

\item {} 
\sphinxcode{\sphinxupquote{demote:}} Demote the resource, without a
full restart. This is valid only for \sphinxcode{\sphinxupquote{promote}}
actions, and for \sphinxcode{\sphinxupquote{monitor}} actions with both
a nonzero \sphinxcode{\sphinxupquote{interval}} and \sphinxcode{\sphinxupquote{role}} set to
\sphinxcode{\sphinxupquote{Promoted}}; for any other action, a
configuration error will be logged, and the
default behavior will be used. \sphinxstyleemphasis{(since 2.0.5)}

\item {} 
\sphinxcode{\sphinxupquote{restart:}} Stop the resource and start it
again (possibly on a different node).

\item {} 
\sphinxcode{\sphinxupquote{fence:}} STONITH the node on which the
resource failed.

\item {} 
\sphinxcode{\sphinxupquote{standby:}} Move \sphinxstyleemphasis{all} resources away from the
node on which the resource failed.

\end{itemize}
\\
\hline
enabled
&
TRUE
&\phantomsection\label{\detokenize{operations:op-enabled}}
\index{enabled!action property}\index{action!property, enabled}\ignorespaces 
If \sphinxcode{\sphinxupquote{false}}, ignore this operation definition.
This does not suppress all actions of this type,
but is typically used to pause a recurring monitor.
This can complement the resource being unmanaged
({\hyperref[\detokenize{resources:is-managed}]{\sphinxcrossref{\DUrole{std,std-ref}{is-managed}}}} set to \sphinxcode{\sphinxupquote{false}}),
which does not stop recurring operations.
Maintenance mode, which does stop configured this
monitors, overrides this setting. Allowed values:
\sphinxcode{\sphinxupquote{true}}, \sphinxcode{\sphinxupquote{false}}.
\\
\hline
record-pending
&
TRUE
&
\index{record-pending!action property}\index{action!property, record-pending}\ignorespaces 
If \sphinxcode{\sphinxupquote{true}}, the intention to perform the operation
is recorded so that GUIs and CLI tools can indicate
that an operation is in progress.  This is best set
as an \sphinxstyleemphasis{operation default}
(see {\hyperref[\detokenize{operations:s-operation-defaults}]{\sphinxcrossref{\DUrole{std,std-ref}{Setting Global Defaults for Operations}}}}).  Allowed values:
\sphinxcode{\sphinxupquote{true}}, \sphinxcode{\sphinxupquote{false}}.
\\
\hline
role
&&
\index{role!action property}\index{action!property, role}\ignorespaces 
Run the operation only on node(s) that the cluster
thinks should be in the specified role. This only
makes sense for recurring \sphinxcode{\sphinxupquote{monitor}} operations.
Allowed (case-sensitive) values: \sphinxcode{\sphinxupquote{Stopped}},
\sphinxcode{\sphinxupquote{Started}}, and in the case of {\hyperref[\detokenize{collective:s-resource-promotable}]{\sphinxcrossref{\DUrole{std,std-ref}{promotable
clone resources}}}},
\sphinxcode{\sphinxupquote{Unpromoted}} and \sphinxcode{\sphinxupquote{Promoted}}.
\\
\hline
\end{longtable}\sphinxatlongtableend\end{savenotes}

\begin{sphinxadmonition}{note}{Note:}
When \sphinxcode{\sphinxupquote{on-fail}} is set to \sphinxcode{\sphinxupquote{demote}}, recovery from failure by a successful
demote causes the cluster to recalculate whether and where a new instance
should be promoted. The node with the failure is eligible, so if promotion
scores have not changed, it will be promoted again.

There is no direct equivalent of \sphinxcode{\sphinxupquote{migration-threshold}} for the promoted
role, but the same effect can be achieved with a location constraint using a
{\hyperref[\detokenize{rules:rules}]{\sphinxcrossref{\DUrole{std,std-ref}{rule}}}} with a node attribute expression for the resource’s fail
count.

For example, to immediately ban the promoted role from a node with any
failed promote or promoted instance monitor:

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}location} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}loc1\PYGZdq{}} \PYG{n+na}{rsc=}\PYG{l+s}{\PYGZdq{}my\PYGZus{}primitive\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}rule} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}rule1\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}\PYGZhy{}INFINITY\PYGZdq{}} \PYG{n+na}{role=}\PYG{l+s}{\PYGZdq{}Promoted\PYGZdq{}} \PYG{n+na}{boolean\PYGZhy{}op=}\PYG{l+s}{\PYGZdq{}or\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}expression} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}expr1\PYGZdq{}} \PYG{n+na}{attribute=}\PYG{l+s}{\PYGZdq{}fail\PYGZhy{}count\PYGZhy{}my\PYGZus{}primitive\PYGZsh{}promote\PYGZus{}0\PYGZdq{}}
        \PYG{n+na}{operation=}\PYG{l+s}{\PYGZdq{}gte\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}1\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}expression} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}expr2\PYGZdq{}} \PYG{n+na}{attribute=}\PYG{l+s}{\PYGZdq{}fail\PYGZhy{}count\PYGZhy{}my\PYGZus{}primitive\PYGZsh{}monitor\PYGZus{}10000\PYGZdq{}}
        \PYG{n+na}{operation=}\PYG{l+s}{\PYGZdq{}gte\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}1\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/rule\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/rsc\PYGZus{}location\PYGZgt{}}
\end{sphinxVerbatim}

This example assumes that there is a promotable clone of the \sphinxcode{\sphinxupquote{my\_primitive}}
resource (note that the primitive name, not the clone name, is used in the
rule), and that there is a recurring 10-second-interval monitor configured for
the promoted role (fail count attributes specify the interval in
milliseconds).
\end{sphinxadmonition}


\subsection{Monitoring Resources for Failure}
\label{\detokenize{operations:monitoring-resources-for-failure}}\label{\detokenize{operations:s-resource-monitoring}}
When Pacemaker first starts a resource, it runs one-time \sphinxcode{\sphinxupquote{monitor}} operations
(referred to as \sphinxstyleemphasis{probes}) to ensure the resource is running where it’s
supposed to be, and not running where it’s not supposed to be. (This behavior
can be affected by the \sphinxcode{\sphinxupquote{resource-discovery}} location constraint property.)

Other than those initial probes, Pacemaker will \sphinxstyleemphasis{not} (by default) check that
the resource continues to stay healthy %
\begin{footnote}[1]\sphinxAtStartFootnote
Currently, anyway. Automatic monitoring operations may be added in a future
version of Pacemaker.
%
\end{footnote}.  You must configure \sphinxcode{\sphinxupquote{monitor}}
operations explicitly to perform these checks.

\begin{sphinxShadowBox}
\sphinxstyletopictitle{An OCF resource with a recurring health check}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}primitive} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}Public\PYGZhy{}IP\PYGZdq{}} \PYG{n+na}{class=}\PYG{l+s}{\PYGZdq{}ocf\PYGZdq{}} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}IPaddr\PYGZdq{}} \PYG{n+na}{provider=}\PYG{l+s}{\PYGZdq{}heartbeat\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}operations}\PYG{n+nt}{\PYGZgt{}}
     \PYG{n+nt}{\PYGZlt{}op} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}Public\PYGZhy{}IP\PYGZhy{}start\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}start\PYGZdq{}} \PYG{n+na}{timeout=}\PYG{l+s}{\PYGZdq{}60s\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
     \PYG{n+nt}{\PYGZlt{}op} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}Public\PYGZhy{}IP\PYGZhy{}monitor\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}monitor\PYGZdq{}} \PYG{n+na}{interval=}\PYG{l+s}{\PYGZdq{}60s\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}/operations\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}instance\PYGZus{}attributes} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}params\PYGZhy{}public\PYGZhy{}ip\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
     \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}public\PYGZhy{}ip\PYGZhy{}addr\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}ip\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}192.0.2.2\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}/instance\PYGZus{}attributes\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/primitive\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

By default, a \sphinxcode{\sphinxupquote{monitor}} operation will ensure that the resource is running
where it is supposed to. The \sphinxcode{\sphinxupquote{target-role}} property can be used for further
checking.

For example, if a resource has one \sphinxcode{\sphinxupquote{monitor}} operation with
\sphinxcode{\sphinxupquote{interval=10 role=Started}} and a second \sphinxcode{\sphinxupquote{monitor}} operation with
\sphinxcode{\sphinxupquote{interval=11 role=Stopped}}, the cluster will run the first monitor on any nodes
it thinks \sphinxstyleemphasis{should} be running the resource, and the second monitor on any nodes
that it thinks \sphinxstyleemphasis{should not} be running the resource (for the truly paranoid,
who want to know when an administrator manually starts a service by mistake).

\begin{sphinxadmonition}{note}{Note:}
Currently, monitors with \sphinxcode{\sphinxupquote{role=Stopped}} are not implemented for
{\hyperref[\detokenize{collective:s-resource-clone}]{\sphinxcrossref{\DUrole{std,std-ref}{clone}}}} resources.
\end{sphinxadmonition}


\subsection{Setting Global Defaults for Operations}
\label{\detokenize{operations:setting-global-defaults-for-operations}}\label{\detokenize{operations:s-operation-defaults}}
You can change the global default values for operation properties
in a given cluster. These are defined in an \sphinxcode{\sphinxupquote{op\_defaults}} section
of the CIB’s \sphinxcode{\sphinxupquote{configuration}} section, and can be set with
\sphinxcode{\sphinxupquote{crm\_attribute}}.  For example,

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYGZsh{} crm\PYGZus{}attribute \PYGZhy{}\PYGZhy{}type op\PYGZus{}defaults \PYGZhy{}\PYGZhy{}name timeout \PYGZhy{}\PYGZhy{}update 20s
\end{sphinxVerbatim}

would default each operation’s \sphinxcode{\sphinxupquote{timeout}} to 20 seconds.  If an
operation’s definition also includes a value for \sphinxcode{\sphinxupquote{timeout}}, then that
value would be used for that operation instead.


\subsection{When Implicit Operations Take a Long Time}
\label{\detokenize{operations:when-implicit-operations-take-a-long-time}}
The cluster will always perform a number of implicit operations: \sphinxcode{\sphinxupquote{start}},
\sphinxcode{\sphinxupquote{stop}} and a non-recurring \sphinxcode{\sphinxupquote{monitor}} operation used at startup to check
whether the resource is already active.  If one of these is taking too long,
then you can create an entry for them and specify a longer timeout.

\begin{sphinxShadowBox}
\sphinxstyletopictitle{An OCF resource with custom timeouts for its implicit actions}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}primitive} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}Public\PYGZhy{}IP\PYGZdq{}} \PYG{n+na}{class=}\PYG{l+s}{\PYGZdq{}ocf\PYGZdq{}} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}IPaddr\PYGZdq{}} \PYG{n+na}{provider=}\PYG{l+s}{\PYGZdq{}heartbeat\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}operations}\PYG{n+nt}{\PYGZgt{}}
     \PYG{n+nt}{\PYGZlt{}op} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}public\PYGZhy{}ip\PYGZhy{}startup\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}monitor\PYGZdq{}} \PYG{n+na}{interval=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{timeout=}\PYG{l+s}{\PYGZdq{}90s\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
     \PYG{n+nt}{\PYGZlt{}op} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}public\PYGZhy{}ip\PYGZhy{}start\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}start\PYGZdq{}} \PYG{n+na}{interval=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{timeout=}\PYG{l+s}{\PYGZdq{}180s\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
     \PYG{n+nt}{\PYGZlt{}op} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}public\PYGZhy{}ip\PYGZhy{}stop\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}stop\PYGZdq{}} \PYG{n+na}{interval=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{timeout=}\PYG{l+s}{\PYGZdq{}15min\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}/operations\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}instance\PYGZus{}attributes} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}params\PYGZhy{}public\PYGZhy{}ip\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
     \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}public\PYGZhy{}ip\PYGZhy{}addr\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}ip\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}192.0.2.2\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}/instance\PYGZus{}attributes\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/primitive\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}


\subsection{Multiple Monitor Operations}
\label{\detokenize{operations:multiple-monitor-operations}}
Provided no two operations (for a single resource) have the same name
and interval, you can have as many \sphinxcode{\sphinxupquote{monitor}} operations as you like.
In this way, you can do a superficial health check every minute and
progressively more intense ones at higher intervals.

To tell the resource agent what kind of check to perform, you need to
provide each monitor with a different value for a common parameter.
The OCF standard creates a special parameter called \sphinxcode{\sphinxupquote{OCF\_CHECK\_LEVEL}}
for this purpose and dictates that it is “made available to the
resource agent without the normal \sphinxcode{\sphinxupquote{OCF\_RESKEY}} prefix”.

Whatever name you choose, you can specify it by adding an
\sphinxcode{\sphinxupquote{instance\_attributes}} block to the \sphinxcode{\sphinxupquote{op}} tag. It is up to each
resource agent to look for the parameter and decide how to use it.

\begin{sphinxShadowBox}
\sphinxstyletopictitle{An OCF resource with two recurring health checks, performing
        different levels of checks specified via \sphinxstyleliteralintitle{\sphinxupquote{OCF\_CHECK\_LEVEL}}.}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}primitive} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}Public\PYGZhy{}IP\PYGZdq{}} \PYG{n+na}{class=}\PYG{l+s}{\PYGZdq{}ocf\PYGZdq{}} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}IPaddr\PYGZdq{}} \PYG{n+na}{provider=}\PYG{l+s}{\PYGZdq{}heartbeat\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}operations}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}op} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}public\PYGZhy{}ip\PYGZhy{}health\PYGZhy{}60\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}monitor\PYGZdq{}} \PYG{n+na}{interval=}\PYG{l+s}{\PYGZdq{}60\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
         \PYG{n+nt}{\PYGZlt{}instance\PYGZus{}attributes} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}params\PYGZhy{}public\PYGZhy{}ip\PYGZhy{}depth\PYGZhy{}60\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
            \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}public\PYGZhy{}ip\PYGZhy{}depth\PYGZhy{}60\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}OCF\PYGZus{}CHECK\PYGZus{}LEVEL\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}10\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
         \PYG{n+nt}{\PYGZlt{}/instance\PYGZus{}attributes\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}/op\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}op} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}public\PYGZhy{}ip\PYGZhy{}health\PYGZhy{}300\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}monitor\PYGZdq{}} \PYG{n+na}{interval=}\PYG{l+s}{\PYGZdq{}300\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
         \PYG{n+nt}{\PYGZlt{}instance\PYGZus{}attributes} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}params\PYGZhy{}public\PYGZhy{}ip\PYGZhy{}depth\PYGZhy{}300\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
            \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}public\PYGZhy{}ip\PYGZhy{}depth\PYGZhy{}300\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}OCF\PYGZus{}CHECK\PYGZus{}LEVEL\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}20\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
         \PYG{n+nt}{\PYGZlt{}/instance\PYGZus{}attributes\PYGZgt{}}
     \PYG{n+nt}{\PYGZlt{}/op\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}/operations\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}instance\PYGZus{}attributes} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}params\PYGZhy{}public\PYGZhy{}ip\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
       \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}public\PYGZhy{}ip\PYGZhy{}level\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}ip\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}192.0.2.2\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}/instance\PYGZus{}attributes\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/primitive\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}


\subsection{Disabling a Monitor Operation}
\label{\detokenize{operations:disabling-a-monitor-operation}}
The easiest way to stop a recurring monitor is to just delete it.
However, there can be times when you only want to disable it
temporarily.  In such cases, simply add \sphinxcode{\sphinxupquote{enabled=false}} to the
operation’s definition.

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Example of an OCF resource with a disabled health check}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}primitive} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}Public\PYGZhy{}IP\PYGZdq{}} \PYG{n+na}{class=}\PYG{l+s}{\PYGZdq{}ocf\PYGZdq{}} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}IPaddr\PYGZdq{}} \PYG{n+na}{provider=}\PYG{l+s}{\PYGZdq{}heartbeat\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}operations}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}op} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}public\PYGZhy{}ip\PYGZhy{}check\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}monitor\PYGZdq{}} \PYG{n+na}{interval=}\PYG{l+s}{\PYGZdq{}60s\PYGZdq{}} \PYG{n+na}{enabled=}\PYG{l+s}{\PYGZdq{}false\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}/operations\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}instance\PYGZus{}attributes} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}params\PYGZhy{}public\PYGZhy{}ip\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}public\PYGZhy{}ip\PYGZhy{}addr\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}ip\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}192.0.2.2\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}/instance\PYGZus{}attributes\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/primitive\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

This can be achieved from the command line by executing:

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYGZsh{} cibadmin \PYGZhy{}\PYGZhy{}modify \PYGZhy{}\PYGZhy{}xml\PYGZhy{}text \PYGZsq{}\PYGZlt{}op id=\PYGZdq{}public\PYGZhy{}ip\PYGZhy{}check\PYGZdq{} enabled=\PYGZdq{}false\PYGZdq{}/\PYGZgt{}\PYGZsq{}
\end{sphinxVerbatim}

Once you’ve done whatever you needed to do, you can then re-enable it with

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYGZsh{} cibadmin \PYGZhy{}\PYGZhy{}modify \PYGZhy{}\PYGZhy{}xml\PYGZhy{}text \PYGZsq{}\PYGZlt{}op id=\PYGZdq{}public\PYGZhy{}ip\PYGZhy{}check\PYGZdq{} enabled=\PYGZdq{}true\PYGZdq{}/\PYGZgt{}\PYGZsq{}
\end{sphinxVerbatim}

\index{start-delay!operation attribute}\index{interval-origin!operation attribute}\index{interval!interval-origin}\index{operation!interval-origin}\index{operation!start-delay}\ignorespaces 

\subsection{Specifying When Recurring Actions are Performed}
\label{\detokenize{operations:specifying-when-recurring-actions-are-performed}}\label{\detokenize{operations:index-9}}
By default, recurring actions are scheduled relative to when the resource
started. In some cases, you might prefer that a recurring action start relative
to a specific date and time. For example, you might schedule an in-depth
monitor to run once every 24 hours, and want it to run outside business hours.

To do this, set the operation’s \sphinxcode{\sphinxupquote{interval-origin}}. The cluster uses this point
to calculate the correct \sphinxcode{\sphinxupquote{start-delay}} such that the operation will occur
at \sphinxcode{\sphinxupquote{interval-origin}} plus a multiple of the operation interval.

For example, if the recurring operation’s interval is 24h, its
\sphinxcode{\sphinxupquote{interval-origin}} is set to 02:00, and it is currently 14:32, then the
cluster would initiate the operation after 11 hours and 28 minutes.

The value specified for \sphinxcode{\sphinxupquote{interval}} and \sphinxcode{\sphinxupquote{interval-origin}} can be any
date/time conforming to the
\sphinxhref{https://en.wikipedia.org/wiki/ISO\_8601}{ISO8601 standard}. By way of
example, to specify an operation that would run on the first Monday of
2021 and every Monday after that, you would add:

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Example recurring action that runs relative to base date/time}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}op} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}intensive\PYGZhy{}monitor\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}monitor\PYGZdq{}} \PYG{n+na}{interval=}\PYG{l+s}{\PYGZdq{}P7D\PYGZdq{}} \PYG{n+na}{interval\PYGZhy{}origin=}\PYG{l+s}{\PYGZdq{}2021\PYGZhy{}W01\PYGZhy{}1\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

\index{resource!failure recovery}\index{operation!failure recovery}\ignorespaces 

\subsection{Handling Resource Failure}
\label{\detokenize{operations:handling-resource-failure}}\label{\detokenize{operations:failure-handling}}\label{\detokenize{operations:index-10}}
By default, Pacemaker will attempt to recover failed resources by restarting
them. However, failure recovery is highly configurable.

\index{resource!failure count}\index{operation!failure count}\ignorespaces 

\subsubsection{Failure Counts}
\label{\detokenize{operations:failure-counts}}\label{\detokenize{operations:index-11}}
Pacemaker tracks resource failures for each combination of node, resource, and
operation (start, stop, monitor, etc.).

You can query the fail count for a particular node, resource, and/or operation
using the \sphinxcode{\sphinxupquote{crm\_failcount}} command. For example, to see how many times the
10-second monitor for \sphinxcode{\sphinxupquote{myrsc}} has failed on \sphinxcode{\sphinxupquote{node1}}, run:

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYGZsh{} crm\PYGZus{}failcount \PYGZhy{}\PYGZhy{}query \PYGZhy{}r myrsc \PYGZhy{}N node1 \PYGZhy{}n monitor \PYGZhy{}I 10s
\end{sphinxVerbatim}

If you omit the node, \sphinxcode{\sphinxupquote{crm\_failcount}} will use the local node. If you omit
the operation and interval, \sphinxcode{\sphinxupquote{crm\_failcount}} will display the sum of the fail
counts for all operations on the resource.

You can use \sphinxcode{\sphinxupquote{crm\_resource -{-}cleanup}} or \sphinxcode{\sphinxupquote{crm\_failcount -{-}delete}} to clear
fail counts. For example, to clear the above monitor failures, run:

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYGZsh{} crm\PYGZus{}resource \PYGZhy{}\PYGZhy{}cleanup \PYGZhy{}r myrsc \PYGZhy{}N node1 \PYGZhy{}n monitor \PYGZhy{}I 10s
\end{sphinxVerbatim}

If you omit the resource, \sphinxcode{\sphinxupquote{crm\_resource -{-}cleanup}} will clear failures for
all resources. If you omit the node, it will clear failures on all nodes. If
you omit the operation and interval, it will clear the failures for all
operations on the resource.

\begin{sphinxadmonition}{note}{Note:}
Even when cleaning up only a single operation, all failed operations will
disappear from the status display. This allows us to trigger a re-check of
the resource’s current status.
\end{sphinxadmonition}

Higher-level tools may provide other commands for querying and clearing
fail counts.

The \sphinxcode{\sphinxupquote{crm\_mon}} tool shows the current cluster status, including any failed
operations. To see the current fail counts for any failed resources, call
\sphinxcode{\sphinxupquote{crm\_mon}} with the \sphinxcode{\sphinxupquote{-{-}failcounts}} option. This shows the fail counts per
resource (that is, the sum of any operation fail counts for the resource).

\index{migration-threshold!resource meta-attribute}\index{resource!migration-threshold}\ignorespaces 

\subsubsection{Failure Response}
\label{\detokenize{operations:failure-response}}\label{\detokenize{operations:index-12}}
Normally, if a running resource fails, pacemaker will try to stop it and start
it again. Pacemaker will choose the best location to start it each time, which
may be the same node that it failed on.

However, if a resource fails repeatedly, it is possible that there is an
underlying problem on that node, and you might desire trying a different node
in such a case. Pacemaker allows you to set your preference via the
\sphinxcode{\sphinxupquote{migration-threshold}} resource meta-attribute. %
\begin{footnote}[2]\sphinxAtStartFootnote
The naming of this option was perhaps unfortunate as it is easily
confused with live migration, the process of moving a resource from one
node to another without stopping it.  Xen virtual guests are the most
common example of resources that can be migrated in this manner.
%
\end{footnote}

If you define \sphinxcode{\sphinxupquote{migration-threshold}} to \sphinxstyleemphasis{N} for a resource, it will be banned
from the original node after \sphinxstyleemphasis{N} failures there.

\begin{sphinxadmonition}{note}{Note:}
The \sphinxcode{\sphinxupquote{migration-threshold}} is per \sphinxstyleemphasis{resource}, even though fail counts are
tracked per \sphinxstyleemphasis{operation}. The operation fail counts are added together
to compare against the \sphinxcode{\sphinxupquote{migration-threshold}}.
\end{sphinxadmonition}

By default, fail counts remain until manually cleared by an administrator
using \sphinxcode{\sphinxupquote{crm\_resource -{-}cleanup}} or \sphinxcode{\sphinxupquote{crm\_failcount -{-}delete}} (hopefully after
first fixing the failure’s cause). It is possible to have fail counts expire
automatically by setting the \sphinxcode{\sphinxupquote{failure-timeout}} resource meta-attribute.

\begin{sphinxadmonition}{important}{Important:}
A successful operation does not clear past failures. If a recurring monitor
operation fails once, succeeds many times, then fails again days later, its
fail count is 2. Fail counts are cleared only by manual intervention or
failure timeout.
\end{sphinxadmonition}

For example, setting \sphinxcode{\sphinxupquote{migration-threshold}} to 2 and \sphinxcode{\sphinxupquote{failure-timeout}} to
\sphinxcode{\sphinxupquote{60s}} would cause the resource to move to a new node after 2 failures, and
allow it to move back (depending on stickiness and constraint scores) after one
minute.

\begin{sphinxadmonition}{note}{Note:}
\sphinxcode{\sphinxupquote{failure-timeout}} is measured since the most recent failure. That is, older
failures do not individually time out and lower the fail count. Instead, all
failures are timed out simultaneously (and the fail count is reset to 0) if
there is no new failure for the timeout period.
\end{sphinxadmonition}

There are two exceptions to the migration threshold: when a resource either
fails to start or fails to stop.

If the cluster property \sphinxcode{\sphinxupquote{start-failure-is-fatal}} is set to \sphinxcode{\sphinxupquote{true}} (which is
the default), start failures cause the fail count to be set to \sphinxcode{\sphinxupquote{INFINITY}} and
thus always cause the resource to move immediately.

Stop failures are slightly different and crucial.  If a resource fails to stop
and fencing is enabled, then the cluster will fence the node in order to be
able to start the resource elsewhere.  If fencing is disabled, then the cluster
has no way to continue and will not try to start the resource elsewhere, but
will try to stop it again after any failure timeout or clearing.

\index{reload}\index{reload-agent}\ignorespaces 

\subsection{Reloading an Agent After a Definition Change}
\label{\detokenize{operations:reloading-an-agent-after-a-definition-change}}\label{\detokenize{operations:index-13}}
The cluster automatically detects changes to the configuration of active
resources. The cluster’s normal response is to stop the service (using the old
definition) and start it again (with the new definition). This works, but some
resource agents are smarter and can be told to use a new set of options without
restarting.

To take advantage of this capability, the resource agent must:
\begin{itemize}
\item {} 
Implement the \sphinxcode{\sphinxupquote{reload-agent}} action. What it should do depends completely
on your application!

\begin{sphinxadmonition}{note}{Note:}
Resource agents may also implement a \sphinxcode{\sphinxupquote{reload}} action to make the managed
service reload its own \sphinxstyleemphasis{native} configuration. This is different from
\sphinxcode{\sphinxupquote{reload-agent}}, which makes effective changes in the resource’s
\sphinxstyleemphasis{Pacemaker} configuration (specifically, the values of the agent’s
reloadable parameters).
\end{sphinxadmonition}

\item {} 
Advertise the \sphinxcode{\sphinxupquote{reload-agent}} operation in the \sphinxcode{\sphinxupquote{actions}} section of its
meta-data.

\item {} 
Set the \sphinxcode{\sphinxupquote{reloadable}} attribute to 1 in the \sphinxcode{\sphinxupquote{parameters}} section of
its meta-data for any parameters eligible to be reloaded after a change.

\end{itemize}

Once these requirements are satisfied, the cluster will automatically know to
reload the resource (instead of restarting) when a reloadable parameter
changes.

\begin{sphinxadmonition}{note}{Note:}
Metadata will not be re-read unless the resource needs to be started. If you
edit the agent of an already active resource to set a parameter reloadable,
the resource may restart the first time the parameter value changes.
\end{sphinxadmonition}

\begin{sphinxadmonition}{note}{Note:}
If both a reloadable and non-reloadable parameter are changed
simultaneously, the resource will be restarted.
\end{sphinxadmonition}


\subsection{Migrating Resources}
\label{\detokenize{operations:migrating-resources}}\label{\detokenize{operations:live-migration}}
Normally, when the cluster needs to move a resource, it fully restarts the
resource (that is, it stops the resource on the current node and starts it on
the new node).

However, some types of resources, such as many virtual machines, are able to
move to another location without loss of state (often referred to as live
migration or hot migration). In pacemaker, this is called live migration.
Pacemaker can be configured to migrate a resource when moving it, rather than
restarting it.

Not all resources are able to migrate; see the
{\hyperref[\detokenize{operations:migration-checklist}]{\sphinxcrossref{\DUrole{std,std-ref}{migration checklist}}}} below. Even those that can,
won’t do so in all situations. Conceptually, there are two requirements from
which the other prerequisites follow:
\begin{itemize}
\item {} 
The resource must be active and healthy at the old location; and

\item {} 
everything required for the resource to run must be available on both the old
and new locations.

\end{itemize}

The cluster is able to accommodate both \sphinxstyleemphasis{push} and \sphinxstyleemphasis{pull} migration models by
requiring the resource agent to support two special actions: \sphinxcode{\sphinxupquote{migrate\_to}}
(performed on the current location) and \sphinxcode{\sphinxupquote{migrate\_from}} (performed on the
destination).

In push migration, the process on the current location transfers the resource
to the new location where is it later activated. In this scenario, most of the
work would be done in the \sphinxcode{\sphinxupquote{migrate\_to}} action and, if anything, the
activation would occur during \sphinxcode{\sphinxupquote{migrate\_from}}.

Conversely for pull, the \sphinxcode{\sphinxupquote{migrate\_to}} action is practically empty and
\sphinxcode{\sphinxupquote{migrate\_from}} does most of the work, extracting the relevant resource state
from the old location and activating it.

There is no wrong or right way for a resource agent to implement migration, as
long as it works.

\phantomsection\label{\detokenize{operations:migration-checklist}}
\begin{sphinxShadowBox}
\sphinxstyletopictitle{Migration Checklist}
\begin{itemize}
\item {} 
The resource may not be a clone.

\item {} 
The resource agent standard must be OCF.

\item {} 
The resource must not be in a failed or degraded state.

\item {} 
The resource agent must support \sphinxcode{\sphinxupquote{migrate\_to}} and \sphinxcode{\sphinxupquote{migrate\_from}}
actions, and advertise them in its meta-data.

\item {} 
The resource must have the \sphinxcode{\sphinxupquote{allow-migrate}} meta-attribute set to
\sphinxcode{\sphinxupquote{true}} (which is not the default).

\end{itemize}
\end{sphinxShadowBox}

If an otherwise migratable resource depends on another resource via an ordering
constraint, there are special situations in which it will be restarted rather
than migrated.

For example, if the resource depends on a clone, and at the time the resource
needs to be moved, the clone has instances that are stopping and instances that
are starting, then the resource will be restarted. The scheduler is not yet
able to model this situation correctly and so takes the safer (if less optimal)
path.

Also, if a migratable resource depends on a non-migratable resource, and both
need to be moved, the migratable resource will be restarted.
.. rubric:: Footnotes

\index{constraint}\index{resource!constraint}\ignorespaces 

\section{Resource Constraints}
\label{\detokenize{constraints:resource-constraints}}\label{\detokenize{constraints:constraints}}\label{\detokenize{constraints:index-0}}\label{\detokenize{constraints::doc}}\phantomsection\label{\detokenize{constraints:location-constraint}}
\index{location constraint}\index{constraint!location}\ignorespaces 

\subsection{Deciding Which Nodes a Resource Can Run On}
\label{\detokenize{constraints:deciding-which-nodes-a-resource-can-run-on}}\label{\detokenize{constraints:index-1}}
\sphinxstyleemphasis{Location constraints} tell the cluster which nodes a resource can run on.

There are two alternative strategies. One way is to say that, by default,
resources can run anywhere, and then the location constraints specify nodes
that are not allowed (an \sphinxstyleemphasis{opt-out} cluster). The other way is to start with
nothing able to run anywhere, and use location constraints to selectively
enable allowed nodes (an \sphinxstyleemphasis{opt-in} cluster).

Whether you should choose opt-in or opt-out depends on your
personal preference and the make-up of your cluster.  If most of your
resources can run on most of the nodes, then an opt-out arrangement is
likely to result in a simpler configuration.  On the other-hand, if
most resources can only run on a small subset of nodes, an opt-in
configuration might be simpler.

\index{XML element!rsc\_location}\index{rsc\_location!XML element}\index{constraint!rsc\_location}\ignorespaces 

\subsubsection{Location Properties}
\label{\detokenize{constraints:location-properties}}\label{\detokenize{constraints:index-2}}

\begin{savenotes}\sphinxatlongtablestart\begin{longtable}{|\X{1}{6}|\X{1}{6}|\X{4}{6}|}
\caption{\sphinxstylestrong{Attributes of a rsc\_location Element}\strut}\label{\detokenize{constraints:id5}}\\*[\sphinxlongtablecapskipadjust]
\hline
\sphinxstyletheadfamily 
Attribute
&\sphinxstyletheadfamily 
Default
&\sphinxstyletheadfamily 
Description
\\
\hline
\endfirsthead

\multicolumn{3}{c}%
{\makebox[0pt]{\sphinxtablecontinued{\tablename\ \thetable{} -- continued from previous page}}}\\
\hline
\sphinxstyletheadfamily 
Attribute
&\sphinxstyletheadfamily 
Default
&\sphinxstyletheadfamily 
Description
\\
\hline
\endhead

\hline
\multicolumn{3}{r}{\makebox[0pt][r]{\sphinxtablecontinued{Continued on next page}}}\\
\endfoot

\endlastfoot

id
&&
\index{rsc\_location!attribute, id}\index{attribute!id (rsc\_location)}\index{id!rsc\_location attribute}\ignorespaces 
A unique name for the constraint (required)
\\
\hline
rsc
&&
\index{rsc\_location!attribute, rsc}\index{attribute!rsc (rsc\_location)}\index{rsc!rsc\_location attribute}\ignorespaces 
The name of the resource to which this constraint
applies. A location constraint must either have a
\sphinxcode{\sphinxupquote{rsc}}, have a \sphinxcode{\sphinxupquote{rsc-pattern}}, or contain at
least one resource set.
\\
\hline
rsc-pattern
&&
\index{rsc\_location!attribute, rsc-pattern}\index{attribute!rsc-pattern (rsc\_location)}\index{rsc-pattern!rsc\_location attribute}\ignorespaces 
A pattern matching the names of resources to which
this constraint applies.  The syntax is the same as
\sphinxhref{http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1\_chap09.html\#tag\_09\_04}{POSIX}
extended regular expressions, with the addition of an
initial \sphinxcode{\sphinxupquote{!}} indicating that resources \sphinxstyleemphasis{not} matching
the pattern are selected. If the regular expression
contains submatches, and the constraint is governed by
a {\hyperref[\detokenize{rules:rules}]{\sphinxcrossref{\DUrole{std,std-ref}{rule}}}}, the submatches can be
referenced as \sphinxcode{\sphinxupquote{\%1}} through \sphinxcode{\sphinxupquote{\%9}} in the rule’s
\sphinxcode{\sphinxupquote{score-attribute}} or a rule expression’s \sphinxcode{\sphinxupquote{attribute}}
(see {\hyperref[\detokenize{rules:s-rsc-pattern-rules}]{\sphinxcrossref{\DUrole{std,std-ref}{Specifying location scores using pattern submatches}}}}). A location constraint
must either have a \sphinxcode{\sphinxupquote{rsc}}, have a \sphinxcode{\sphinxupquote{rsc-pattern}}, or
contain at least one resource set.
\\
\hline
node
&&
\index{rsc\_location!attribute, node}\index{attribute!node (rsc\_location)}\index{node!rsc\_location attribute}\ignorespaces 
The name of the node to which this constraint applies.
A location constraint must either have a \sphinxcode{\sphinxupquote{node}} and
\sphinxcode{\sphinxupquote{score}}, or contain at least one rule.
\\
\hline
score
&&
\index{rsc\_location!attribute, score}\index{attribute!score (rsc\_location)}\index{score!rsc\_location attribute}\ignorespaces 
Positive values indicate a preference for running the
affected resource(s) on \sphinxcode{\sphinxupquote{node}} \textendash{} the higher the value,
the stronger the preference. Negative values indicate
the resource(s) should avoid this node (a value of
\sphinxstylestrong{-INFINITY} changes “should” to “must”). A location
constraint must either have a \sphinxcode{\sphinxupquote{node}} and \sphinxcode{\sphinxupquote{score}},
or contain at least one rule.
\\
\hline
resource-discovery
&
always
&
\index{rsc\_location!attribute, resource-discovery}\index{attribute!resource-discovery (rsc\_location)}\index{resource-discovery!rsc\_location attribute}\ignorespaces 
Whether Pacemaker should perform resource discovery
(that is, check whether the resource is already running)
for this resource on this node. This should normally be
left as the default, so that rogue instances of a
service can be stopped when they are running where they
are not supposed to be. However, there are two
situations where disabling resource discovery is a good
idea: when a service is not installed on a node,
discovery might return an error (properly written OCF
agents will not, so this is usually only seen with other
agent types); and when Pacemaker Remote is used to scale
a cluster to hundreds of nodes, limiting resource
discovery to allowed nodes can significantly boost
performance.
\begin{itemize}
\item {} 
\sphinxcode{\sphinxupquote{always:}} Always perform resource discovery for
the specified resource on this node.

\item {} 
\sphinxcode{\sphinxupquote{never:}} Never perform resource discovery for the
specified resource on this node.  This option should
generally be used with a -INFINITY score, although
that is not strictly required.

\item {} 
\sphinxcode{\sphinxupquote{exclusive:}} Perform resource discovery for the
specified resource only on this node (and other nodes
similarly marked as \sphinxcode{\sphinxupquote{exclusive}}). Multiple location
constraints using \sphinxcode{\sphinxupquote{exclusive}} discovery for the
same resource across different nodes creates a subset
of nodes resource-discovery is exclusive to.  If a
resource is marked for \sphinxcode{\sphinxupquote{exclusive}} discovery on one
or more nodes, that resource is only allowed to be
placed within that subset of nodes.

\end{itemize}
\\
\hline
\end{longtable}\sphinxatlongtableend\end{savenotes}

\begin{sphinxadmonition}{warning}{Warning:}
Setting \sphinxcode{\sphinxupquote{resource-discovery}} to \sphinxcode{\sphinxupquote{never}} or \sphinxcode{\sphinxupquote{exclusive}} removes Pacemaker’s
ability to detect and stop unwanted instances of a service running
where it’s not supposed to be. It is up to the system administrator (you!)
to make sure that the service can \sphinxstyleemphasis{never} be active on nodes without
\sphinxcode{\sphinxupquote{resource-discovery}} (such as by leaving the relevant software uninstalled).
\end{sphinxadmonition}

\index{Asymmetrical Clusters}\index{Opt-In Clusters}\ignorespaces 

\subsubsection{Asymmetrical “Opt-In” Clusters}
\label{\detokenize{constraints:asymmetrical-opt-in-clusters}}\label{\detokenize{constraints:index-9}}
To create an opt-in cluster, start by preventing resources from running anywhere
by default:

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYGZsh{} crm\PYGZus{}attribute \PYGZhy{}\PYGZhy{}name symmetric\PYGZhy{}cluster \PYGZhy{}\PYGZhy{}update false
\end{sphinxVerbatim}

Then start enabling nodes.  The following fragment says that the web
server prefers \sphinxstylestrong{sles-1}, the database prefers \sphinxstylestrong{sles-2} and both can
fail over to \sphinxstylestrong{sles-3} if their most preferred node fails.

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Opt-in location constraints for two resources}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}constraints}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}location} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}loc\PYGZhy{}1\PYGZdq{}} \PYG{n+na}{rsc=}\PYG{l+s}{\PYGZdq{}Webserver\PYGZdq{}} \PYG{n+na}{node=}\PYG{l+s}{\PYGZdq{}sles\PYGZhy{}1\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}200\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}location} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}loc\PYGZhy{}2\PYGZdq{}} \PYG{n+na}{rsc=}\PYG{l+s}{\PYGZdq{}Webserver\PYGZdq{}} \PYG{n+na}{node=}\PYG{l+s}{\PYGZdq{}sles\PYGZhy{}3\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}location} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}loc\PYGZhy{}3\PYGZdq{}} \PYG{n+na}{rsc=}\PYG{l+s}{\PYGZdq{}Database\PYGZdq{}} \PYG{n+na}{node=}\PYG{l+s}{\PYGZdq{}sles\PYGZhy{}2\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}200\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}location} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}loc\PYGZhy{}4\PYGZdq{}} \PYG{n+na}{rsc=}\PYG{l+s}{\PYGZdq{}Database\PYGZdq{}} \PYG{n+na}{node=}\PYG{l+s}{\PYGZdq{}sles\PYGZhy{}3\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/constraints\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

\index{Symmetrical Clusters}\index{Opt-Out Clusters}\ignorespaces 

\subsubsection{Symmetrical “Opt-Out” Clusters}
\label{\detokenize{constraints:symmetrical-opt-out-clusters}}\label{\detokenize{constraints:index-10}}
To create an opt-out cluster, start by allowing resources to run
anywhere by default:

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYGZsh{} crm\PYGZus{}attribute \PYGZhy{}\PYGZhy{}name symmetric\PYGZhy{}cluster \PYGZhy{}\PYGZhy{}update true
\end{sphinxVerbatim}

Then start disabling nodes.  The following fragment is the equivalent
of the above opt-in configuration.

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Opt-out location constraints for two resources}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}constraints}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}location} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}loc\PYGZhy{}1\PYGZdq{}} \PYG{n+na}{rsc=}\PYG{l+s}{\PYGZdq{}Webserver\PYGZdq{}} \PYG{n+na}{node=}\PYG{l+s}{\PYGZdq{}sles\PYGZhy{}1\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}200\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}location} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}loc\PYGZhy{}2\PYGZhy{}do\PYGZhy{}not\PYGZhy{}run\PYGZdq{}} \PYG{n+na}{rsc=}\PYG{l+s}{\PYGZdq{}Webserver\PYGZdq{}} \PYG{n+na}{node=}\PYG{l+s}{\PYGZdq{}sles\PYGZhy{}2\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}\PYGZhy{}INFINITY\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}location} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}loc\PYGZhy{}3\PYGZhy{}do\PYGZhy{}not\PYGZhy{}run\PYGZdq{}} \PYG{n+na}{rsc=}\PYG{l+s}{\PYGZdq{}Database\PYGZdq{}} \PYG{n+na}{node=}\PYG{l+s}{\PYGZdq{}sles\PYGZhy{}1\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}\PYGZhy{}INFINITY\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}location} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}loc\PYGZhy{}4\PYGZdq{}} \PYG{n+na}{rsc=}\PYG{l+s}{\PYGZdq{}Database\PYGZdq{}} \PYG{n+na}{node=}\PYG{l+s}{\PYGZdq{}sles\PYGZhy{}2\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}200\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/constraints\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}


\subsubsection{What if Two Nodes Have the Same Score}
\label{\detokenize{constraints:what-if-two-nodes-have-the-same-score}}\label{\detokenize{constraints:node-score-equal}}
If two nodes have the same score, then the cluster will choose one.
This choice may seem random and may not be what was intended, however
the cluster was not given enough information to know any better.

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Constraints where a resource prefers two nodes equally}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}constraints}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}location} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}loc\PYGZhy{}1\PYGZdq{}} \PYG{n+na}{rsc=}\PYG{l+s}{\PYGZdq{}Webserver\PYGZdq{}} \PYG{n+na}{node=}\PYG{l+s}{\PYGZdq{}sles\PYGZhy{}1\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}INFINITY\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}location} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}loc\PYGZhy{}2\PYGZdq{}} \PYG{n+na}{rsc=}\PYG{l+s}{\PYGZdq{}Webserver\PYGZdq{}} \PYG{n+na}{node=}\PYG{l+s}{\PYGZdq{}sles\PYGZhy{}2\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}INFINITY\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}location} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}loc\PYGZhy{}3\PYGZdq{}} \PYG{n+na}{rsc=}\PYG{l+s}{\PYGZdq{}Database\PYGZdq{}} \PYG{n+na}{node=}\PYG{l+s}{\PYGZdq{}sles\PYGZhy{}1\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}500\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}location} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}loc\PYGZhy{}4\PYGZdq{}} \PYG{n+na}{rsc=}\PYG{l+s}{\PYGZdq{}Database\PYGZdq{}} \PYG{n+na}{node=}\PYG{l+s}{\PYGZdq{}sles\PYGZhy{}2\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}300\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}location} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}loc\PYGZhy{}5\PYGZdq{}} \PYG{n+na}{rsc=}\PYG{l+s}{\PYGZdq{}Database\PYGZdq{}} \PYG{n+na}{node=}\PYG{l+s}{\PYGZdq{}sles\PYGZhy{}2\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}200\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/constraints\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

In the example above, assuming no other constraints and an inactive
cluster, \sphinxstylestrong{Webserver} would probably be placed on \sphinxstylestrong{sles-1} and \sphinxstylestrong{Database} on
\sphinxstylestrong{sles-2}.  It would likely have placed \sphinxstylestrong{Webserver} based on the node’s
uname and \sphinxstylestrong{Database} based on the desire to spread the resource load
evenly across the cluster.  However other factors can also be involved
in more complex configurations.


\subsubsection{Specifying locations using pattern matching}
\label{\detokenize{constraints:specifying-locations-using-pattern-matching}}\label{\detokenize{constraints:s-rsc-pattern}}
A location constraint can affect all resources whose IDs match a given pattern.
The following example bans resources named \sphinxstylestrong{ip-httpd}, \sphinxstylestrong{ip-asterisk},
\sphinxstylestrong{ip-gateway}, etc., from \sphinxstylestrong{node1}.

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Location constraint banning all resources matching a pattern from one node}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}constraints}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}location} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}ban\PYGZhy{}ips\PYGZhy{}from\PYGZhy{}node1\PYGZdq{}} \PYG{n+na}{rsc\PYGZhy{}pattern=}\PYG{l+s}{\PYGZdq{}ip\PYGZhy{}.*\PYGZdq{}} \PYG{n+na}{node=}\PYG{l+s}{\PYGZdq{}node1\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}\PYGZhy{}INFINITY\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/constraints\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

\index{constraint!ordering}\index{resource!start order}\ignorespaces 

\subsection{Specifying the Order in which Resources Should Start/Stop}
\label{\detokenize{constraints:specifying-the-order-in-which-resources-should-start-stop}}\label{\detokenize{constraints:s-resource-ordering}}\label{\detokenize{constraints:index-11}}
\sphinxstyleemphasis{Ordering constraints} tell the cluster the order in which certain
resource actions should occur.

\begin{sphinxadmonition}{important}{Important:}
Ordering constraints affect \sphinxstyleemphasis{only} the ordering of resource actions;
they do \sphinxstyleemphasis{not} require that the resources be placed on the
same node. If you want resources to be started on the same node
\sphinxstyleemphasis{and} in a specific order, you need both an ordering constraint \sphinxstyleemphasis{and}
a colocation constraint (see {\hyperref[\detokenize{constraints:s-resource-colocation}]{\sphinxcrossref{\DUrole{std,std-ref}{Placing Resources Relative to other Resources}}}}), or
alternatively, a group (see {\hyperref[\detokenize{collective:group-resources}]{\sphinxcrossref{\DUrole{std,std-ref}{Groups - A Syntactic Shortcut}}}}).
\end{sphinxadmonition}

\index{XML element!rsc\_order}\index{rsc\_order!XML element}\index{constraint!rsc\_order}\index{rsc\_order!constraint}\ignorespaces 

\subsubsection{Ordering Properties}
\label{\detokenize{constraints:ordering-properties}}\label{\detokenize{constraints:index-12}}

\begin{savenotes}\sphinxatlongtablestart\begin{longtable}{|\X{1}{7}|\X{2}{7}|\X{4}{7}|}
\caption{\sphinxstylestrong{Attributes of a rsc\_order Element}\strut}\label{\detokenize{constraints:id6}}\\*[\sphinxlongtablecapskipadjust]
\hline
\sphinxstyletheadfamily 
Field
&\sphinxstyletheadfamily 
Default
&\sphinxstyletheadfamily 
Description
\\
\hline
\endfirsthead

\multicolumn{3}{c}%
{\makebox[0pt]{\sphinxtablecontinued{\tablename\ \thetable{} -- continued from previous page}}}\\
\hline
\sphinxstyletheadfamily 
Field
&\sphinxstyletheadfamily 
Default
&\sphinxstyletheadfamily 
Description
\\
\hline
\endhead

\hline
\multicolumn{3}{r}{\makebox[0pt][r]{\sphinxtablecontinued{Continued on next page}}}\\
\endfoot

\endlastfoot

id
&&
\index{rsc\_order!attribute, id}\index{attribute!id (rsc\_order)}\index{id!rsc\_order attribute}\ignorespaces 
A unique name for the constraint
\\
\hline
first
&&
\index{rsc\_order!attribute, first}\index{attribute!first (rsc\_order)}\index{first!rsc\_order attribute}\ignorespaces 
Name of the resource that the \sphinxcode{\sphinxupquote{then}} resource
depends on
\\
\hline
then
&&
\index{rsc\_order!attribute, then}\index{attribute!then (rsc\_order)}\index{then!rsc\_order attribute}\ignorespaces 
Name of the dependent resource
\\
\hline
first-action
&
start
&
\index{rsc\_order!attribute, first-action}\index{attribute!first-action (rsc\_order)}\index{first-action!rsc\_order attribute}\ignorespaces 
The action that the \sphinxcode{\sphinxupquote{first}} resource must complete
before \sphinxcode{\sphinxupquote{then-action}} can be initiated for the \sphinxcode{\sphinxupquote{then}}
resource.  Allowed values: \sphinxcode{\sphinxupquote{start}}, \sphinxcode{\sphinxupquote{stop}},
\sphinxcode{\sphinxupquote{promote}}, \sphinxcode{\sphinxupquote{demote}}.
\\
\hline
then-action
&
value of \sphinxcode{\sphinxupquote{first-action}}
&
\index{rsc\_order!attribute, then-action}\index{attribute!then-action (rsc\_order)}\index{first-action!rsc\_order attribute}\ignorespaces 
The action that the \sphinxcode{\sphinxupquote{then}} resource can execute only
after the \sphinxcode{\sphinxupquote{first-action}} on the \sphinxcode{\sphinxupquote{first}} resource has
completed.  Allowed values: \sphinxcode{\sphinxupquote{start}}, \sphinxcode{\sphinxupquote{stop}},
\sphinxcode{\sphinxupquote{promote}}, \sphinxcode{\sphinxupquote{demote}}.
\\
\hline
kind
&
Mandatory
&
\index{rsc\_order!attribute, kind}\index{attribute!kind (rsc\_order)}\index{kind!rsc\_order attribute}\ignorespaces 
How to enforce the constraint. Allowed values:
\begin{itemize}
\item {} 
\sphinxcode{\sphinxupquote{Mandatory:}} \sphinxcode{\sphinxupquote{then-action}} will never be initiated
for the \sphinxcode{\sphinxupquote{then}} resource unless and until \sphinxcode{\sphinxupquote{first-action}}
successfully completes for the \sphinxcode{\sphinxupquote{first}} resource.

\item {} 
\sphinxcode{\sphinxupquote{Optional:}} The constraint applies only if both specified
resource actions are scheduled in the same transition
(that is, in response to the same cluster state). This
means that \sphinxcode{\sphinxupquote{then-action}} is allowed on the \sphinxcode{\sphinxupquote{then}}
resource regardless of the state of the \sphinxcode{\sphinxupquote{first}} resource,
but if both actions happen to be scheduled at the same time,
they will be ordered.

\item {} 
\sphinxcode{\sphinxupquote{Serialize:}} Ensure that the specified actions are never
performed concurrently for the specified resources.
\sphinxcode{\sphinxupquote{First-action}} and \sphinxcode{\sphinxupquote{then-action}} can be executed in either
order, but one must complete before the other can be initiated.
An example use case is when resource start-up puts a high load
on the host.

\end{itemize}
\\
\hline
symmetrical
&
TRUE for \sphinxcode{\sphinxupquote{Mandatory}} and
\sphinxcode{\sphinxupquote{Optional}} kinds. FALSE
for \sphinxcode{\sphinxupquote{Serialize}} kind.
&
\index{rsc\_order!attribute, symmetrical}\index{attribute!symmetrical (rsc)order)}\index{symmetrical!rsc\_order attribute}\ignorespaces 
If true, the reverse of the constraint applies for the
opposite action (for example, if B starts after A starts,
then B stops before A stops).  \sphinxcode{\sphinxupquote{Serialize}} orders cannot
be symmetrical.
\\
\hline
\end{longtable}\sphinxatlongtableend\end{savenotes}

\sphinxcode{\sphinxupquote{Promote}} and \sphinxcode{\sphinxupquote{demote}} apply to {\hyperref[\detokenize{collective:s-resource-promotable}]{\sphinxcrossref{\DUrole{std,std-ref}{promotable}}}}
clone resources.


\subsubsection{Optional and mandatory ordering}
\label{\detokenize{constraints:optional-and-mandatory-ordering}}
Here is an example of ordering constraints where \sphinxstylestrong{Database} \sphinxstyleemphasis{must} start before
\sphinxstylestrong{Webserver}, and \sphinxstylestrong{IP} \sphinxstyleemphasis{should} start before \sphinxstylestrong{Webserver} if they both need to be
started:

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Optional and mandatory ordering constraints}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}constraints}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}order} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}order\PYGZhy{}1\PYGZdq{}} \PYG{n+na}{first=}\PYG{l+s}{\PYGZdq{}IP\PYGZdq{}} \PYG{n+na}{then=}\PYG{l+s}{\PYGZdq{}Webserver\PYGZdq{}} \PYG{n+na}{kind=}\PYG{l+s}{\PYGZdq{}Optional\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}order} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}order\PYGZhy{}2\PYGZdq{}} \PYG{n+na}{first=}\PYG{l+s}{\PYGZdq{}Database\PYGZdq{}} \PYG{n+na}{then=}\PYG{l+s}{\PYGZdq{}Webserver\PYGZdq{}} \PYG{n+na}{kind=}\PYG{l+s}{\PYGZdq{}Mandatory\PYGZdq{}} \PYG{n+nt}{/\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/constraints\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

Because the above example lets \sphinxcode{\sphinxupquote{symmetrical}} default to TRUE, \sphinxstylestrong{Webserver}
must be stopped before \sphinxstylestrong{Database} can be stopped, and \sphinxstylestrong{Webserver} should be
stopped before \sphinxstylestrong{IP} if they both need to be stopped.


\subsubsection{Symmetric and asymmetric ordering}
\label{\detokenize{constraints:symmetric-and-asymmetric-ordering}}
A mandatory symmetric ordering of “start A then start B” implies not only that
the start actions must be ordered, but that B is not allowed to be active
unless A is active. For example, if the ordering is added to the configuration
when A is stopped (due to target-role, failure, etc.) and B is already active,
then B will be stopped.

By contrast, asymmetric ordering of “start A then start B” means the stops can
occur in either order, which implies that B \sphinxstyleemphasis{can} remain active in the same
situation.

\index{colocation}\index{constraint!colocation}\index{resource!location relative to other resources}\ignorespaces 

\subsection{Placing Resources Relative to other Resources}
\label{\detokenize{constraints:placing-resources-relative-to-other-resources}}\label{\detokenize{constraints:s-resource-colocation}}\label{\detokenize{constraints:index-20}}
\sphinxstyleemphasis{Colocation constraints} tell the cluster that the location of one resource
depends on the location of another one.

Colocation has an important side-effect: it affects the order in which
resources are assigned to a node. Think about it: You can’t place A relative to
B unless you know where B is %
\begin{footnote}[1]\sphinxAtStartFootnote
While the human brain is sophisticated enough to read the constraint
in any order and choose the correct one depending on the situation,
the cluster is not quite so smart. Yet.
%
\end{footnote}.

So when you are creating colocation constraints, it is important to
consider whether you should colocate A with B, or B with A.

\begin{sphinxadmonition}{important}{Important:}
Colocation constraints affect \sphinxstyleemphasis{only} the placement of resources; they do \sphinxstyleemphasis{not}
require that the resources be started in a particular order. If you want
resources to be started on the same node \sphinxstyleemphasis{and} in a specific order, you need
both an ordering constraint (see {\hyperref[\detokenize{constraints:s-resource-ordering}]{\sphinxcrossref{\DUrole{std,std-ref}{Specifying the Order in which Resources Should Start/Stop}}}}) \sphinxstyleemphasis{and} a colocation
constraint, or alternatively, a group (see {\hyperref[\detokenize{collective:group-resources}]{\sphinxcrossref{\DUrole{std,std-ref}{Groups - A Syntactic Shortcut}}}}).
\end{sphinxadmonition}

\index{XML element!rsc\_colocation}\index{rsc\_colocation!XML element}\index{constraint!rsc\_colocation}\ignorespaces 

\subsubsection{Colocation Properties}
\label{\detokenize{constraints:colocation-properties}}\label{\detokenize{constraints:index-21}}

\begin{savenotes}\sphinxatlongtablestart\begin{longtable}{|\X{2}{9}|\X{2}{9}|\X{5}{9}|}
\caption{\sphinxstylestrong{Attributes of a rsc\_colocation Constraint}\strut}\label{\detokenize{constraints:id7}}\\*[\sphinxlongtablecapskipadjust]
\hline
\sphinxstyletheadfamily 
Field
&\sphinxstyletheadfamily 
Default
&\sphinxstyletheadfamily 
Description
\\
\hline
\endfirsthead

\multicolumn{3}{c}%
{\makebox[0pt]{\sphinxtablecontinued{\tablename\ \thetable{} -- continued from previous page}}}\\
\hline
\sphinxstyletheadfamily 
Field
&\sphinxstyletheadfamily 
Default
&\sphinxstyletheadfamily 
Description
\\
\hline
\endhead

\hline
\multicolumn{3}{r}{\makebox[0pt][r]{\sphinxtablecontinued{Continued on next page}}}\\
\endfoot

\endlastfoot

id
&&
\index{rsc\_colocation!attribute, id}\index{attribute!id (rsc\_colocation)}\index{id!rsc\_colocation attribute}\ignorespaces 
A unique name for the constraint (required).
\\
\hline
rsc
&&
\index{rsc\_colocation!attribute, rsc}\index{attribute!rsc (rsc\_colocation)}\index{rsc!rsc\_colocation attribute}\ignorespaces 
The name of a resource that should be located
relative to \sphinxcode{\sphinxupquote{with-rsc}}. A colocation constraint must
either contain at least one
{\hyperref[\detokenize{constraints:s-resource-sets}]{\sphinxcrossref{\DUrole{std,std-ref}{resource set}}}}, or specify both
\sphinxcode{\sphinxupquote{rsc}} and \sphinxcode{\sphinxupquote{with-rsc}}.
\\
\hline
with-rsc
&&
\index{rsc\_colocation!attribute, with-rsc}\index{attribute!with-rsc (rsc\_colocation)}\index{with-rsc!rsc\_colocation attribute}\ignorespaces 
The name of the resource used as the colocation
target. The cluster will decide where to put this
resource first and then decide where to put \sphinxcode{\sphinxupquote{rsc}}.
A colocation constraint must either contain at least
one {\hyperref[\detokenize{constraints:s-resource-sets}]{\sphinxcrossref{\DUrole{std,std-ref}{resource set}}}}, or specify
both \sphinxcode{\sphinxupquote{rsc}} and \sphinxcode{\sphinxupquote{with-rsc}}.
\\
\hline
node-attribute
&
\#uname
&
\index{rsc\_colocation!attribute, node-attribute}\index{attribute!node-attribute (rsc\_colocation)}\index{node-attribute!rsc\_colocation attribute}\ignorespaces 
If \sphinxcode{\sphinxupquote{rsc}} and \sphinxcode{\sphinxupquote{with-rsc}} are specified, this node
attribute must be the same on the node running \sphinxcode{\sphinxupquote{rsc}}
and the node running \sphinxcode{\sphinxupquote{with-rsc}} for the constraint
to be satisfied. (For details, see
{\hyperref[\detokenize{constraints:s-coloc-attribute}]{\sphinxcrossref{\DUrole{std,std-ref}{Colocation by Node Attribute}}}}.)
\\
\hline
score
&
0
&
\index{rsc\_colocation!attribute, score}\index{attribute!score (rsc\_colocation)}\index{score!rsc\_colocation attribute}\ignorespaces 
Positive values indicate the resources should run on
the same node. Negative values indicate the resources
should run on different nodes. Values of
+/- \sphinxcode{\sphinxupquote{INFINITY}} change “should” to “must”.
\\
\hline
rsc-role
&
Started
&
\index{clone!ordering constraint, rsc-role}\index{ordering constraint!rsc-role (clone)}\index{rsc-role!clone ordering constraint}\ignorespaces 
If \sphinxcode{\sphinxupquote{rsc}} and \sphinxcode{\sphinxupquote{with-rsc}} are specified, and \sphinxcode{\sphinxupquote{rsc}}
is a {\hyperref[\detokenize{collective:s-resource-promotable}]{\sphinxcrossref{\DUrole{std,std-ref}{promotable clone}}}},
the constraint applies only to \sphinxcode{\sphinxupquote{rsc}} instances in
this role. Allowed values: \sphinxcode{\sphinxupquote{Started}}, \sphinxcode{\sphinxupquote{Stopped}},
\sphinxcode{\sphinxupquote{Promoted}}, \sphinxcode{\sphinxupquote{Unpromoted}}. For details, see
{\hyperref[\detokenize{collective:promotable-clone-constraints}]{\sphinxcrossref{\DUrole{std,std-ref}{Promotable Clone Constraints}}}}.
\\
\hline
with-rsc-role
&
Started
&
\index{clone!ordering constraint, with-rsc-role}\index{ordering constraint!with-rsc-role (clone)}\index{with-rsc-role!clone ordering constraint}\ignorespaces 
If \sphinxcode{\sphinxupquote{rsc}} and \sphinxcode{\sphinxupquote{with-rsc}} are specified, and
\sphinxcode{\sphinxupquote{with-rsc}} is a
{\hyperref[\detokenize{collective:s-resource-promotable}]{\sphinxcrossref{\DUrole{std,std-ref}{promotable clone}}}}, the
constraint applies only to \sphinxcode{\sphinxupquote{with-rsc}} instances in
this role. Allowed values: \sphinxcode{\sphinxupquote{Started}}, \sphinxcode{\sphinxupquote{Stopped}},
\sphinxcode{\sphinxupquote{Promoted}}, \sphinxcode{\sphinxupquote{Unpromoted}}. For details, see
{\hyperref[\detokenize{collective:promotable-clone-constraints}]{\sphinxcrossref{\DUrole{std,std-ref}{Promotable Clone Constraints}}}}.
\\
\hline
influence
&
value of
\sphinxcode{\sphinxupquote{critical}}
meta-attribute
for \sphinxcode{\sphinxupquote{rsc}}
&
\index{rsc\_colocation!attribute, influence}\index{attribute!influence (rsc\_colocation)}\index{influence!rsc\_colocation attribute}\ignorespaces 
Whether to consider the location preferences of
\sphinxcode{\sphinxupquote{rsc}} when \sphinxcode{\sphinxupquote{with-rsc}} is already active. Allowed
values: \sphinxcode{\sphinxupquote{true}}, \sphinxcode{\sphinxupquote{false}}. For details, see
{\hyperref[\detokenize{constraints:s-coloc-influence}]{\sphinxcrossref{\DUrole{std,std-ref}{Colocation Influence}}}}. \sphinxstyleemphasis{(since 2.1.0)}
\\
\hline
\end{longtable}\sphinxatlongtableend\end{savenotes}


\subsubsection{Mandatory Placement}
\label{\detokenize{constraints:mandatory-placement}}
Mandatory placement occurs when the constraint’s score is
\sphinxstylestrong{+INFINITY} or \sphinxstylestrong{-INFINITY}.  In such cases, if the constraint can’t be
satisfied, then the \sphinxstylestrong{rsc} resource is not permitted to run.  For
\sphinxcode{\sphinxupquote{score=INFINITY}}, this includes cases where the \sphinxcode{\sphinxupquote{with-rsc}} resource is
not active.

If you need resource \sphinxstylestrong{A} to always run on the same machine as
resource \sphinxstylestrong{B}, you would add the following constraint:

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Mandatory colocation constraint for two resources}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}colocation} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}colocate\PYGZdq{}} \PYG{n+na}{rsc=}\PYG{l+s}{\PYGZdq{}A\PYGZdq{}} \PYG{n+na}{with\PYGZhy{}rsc=}\PYG{l+s}{\PYGZdq{}B\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}INFINITY\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

Remember, because \sphinxstylestrong{INFINITY} was used, if \sphinxstylestrong{B} can’t run on any
of the cluster nodes (for whatever reason) then \sphinxstylestrong{A} will not
be allowed to run. Whether \sphinxstylestrong{A} is running or not has no effect on \sphinxstylestrong{B}.

Alternatively, you may want the opposite \textendash{} that \sphinxstylestrong{A} \sphinxstyleemphasis{cannot}
run on the same machine as \sphinxstylestrong{B}.  In this case, use \sphinxcode{\sphinxupquote{score="-INFINITY"}}.

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Mandatory anti-colocation constraint for two resources}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}colocation} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}anti\PYGZhy{}colocate\PYGZdq{}} \PYG{n+na}{rsc=}\PYG{l+s}{\PYGZdq{}A\PYGZdq{}} \PYG{n+na}{with\PYGZhy{}rsc=}\PYG{l+s}{\PYGZdq{}B\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}\PYGZhy{}INFINITY\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

Again, by specifying \sphinxstylestrong{-INFINITY}, the constraint is binding.  So if the
only place left to run is where \sphinxstylestrong{B} already is, then \sphinxstylestrong{A} may not run anywhere.

As with \sphinxstylestrong{INFINITY}, \sphinxstylestrong{B} can run even if \sphinxstylestrong{A} is stopped.  However, in this
case \sphinxstylestrong{A} also can run if \sphinxstylestrong{B} is stopped, because it still meets the
constraint of \sphinxstylestrong{A} and \sphinxstylestrong{B} not running on the same node.


\subsubsection{Advisory Placement}
\label{\detokenize{constraints:advisory-placement}}
If mandatory placement is about “must” and “must not”, then advisory
placement is the “I’d prefer if” alternative.

For colocation constraints with scores greater than \sphinxstylestrong{-INFINITY} and less than
\sphinxstylestrong{INFINITY}, the cluster will try to accommodate your wishes, but may ignore
them if other factors outweigh the colocation score. Those factors might
include other constraints, resource stickiness, failure thresholds, whether
other resources would be prevented from being active, etc.

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Advisory colocation constraint for two resources}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}colocation} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}colocate\PYGZhy{}maybe\PYGZdq{}} \PYG{n+na}{rsc=}\PYG{l+s}{\PYGZdq{}A\PYGZdq{}} \PYG{n+na}{with\PYGZhy{}rsc=}\PYG{l+s}{\PYGZdq{}B\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}500\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}


\subsubsection{Colocation by Node Attribute}
\label{\detokenize{constraints:colocation-by-node-attribute}}\label{\detokenize{constraints:s-coloc-attribute}}
The \sphinxcode{\sphinxupquote{node-attribute}} property of a colocation constraints allows you to express
the requirement, “these resources must be on similar nodes”.

As an example, imagine that you have two Storage Area Networks (SANs) that are
not controlled by the cluster, and each node is connected to one or the other.
You may have two resources \sphinxstylestrong{r1} and \sphinxstylestrong{r2} such that \sphinxstylestrong{r2} needs to use the same
SAN as \sphinxstylestrong{r1}, but doesn’t necessarily have to be on the same exact node.
In such a case, you could define a {\hyperref[\detokenize{nodes:node-attributes}]{\sphinxcrossref{\DUrole{std,std-ref}{node attribute}}}} named
\sphinxstylestrong{san}, with the value \sphinxstylestrong{san1} or \sphinxstylestrong{san2} on each node as appropriate. Then, you
could colocate \sphinxstylestrong{r2} with \sphinxstylestrong{r1} using \sphinxcode{\sphinxupquote{node-attribute}} set to \sphinxstylestrong{san}.


\subsubsection{Colocation Influence}
\label{\detokenize{constraints:colocation-influence}}\label{\detokenize{constraints:s-coloc-influence}}
By default, if A is colocated with B, the cluster will take into account A’s
preferences when deciding where to place B, to maximize the chance that both
resources can run.

For a detailed look at exactly how this occurs, see
\sphinxhref{http://clusterlabs.org/doc/Colocation\_Explained.pdf}{Colocation Explained}.

However, if \sphinxcode{\sphinxupquote{influence}} is set to \sphinxcode{\sphinxupquote{false}} in the colocation constraint,
this will happen only if B is inactive and needing to be started. If B is
already active, A’s preferences will have no effect on placing B.

An example of what effect this would have and when it would be desirable would
be a nonessential reporting tool colocated with a resource-intensive service
that takes a long time to start. If the reporting tool fails enough times to
reach its migration threshold, by default the cluster will want to move both
resources to another node if possible. Setting \sphinxcode{\sphinxupquote{influence}} to \sphinxcode{\sphinxupquote{false}} on
the colocation constraint would mean that the reporting tool would be stopped
in this situation instead, to avoid forcing the service to move.

The \sphinxcode{\sphinxupquote{critical}} resource meta-attribute is a convenient way to specify the
default for all colocation constraints and groups involving a particular
resource.

\begin{sphinxadmonition}{note}{Note:}
If a noncritical resource is a member of a group, all later members of the
group will be treated as noncritical, even if they are marked as (or left to
default to) critical.
\end{sphinxadmonition}


\subsection{Resource Sets}
\label{\detokenize{constraints:resource-sets}}\label{\detokenize{constraints:s-resource-sets}}
\index{constraint!resource set}\index{resource!resource set}\ignorespaces 
\sphinxstyleemphasis{Resource sets} allow multiple resources to be affected by a single constraint.

\begin{sphinxShadowBox}
\sphinxstyletopictitle{A set of 3 resources}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}resource\PYGZus{}set} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}resource\PYGZhy{}set\PYGZhy{}example\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}A\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}B\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}C\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/resource\PYGZus{}set\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

Resource sets are valid inside \sphinxcode{\sphinxupquote{rsc\_location}}, \sphinxcode{\sphinxupquote{rsc\_order}}
(see {\hyperref[\detokenize{constraints:s-resource-sets-ordering}]{\sphinxcrossref{\DUrole{std,std-ref}{Ordering Sets of Resources}}}}), \sphinxcode{\sphinxupquote{rsc\_colocation}}
(see {\hyperref[\detokenize{constraints:s-resource-sets-colocation}]{\sphinxcrossref{\DUrole{std,std-ref}{Colocating Sets of Resources}}}}), and \sphinxcode{\sphinxupquote{rsc\_ticket}}
(see {\hyperref[\detokenize{multi-site-clusters:ticket-constraints}]{\sphinxcrossref{\DUrole{std,std-ref}{Configuring Ticket Dependencies}}}}) constraints.

A resource set has a number of properties that can be set, though not all
have an effect in all contexts.

\index{XML element!resource\_set}\index{resource\_set!XML element}\ignorespaces 

\begin{savenotes}\sphinxatlongtablestart\begin{longtable}{|\X{2}{9}|\X{2}{9}|\X{5}{9}|}
\caption{\sphinxstylestrong{Attributes of a resource\_set Element}\strut}\label{\detokenize{constraints:id8}}\label{\detokenize{constraints:index-31}}\\*[\sphinxlongtablecapskipadjust]
\hline
\sphinxstyletheadfamily 
Field
&\sphinxstyletheadfamily 
Default
&\sphinxstyletheadfamily 
Description
\\
\hline
\endfirsthead

\multicolumn{3}{c}%
{\makebox[0pt]{\sphinxtablecontinued{\tablename\ \thetable{} -- continued from previous page}}}\\
\hline
\sphinxstyletheadfamily 
Field
&\sphinxstyletheadfamily 
Default
&\sphinxstyletheadfamily 
Description
\\
\hline
\endhead

\hline
\multicolumn{3}{r}{\makebox[0pt][r]{\sphinxtablecontinued{Continued on next page}}}\\
\endfoot

\endlastfoot

id
&&
\index{resource\_set!attribute, id}\index{attribute!id (resource\_set)}\index{id!resource\_set attribute}\ignorespaces 
A unique name for the set (required)
\\
\hline
sequential
&
true
&
\index{resource\_set!attribute, sequential}\index{attribute!sequential (resource\_set)}\index{sequential!resource\_set attribute}\ignorespaces 
Whether the members of the set must be acted on in
order.  Meaningful within \sphinxcode{\sphinxupquote{rsc\_order}} and
\sphinxcode{\sphinxupquote{rsc\_colocation}}.
\\
\hline
require-all
&
true
&
\index{resource\_set!attribute, require-all}\index{attribute!require-all (resource\_set)}\index{require-all!resource\_set attribute}\ignorespaces 
Whether all members of the set must be active before
continuing.  With the current implementation, the
cluster may continue even if only one member of the
set is started, but if more than one member of the set
is starting at the same time, the cluster will still
wait until all of those have started before continuing
(this may change in future versions).  Meaningful
within \sphinxcode{\sphinxupquote{rsc\_order}}.
\\
\hline
role
&&
\index{resource\_set!attribute, role}\index{attribute!role (resource\_set)}\index{role!resource\_set attribute}\ignorespaces 
The constraint applies only to resource set members
that are {\hyperref[\detokenize{collective:s-resource-promotable}]{\sphinxcrossref{\DUrole{std,std-ref}{Promotable clones}}}} in this
role.  Meaningful within \sphinxcode{\sphinxupquote{rsc\_location}},
\sphinxcode{\sphinxupquote{rsc\_colocation}} and \sphinxcode{\sphinxupquote{rsc\_ticket}}.
Allowed values: \sphinxcode{\sphinxupquote{Started}}, \sphinxcode{\sphinxupquote{Promoted}},
\sphinxcode{\sphinxupquote{Unpromoted}}. For details, see
{\hyperref[\detokenize{collective:promotable-clone-constraints}]{\sphinxcrossref{\DUrole{std,std-ref}{Promotable Clone Constraints}}}}.
\\
\hline
action
&
value of
\sphinxcode{\sphinxupquote{first-action}}
in the enclosing
ordering
constraint
&
\index{resource\_set!attribute, action}\index{attribute!action (resource\_set)}\index{action!resource\_set attribute}\ignorespaces 
The action that applies to \sphinxstyleemphasis{all members} of the set.
Meaningful within \sphinxcode{\sphinxupquote{rsc\_order}}. Allowed values:
\sphinxcode{\sphinxupquote{start}}, \sphinxcode{\sphinxupquote{stop}}, \sphinxcode{\sphinxupquote{promote}}, \sphinxcode{\sphinxupquote{demote}}.
\\
\hline
score
&&
\index{resource\_set!attribute, score}\index{attribute!score (resource\_set)}\index{score!resource\_set attribute}\ignorespaces 
\sphinxstyleemphasis{Advanced use only.} Use a specific score for this
set within the constraint.
\\
\hline
\end{longtable}\sphinxatlongtableend\end{savenotes}


\subsection{Ordering Sets of Resources}
\label{\detokenize{constraints:ordering-sets-of-resources}}\label{\detokenize{constraints:s-resource-sets-ordering}}
A common situation is for an administrator to create a chain of ordered
resources, such as:

\begin{sphinxShadowBox}
\sphinxstyletopictitle{A chain of ordered resources}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}constraints}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}order} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}order\PYGZhy{}1\PYGZdq{}} \PYG{n+na}{first=}\PYG{l+s}{\PYGZdq{}A\PYGZdq{}} \PYG{n+na}{then=}\PYG{l+s}{\PYGZdq{}B\PYGZdq{}} \PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}order} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}order\PYGZhy{}2\PYGZdq{}} \PYG{n+na}{first=}\PYG{l+s}{\PYGZdq{}B\PYGZdq{}} \PYG{n+na}{then=}\PYG{l+s}{\PYGZdq{}C\PYGZdq{}} \PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}order} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}order\PYGZhy{}3\PYGZdq{}} \PYG{n+na}{first=}\PYG{l+s}{\PYGZdq{}C\PYGZdq{}} \PYG{n+na}{then=}\PYG{l+s}{\PYGZdq{}D\PYGZdq{}} \PYG{n+nt}{/\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/constraints\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Visual representation of the four resources’ start order for the above constraints}

\noindent\sphinxincludegraphics{{resource-set}.png}
\end{sphinxShadowBox}


\subsubsection{Ordered Set}
\label{\detokenize{constraints:ordered-set}}
To simplify this situation, {\hyperref[\detokenize{constraints:s-resource-sets}]{\sphinxcrossref{\DUrole{std,std-ref}{Resource Sets}}}} can be used within ordering
constraints:

\begin{sphinxShadowBox}
\sphinxstyletopictitle{A chain of ordered resources expressed as a set}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}constraints}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}order} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}order\PYGZhy{}1\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}set} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}ordered\PYGZhy{}set\PYGZhy{}example\PYGZdq{}} \PYG{n+na}{sequential=}\PYG{l+s}{\PYGZdq{}true\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}A\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}B\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}C\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}D\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}/resource\PYGZus{}set\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/rsc\PYGZus{}order\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/constraints\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

While the set-based format is not less verbose, it is significantly easier to
get right and maintain.

\begin{sphinxadmonition}{important}{Important:}
If you use a higher-level tool, pay attention to how it exposes this
functionality. Depending on the tool, creating a set \sphinxstylestrong{A B} may be equivalent to
\sphinxstylestrong{A then B}, or \sphinxstylestrong{B then A}.
\end{sphinxadmonition}


\subsubsection{Ordering Multiple Sets}
\label{\detokenize{constraints:ordering-multiple-sets}}
The syntax can be expanded to allow sets of resources to be ordered relative to
each other, where the members of each individual set may be ordered or
unordered (controlled by the \sphinxcode{\sphinxupquote{sequential}} property). In the example below, \sphinxstylestrong{A}
and \sphinxstylestrong{B} can both start in parallel, as can \sphinxstylestrong{C} and \sphinxstylestrong{D}, however \sphinxstylestrong{C} and
\sphinxstylestrong{D} can only start once \sphinxstyleemphasis{both} \sphinxstylestrong{A} \sphinxstyleemphasis{and} \sphinxstylestrong{B} are active.

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Ordered sets of unordered resources}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}constraints}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}order} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}order\PYGZhy{}1\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}set} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}ordered\PYGZhy{}set\PYGZhy{}1\PYGZdq{}} \PYG{n+na}{sequential=}\PYG{l+s}{\PYGZdq{}false\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
            \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}A\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
            \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}B\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}/resource\PYGZus{}set\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}set} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}ordered\PYGZhy{}set\PYGZhy{}2\PYGZdq{}} \PYG{n+na}{sequential=}\PYG{l+s}{\PYGZdq{}false\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
            \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}C\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
            \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}D\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}/resource\PYGZus{}set\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/rsc\PYGZus{}order\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/constraints\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Visual representation of the start order for two ordered sets of
        unordered resources}

\noindent\sphinxincludegraphics{{two-sets}.png}
\end{sphinxShadowBox}

Of course either set \textendash{} or both sets \textendash{} of resources can also be internally
ordered (by setting \sphinxcode{\sphinxupquote{sequential="true"}}) and there is no limit to the number
of sets that can be specified.

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Advanced use of set ordering - Three ordered sets, two of which are
        internally unordered}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}constraints}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}order} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}order\PYGZhy{}1\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}set} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}ordered\PYGZhy{}set\PYGZhy{}1\PYGZdq{}} \PYG{n+na}{sequential=}\PYG{l+s}{\PYGZdq{}false\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}A\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}B\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}/resource\PYGZus{}set\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}set} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}ordered\PYGZhy{}set\PYGZhy{}2\PYGZdq{}} \PYG{n+na}{sequential=}\PYG{l+s}{\PYGZdq{}true\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}C\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}D\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}/resource\PYGZus{}set\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}set} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}ordered\PYGZhy{}set\PYGZhy{}3\PYGZdq{}} \PYG{n+na}{sequential=}\PYG{l+s}{\PYGZdq{}false\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}E\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}F\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}/resource\PYGZus{}set\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/rsc\PYGZus{}order\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/constraints\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Visual representation of the start order for the three sets defined above}

\noindent\sphinxincludegraphics{{three-sets}.png}
\end{sphinxShadowBox}

\begin{sphinxadmonition}{important}{Important:}
An ordered set with \sphinxcode{\sphinxupquote{sequential=false}} makes sense only if there is another
set in the constraint. Otherwise, the constraint has no effect.
\end{sphinxadmonition}


\subsubsection{Resource Set OR Logic}
\label{\detokenize{constraints:resource-set-or-logic}}
The unordered set logic discussed so far has all been “AND” logic.  To illustrate
this take the 3 resource set figure in the previous section.  Those sets can be
expressed, \sphinxstylestrong{(A and B) then (C) then (D) then (E and F)}.

Say for example we want to change the first set, \sphinxstylestrong{(A and B)}, to use “OR” logic
so the sets look like this: \sphinxstylestrong{(A or B) then (C) then (D) then (E and F)}.  This
functionality can be achieved through the use of the \sphinxcode{\sphinxupquote{require-all}} option.
This option defaults to TRUE which is why the “AND” logic is used by default.
Setting \sphinxcode{\sphinxupquote{require-all=false}} means only one resource in the set needs to be
started before continuing on to the next set.

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Resource Set “OR” logic: Three ordered sets, where the first set is
        internally unordered with “OR” logic}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}constraints}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}order} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}order\PYGZhy{}1\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}set} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}ordered\PYGZhy{}set\PYGZhy{}1\PYGZdq{}} \PYG{n+na}{sequential=}\PYG{l+s}{\PYGZdq{}false\PYGZdq{}} \PYG{n+na}{require\PYGZhy{}all=}\PYG{l+s}{\PYGZdq{}false\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}A\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}B\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}/resource\PYGZus{}set\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}set} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}ordered\PYGZhy{}set\PYGZhy{}2\PYGZdq{}} \PYG{n+na}{sequential=}\PYG{l+s}{\PYGZdq{}true\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}C\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}D\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}/resource\PYGZus{}set\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}set} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}ordered\PYGZhy{}set\PYGZhy{}3\PYGZdq{}} \PYG{n+na}{sequential=}\PYG{l+s}{\PYGZdq{}false\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}E\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}F\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}/resource\PYGZus{}set\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/rsc\PYGZus{}order\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/constraints\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

\begin{sphinxadmonition}{important}{Important:}
An ordered set with \sphinxcode{\sphinxupquote{require-all=false}} makes sense only in conjunction with
\sphinxcode{\sphinxupquote{sequential=false}}. Think of it like this: \sphinxcode{\sphinxupquote{sequential=false}} modifies the set
to be an unordered set using “AND” logic by default, and adding
\sphinxcode{\sphinxupquote{require-all=false}} flips the unordered set’s “AND” logic to “OR” logic.
\end{sphinxadmonition}


\subsection{Colocating Sets of Resources}
\label{\detokenize{constraints:colocating-sets-of-resources}}\label{\detokenize{constraints:s-resource-sets-colocation}}
Another common situation is for an administrator to create a set of
colocated resources.

The simplest way to do this is to define a resource group (see
{\hyperref[\detokenize{collective:group-resources}]{\sphinxcrossref{\DUrole{std,std-ref}{Groups - A Syntactic Shortcut}}}}), but that cannot always accurately express the desired
relationships. For example, maybe the resources do not need to be ordered.

Another way would be to define each relationship as an individual constraint,
but that causes a difficult-to-follow constraint explosion as the number of
resources and combinations grow.

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Colocation chain as individual constraints, where A is placed first,
        then B, then C, then D}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}constraints}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}colocation} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}coloc\PYGZhy{}1\PYGZdq{}} \PYG{n+na}{rsc=}\PYG{l+s}{\PYGZdq{}D\PYGZdq{}} \PYG{n+na}{with\PYGZhy{}rsc=}\PYG{l+s}{\PYGZdq{}C\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}INFINITY\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}colocation} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}coloc\PYGZhy{}2\PYGZdq{}} \PYG{n+na}{rsc=}\PYG{l+s}{\PYGZdq{}C\PYGZdq{}} \PYG{n+na}{with\PYGZhy{}rsc=}\PYG{l+s}{\PYGZdq{}B\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}INFINITY\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}colocation} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}coloc\PYGZhy{}3\PYGZdq{}} \PYG{n+na}{rsc=}\PYG{l+s}{\PYGZdq{}B\PYGZdq{}} \PYG{n+na}{with\PYGZhy{}rsc=}\PYG{l+s}{\PYGZdq{}A\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}INFINITY\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/constraints\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

To express complicated relationships with a simplified syntax %
\begin{footnote}[2]\sphinxAtStartFootnote
which is not the same as saying easy to follow
%
\end{footnote},
{\hyperref[\detokenize{constraints:s-resource-sets}]{\sphinxcrossref{\DUrole{std,std-ref}{resource sets}}}} can be used within colocation constraints.

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Equivalent colocation chain expressed using \sphinxstylestrong{resource\_set}}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}constraints}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}colocation} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}coloc\PYGZhy{}1\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}INFINITY\PYGZdq{}} \PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}set} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}colocated\PYGZhy{}set\PYGZhy{}example\PYGZdq{}} \PYG{n+na}{sequential=}\PYG{l+s}{\PYGZdq{}true\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}A\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}B\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}C\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}D\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}/resource\PYGZus{}set\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/rsc\PYGZus{}colocation\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/constraints\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

\begin{sphinxadmonition}{note}{Note:}
Within a \sphinxcode{\sphinxupquote{resource\_set}}, the resources are listed in the order they are
\sphinxstyleemphasis{placed}, which is the reverse of the order in which they are \sphinxstyleemphasis{colocated}.
In the above example, resource \sphinxstylestrong{A} is placed before resource \sphinxstylestrong{B}, which is
the same as saying resource \sphinxstylestrong{B} is colocated with resource \sphinxstylestrong{A}.
\end{sphinxadmonition}

As with individual constraints, a resource that can’t be active prevents any
resource that must be colocated with it from being active. In both of the two
previous examples, if \sphinxstylestrong{B} is unable to run, then both \sphinxstylestrong{C} and by inference \sphinxstylestrong{D}
must remain stopped.

\begin{sphinxadmonition}{important}{Important:}
If you use a higher-level tool, pay attention to how it exposes this
functionality. Depending on the tool, creating a set \sphinxstylestrong{A B} may be equivalent to
\sphinxstylestrong{A with B}, or \sphinxstylestrong{B with A}.
\end{sphinxadmonition}

Resource sets can also be used to tell the cluster that entire \sphinxstyleemphasis{sets} of
resources must be colocated relative to each other, while the individual
members within any one set may or may not be colocated relative to each other
(determined by the set’s \sphinxcode{\sphinxupquote{sequential}} property).

In the following example, resources \sphinxstylestrong{B}, \sphinxstylestrong{C}, and \sphinxstylestrong{D} will each be colocated
with \sphinxstylestrong{A} (which will be placed first). \sphinxstylestrong{A} must be able to run in order for any
of the resources to run, but any of \sphinxstylestrong{B}, \sphinxstylestrong{C}, or \sphinxstylestrong{D} may be stopped without
affecting any of the others.

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Using colocated sets to specify a shared dependency}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}constraints}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}colocation} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}coloc\PYGZhy{}1\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}INFINITY\PYGZdq{}} \PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}set} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}colocated\PYGZhy{}set\PYGZhy{}2\PYGZdq{}} \PYG{n+na}{sequential=}\PYG{l+s}{\PYGZdq{}false\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}B\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}C\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}D\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}/resource\PYGZus{}set\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}set} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}colocated\PYGZhy{}set\PYGZhy{}1\PYGZdq{}} \PYG{n+na}{sequential=}\PYG{l+s}{\PYGZdq{}true\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}A\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}/resource\PYGZus{}set\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/rsc\PYGZus{}colocation\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/constraints\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

\begin{sphinxadmonition}{note}{Note:}
Pay close attention to the order in which resources and sets are listed.
While the members of any one sequential set are placed first to last (i.e., the
colocation dependency is last with first), multiple sets are placed last to
first (i.e. the colocation dependency is first with last).
\end{sphinxadmonition}

\begin{sphinxadmonition}{important}{Important:}
A colocated set with \sphinxcode{\sphinxupquote{sequential="false"}} makes sense only if there is
another set in the constraint. Otherwise, the constraint has no effect.
\end{sphinxadmonition}

There is no inherent limit to the number and size of the sets used.
The only thing that matters is that in order for any member of one set
in the constraint to be active, all members of sets listed after it must also
be active (and naturally on the same node); and if a set has \sphinxcode{\sphinxupquote{sequential="true"}},
then in order for one member of that set to be active, all members listed
before it must also be active.

If desired, you can restrict the dependency to instances of promotable clone
resources that are in a specific role, using the set’s \sphinxcode{\sphinxupquote{role}} property.

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Colocation in which the members of the middle set have no
        interdependencies, and the last set listed applies only to promoted
        instances}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}constraints}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}colocation} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}coloc\PYGZhy{}1\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}INFINITY\PYGZdq{}} \PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}set} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}colocated\PYGZhy{}set\PYGZhy{}1\PYGZdq{}} \PYG{n+na}{sequential=}\PYG{l+s}{\PYGZdq{}true\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}F\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}G\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}/resource\PYGZus{}set\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}set} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}colocated\PYGZhy{}set\PYGZhy{}2\PYGZdq{}} \PYG{n+na}{sequential=}\PYG{l+s}{\PYGZdq{}false\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}C\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}D\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}E\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}/resource\PYGZus{}set\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}set} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}colocated\PYGZhy{}set\PYGZhy{}3\PYGZdq{}} \PYG{n+na}{sequential=}\PYG{l+s}{\PYGZdq{}true\PYGZdq{}} \PYG{n+na}{role=}\PYG{l+s}{\PYGZdq{}Promoted\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}A\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}B\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}/resource\PYGZus{}set\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/rsc\PYGZus{}colocation\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/constraints\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Visual representation of the above example (resources are placed from
        left to right)}

\noindent\sphinxincludegraphics{{pcmk-colocated-sets}.png}
\end{sphinxShadowBox}

\begin{sphinxadmonition}{note}{Note:}
Unlike ordered sets, colocated sets do not use the \sphinxcode{\sphinxupquote{require-all}} option.
\end{sphinxadmonition}


\subsection{External Resource Dependencies}
\label{\detokenize{constraints:external-resource-dependencies}}
Sometimes, a resource will depend on services that are not managed by the
cluster. An example might be a resource that requires a file system that is
not managed by the cluster but mounted by systemd at boot time.

To accommodate this, the pacemaker systemd service depends on a normally empty
target called \sphinxcode{\sphinxupquote{resource-agents-deps.target}}. The system administrator may
create a unit drop-in for that target specifying the dependencies, to ensure
that the services are started before Pacemaker starts and stopped after
Pacemaker stops.

Typically, this is accomplished by placing a unit file in the
\sphinxcode{\sphinxupquote{/etc/systemd/system/resource-agents-deps.target.d}} directory, with directives
such as \sphinxcode{\sphinxupquote{Requires}} and \sphinxcode{\sphinxupquote{After}} specifying the dependencies as needed.

\index{fencing}\index{STONITH}\ignorespaces 

\section{Fencing}
\label{\detokenize{fencing:fencing}}\label{\detokenize{fencing:index-0}}\label{\detokenize{fencing:id1}}\label{\detokenize{fencing::doc}}

\subsection{What Is Fencing?}
\label{\detokenize{fencing:what-is-fencing}}
\sphinxstyleemphasis{Fencing} is the ability to make a node unable to run resources, even when that
node is unresponsive to cluster commands.

Fencing is also known as \sphinxstyleemphasis{STONITH}, an acronym for “Shoot The Other Node In The
Head”, since the most common fencing method is cutting power to the node.
Another method is “fabric fencing”, cutting the node’s access to some
capability required to run resources (such as network access or a shared disk).

\index{fencing!why necessary}\ignorespaces 

\subsection{Why Is Fencing Necessary?}
\label{\detokenize{fencing:why-is-fencing-necessary}}\label{\detokenize{fencing:index-1}}
Fencing protects your data from being corrupted by malfunctioning nodes or
unintentional concurrent access to shared resources.

Fencing protects against the “split brain” failure scenario, where cluster
nodes have lost the ability to reliably communicate with each other but are
still able to run resources. If the cluster just assumed that uncommunicative
nodes were down, then multiple instances of a resource could be started on
different nodes.

The effect of split brain depends on the resource type. For example, an IP
address brought up on two hosts on a network will cause packets to randomly be
sent to one or the other host, rendering the IP useless. For a database or
clustered file system, the effect could be much more severe, causing data
corruption or divergence.

Fencing is also used when a resource cannot otherwise be stopped. If a
resource fails to stop on a node, it cannot be started on a different node
without risking the same type of conflict as split-brain. Fencing the
original node ensures the resource can be safely started elsewhere.

Users may also configure the \sphinxcode{\sphinxupquote{on-fail}} property of {\hyperref[\detokenize{operations:operation}]{\sphinxcrossref{\DUrole{std,std-ref}{Resource Operations}}}} or the
\sphinxcode{\sphinxupquote{loss-policy}} property of
{\hyperref[\detokenize{multi-site-clusters:ticket-constraints}]{\sphinxcrossref{\DUrole{std,std-ref}{ticket constraints}}}} to \sphinxcode{\sphinxupquote{fence}}, in which
case the cluster will fence the resource’s node if the operation fails or the
ticket is lost.

\index{fencing!device}\ignorespaces 

\subsection{Fence Devices}
\label{\detokenize{fencing:fence-devices}}\label{\detokenize{fencing:index-2}}
A \sphinxstyleemphasis{fence device} or \sphinxstyleemphasis{fencing device} is a special type of resource that
provides the means to fence a node.

Examples of fencing devices include intelligent power switches and IPMI devices
that accept SNMP commands to cut power to a node, and iSCSI controllers that
allow SCSI reservations to be used to cut a node’s access to a shared disk.

Since fencing devices will be used to recover from loss of networking
connectivity to other nodes, it is essential that they do not rely on the same
network as the cluster itself, otherwise that network becomes a single point of
failure.

Since loss of a node due to power outage is indistinguishable from loss of
network connectivity to that node, it is also essential that at least one fence
device for a node does not share power with that node. For example, an on-board
IPMI controller that shares power with its host should not be used as the sole
fencing device for that host.

Since fencing is used to isolate malfunctioning nodes, no fence device should
rely on its target functioning properly. This includes, for example, devices
that ssh into a node and issue a shutdown command (such devices might be
suitable for testing, but never for production).

\index{fencing!agent}\ignorespaces 

\subsection{Fence Agents}
\label{\detokenize{fencing:fence-agents}}\label{\detokenize{fencing:index-3}}
A \sphinxstyleemphasis{fence agent} or \sphinxstyleemphasis{fencing agent} is a \sphinxcode{\sphinxupquote{stonith}}-class resource agent.

The fence agent standard provides commands (such as \sphinxcode{\sphinxupquote{off}} and \sphinxcode{\sphinxupquote{reboot}})
that the cluster can use to fence nodes. As with other resource agent classes,
this allows a layer of abstraction so that Pacemaker doesn’t need any knowledge
about specific fencing technologies \textendash{} that knowledge is isolated in the agent.

Pacemaker supports two fence agent standards, both inherited from
no-longer-active projects:
\begin{itemize}
\item {} 
Red Hat Cluster Suite (RHCS) style: These are typically installed in
\sphinxcode{\sphinxupquote{/usr/sbin}} with names starting with \sphinxcode{\sphinxupquote{fence\_}}.

\item {} 
Linux-HA style: These typically have names starting with \sphinxcode{\sphinxupquote{external/}}.
Pacemaker can support these agents using the \sphinxstylestrong{fence\_legacy} RHCS-style
agent as a wrapper, \sphinxstyleemphasis{if} support was enabled when Pacemaker was built, which
requires the \sphinxcode{\sphinxupquote{cluster-glue}} library.

\end{itemize}


\subsection{When a Fence Device Can Be Used}
\label{\detokenize{fencing:when-a-fence-device-can-be-used}}
Fencing devices do not actually “run” like most services. Typically, they just
provide an interface for sending commands to an external device.

Additionally, fencing may be initiated by Pacemaker, by other cluster-aware
software such as DRBD or DLM, or manually by an administrator, at any point in
the cluster life cycle, including before any resources have been started.

To accommodate this, Pacemaker does not require the fence device resource to be
“started” in order to be used. Whether a fence device is started or not
determines whether a node runs any recurring monitor for the device, and gives
the node a slight preference for being chosen to execute fencing using that
device.

By default, any node can execute any fencing device. If a fence device is
disabled by setting its \sphinxcode{\sphinxupquote{target-role}} to \sphinxcode{\sphinxupquote{Stopped}}, then no node can use
that device. If a location constraint with a negative score prevents a specific
node from “running” a fence device, then that node will never be chosen to
execute fencing using the device. A node may fence itself, but the cluster will
choose that only if no other nodes can do the fencing.

A common configuration scenario is to have one fence device per target node.
In such a case, users often configure anti-location constraints so that
the target node does not monitor its own device.


\subsection{Limitations of Fencing Resources}
\label{\detokenize{fencing:limitations-of-fencing-resources}}
Fencing resources have certain limitations that other resource classes don’t:
\begin{itemize}
\item {} 
They may have only one set of meta-attributes and one set of instance
attributes.

\item {} 
If {\hyperref[\detokenize{rules:rules}]{\sphinxcrossref{\DUrole{std,std-ref}{Rules}}}} are used to determine fencing resource options, these
might be evaluated only when first read, meaning that later changes to the
rules will have no effect. Therefore, it is better to avoid confusion and not
use rules at all with fencing resources.

\end{itemize}

These limitations could be revisited if there is sufficient user demand.

\index{fencing!special instance attributes}\ignorespaces 

\subsection{Special Meta-Attributes for Fencing Resources}
\label{\detokenize{fencing:special-meta-attributes-for-fencing-resources}}\label{\detokenize{fencing:fencing-attributes}}\label{\detokenize{fencing:index-4}}
The table below lists special resource meta-attributes that may be set for any
fencing resource.


\begin{savenotes}\sphinxattablestart
\centering
\sphinxcapstartof{table}
\sphinxcaption{\sphinxstylestrong{Additional Properties of Fencing Resources}}\label{\detokenize{fencing:id3}}
\sphinxaftercaption
\begin{tabular}[t]{|\X{2}{9}|\X{1}{9}|\X{2}{9}|\X{4}{9}|}
\hline
\sphinxstyletheadfamily 
Field
&\sphinxstyletheadfamily 
Type
&\sphinxstyletheadfamily 
Default
&\sphinxstyletheadfamily 
Description
\\
\hline
provides
&
string
&&
\index{provides}\ignorespaces 
Any special capability provided by the
fence device. Currently, only one such
capability is meaningful:
{\hyperref[\detokenize{fencing:unfencing}]{\sphinxcrossref{\DUrole{std,std-ref}{unfencing}}}}.
\\
\hline
\end{tabular}
\par
\sphinxattableend\end{savenotes}


\subsection{Special Instance Attributes for Fencing Resources}
\label{\detokenize{fencing:special-instance-attributes-for-fencing-resources}}
The table below lists special instance attributes that may be set for any
fencing resource (\sphinxstyleemphasis{not} meta-attributes, even though they are interpreted by
Pacemaker rather than the fence agent). These are also listed in the man page
for \sphinxcode{\sphinxupquote{pacemaker-fenced}}.


\begin{savenotes}\sphinxatlongtablestart\begin{longtable}{|\X{2}{9}|\X{1}{9}|\X{2}{9}|\X{4}{9}|}
\caption{\sphinxstylestrong{Additional Properties of Fencing Resources}\strut}\label{\detokenize{fencing:id4}}\\*[\sphinxlongtablecapskipadjust]
\hline
\sphinxstyletheadfamily 
Field
&\sphinxstyletheadfamily 
Type
&\sphinxstyletheadfamily 
Default
&\sphinxstyletheadfamily 
Description
\\
\hline
\endfirsthead

\multicolumn{4}{c}%
{\makebox[0pt]{\sphinxtablecontinued{\tablename\ \thetable{} -- continued from previous page}}}\\
\hline
\sphinxstyletheadfamily 
Field
&\sphinxstyletheadfamily 
Type
&\sphinxstyletheadfamily 
Default
&\sphinxstyletheadfamily 
Description
\\
\hline
\endhead

\hline
\multicolumn{4}{r}{\makebox[0pt][r]{\sphinxtablecontinued{Continued on next page}}}\\
\endfoot

\endlastfoot

stonith-timeout
&
time
&&
\index{stonith-timeout}\ignorespaces 
This is not used by Pacemaker (see the
\sphinxcode{\sphinxupquote{pcmk\_reboot\_timeout}},
\sphinxcode{\sphinxupquote{pcmk\_off\_timeout}}, etc. properties
instead), but it may be used by
Linux-HA fence agents.
\\
\hline
pcmk\_host\_map
&
string
&&
\index{pcmk\_host\_map}\ignorespaces 
A mapping of node names to ports
for devices that do not understand
the node names.

Example: \sphinxcode{\sphinxupquote{node1:1;node2:2,3}} tells
the cluster to use port 1 for
\sphinxcode{\sphinxupquote{node1}} and ports 2 and 3 for
\sphinxcode{\sphinxupquote{node2}}. If \sphinxcode{\sphinxupquote{pcmk\_host\_check}} is
explicitly set to \sphinxcode{\sphinxupquote{static-list}},
either this or \sphinxcode{\sphinxupquote{pcmk\_host\_list}} must
be set. The port portion of the map
may contain special characters such as
spaces if preceded by a backslash
\sphinxstyleemphasis{(since 2.1.2)}.
\\
\hline
pcmk\_host\_list
&
string
&&
\index{pcmk\_host\_list}\ignorespaces 
A list of machines controlled by this
device. If \sphinxcode{\sphinxupquote{pcmk\_host\_check}} is
explicitly set to \sphinxcode{\sphinxupquote{static-list}},
either this or \sphinxcode{\sphinxupquote{pcmk\_host\_map}} must
be set.
\\
\hline
pcmk\_host\_check
&
string
&
Value appropriate
to other
parameters (see
“Default Check
Type” below)
&
\index{pcmk\_host\_check}\ignorespaces 
The method Pacemaker should use to
determine which nodes can be targeted
by this device. Allowed values:
\begin{itemize}
\item {} 
\sphinxcode{\sphinxupquote{static-list:}} targets are listed
in the \sphinxcode{\sphinxupquote{pcmk\_host\_list}} or
\sphinxcode{\sphinxupquote{pcmk\_host\_map}} attribute

\item {} 
\sphinxcode{\sphinxupquote{dynamic-list:}} query the device
via the agent’s \sphinxcode{\sphinxupquote{list}} action

\item {} 
\sphinxcode{\sphinxupquote{status:}} query the device via the
agent’s \sphinxcode{\sphinxupquote{status}} action

\item {} 
\sphinxcode{\sphinxupquote{none:}} assume the device can
fence any node

\end{itemize}
\\
\hline
pcmk\_delay\_max
&
time
&
0s
&
\index{pcmk\_delay\_max}\ignorespaces 
Enable a delay of no more than the
time specified before executing
fencing actions. Pacemaker derives the
overall delay by taking the value of
pcmk\_delay\_base and adding a random
delay value such that the sum is kept
below this maximum. This is sometimes
used in two-node clusters to ensure
that the nodes don’t fence each other
at the same time.
\\
\hline
pcmk\_delay\_base
&
time
&
0s
&
\index{pcmk\_delay\_base}\ignorespaces 
Enable a static delay before executing
fencing actions. This can be used, for
example, in two-node clusters to
ensure that the nodes don’t fence each
other, by having separate fencing
resources with different values. The
node that is fenced with the shorter
delay will lose a fencing race. The
overall delay introduced by pacemaker
is derived from this value plus a
random delay such that the sum is kept
below the maximum delay. A single
device can have different delays per
node using a host map \sphinxstyleemphasis{(since 2.1.2)},
for example \sphinxcode{\sphinxupquote{node1:0s;node2:5s.}}
\\
\hline
pcmk\_action\_limit
&
integer
&
1
&
\index{pcmk\_action\_limit}\ignorespaces 
The maximum number of actions that can
be performed in parallel on this
device. A value of -1 means unlimited.
Node fencing actions initiated by the
cluster (as opposed to an administrator
running the \sphinxcode{\sphinxupquote{stonith\_admin}} tool or
the fencer running recurring device
monitors and \sphinxcode{\sphinxupquote{status}} and \sphinxcode{\sphinxupquote{list}}
commands) are additionally subject to
the \sphinxcode{\sphinxupquote{concurrent-fencing}} cluster
property.
\\
\hline
pcmk\_host\_argument
&
string
&
\sphinxcode{\sphinxupquote{port}} otherwise
\sphinxcode{\sphinxupquote{plug}} if
supported
according to the
metadata of the
fence agent
&
\index{pcmk\_host\_argument}\ignorespaces 
\sphinxstyleemphasis{Advanced use only.} Which parameter
should be supplied to the fence agent
to identify the node to be fenced.
Some devices support neither the
standard \sphinxcode{\sphinxupquote{plug}} nor the deprecated
\sphinxcode{\sphinxupquote{port}} parameter, or may provide
additional ones. Use this to specify
an alternate, device-specific
parameter. A value of \sphinxcode{\sphinxupquote{none}} tells
the cluster not to supply any
additional parameters.
\\
\hline
pcmk\_reboot\_action
&
string
&
reboot
&
\index{pcmk\_reboot\_action}\ignorespaces 
\sphinxstyleemphasis{Advanced use only.} The command to
send to the resource agent in order to
reboot a node. Some devices do not
support the standard commands or may
provide additional ones. Use this to
specify an alternate, device-specific
command.
\\
\hline
pcmk\_reboot\_timeout
&
time
&
60s
&
\index{pcmk\_reboot\_timeout}\ignorespaces 
\sphinxstyleemphasis{Advanced use only.} Specify an
alternate timeout to use for
\sphinxcode{\sphinxupquote{reboot}} actions instead of the
value of \sphinxcode{\sphinxupquote{stonith-timeout}}. Some
devices need much more or less time to
complete than normal. Use this to
specify an alternate, device-specific
timeout.
\\
\hline
pcmk\_reboot\_retries
&
integer
&
2
&
\index{pcmk\_reboot\_retries}\ignorespaces 
\sphinxstyleemphasis{Advanced use only.} The maximum
number of times to retry the
\sphinxcode{\sphinxupquote{reboot}} command within the timeout
period. Some devices do not support
multiple connections, and operations
may fail if the device is busy with
another task, so Pacemaker will
automatically retry the operation, if
there is time remaining. Use this
option to alter the number of times
Pacemaker retries before giving up.
\\
\hline
pcmk\_off\_action
&
string
&
off
&
\index{pcmk\_off\_action}\ignorespaces 
\sphinxstyleemphasis{Advanced use only.} The command to
send to the resource agent in order to
shut down a node. Some devices do not
support the standard commands or may
provide additional ones. Use this to
specify an alternate, device-specific
command.
\\
\hline
pcmk\_off\_timeout
&
time
&
60s
&
\index{pcmk\_off\_timeout}\ignorespaces 
\sphinxstyleemphasis{Advanced use only.} Specify an
alternate timeout to use for
\sphinxcode{\sphinxupquote{off}} actions instead of the
value of \sphinxcode{\sphinxupquote{stonith-timeout}}. Some
devices need much more or less time to
complete than normal. Use this to
specify an alternate, device-specific
timeout.
\\
\hline
pcmk\_off\_retries
&
integer
&
2
&
\index{pcmk\_off\_retries}\ignorespaces 
\sphinxstyleemphasis{Advanced use only.} The maximum
number of times to retry the
\sphinxcode{\sphinxupquote{off}} command within the timeout
period. Some devices do not support
multiple connections, and operations
may fail if the device is busy with
another task, so Pacemaker will
automatically retry the operation, if
there is time remaining. Use this
option to alter the number of times
Pacemaker retries before giving up.
\\
\hline
pcmk\_list\_action
&
string
&
list
&
\index{pcmk\_list\_action}\ignorespaces 
\sphinxstyleemphasis{Advanced use only.} The command to
send to the resource agent in order to
list nodes. Some devices do not
support the standard commands or may
provide additional ones. Use this to
specify an alternate, device-specific
command.
\\
\hline
pcmk\_list\_timeout
&
time
&
60s
&
\index{pcmk\_list\_timeout}\ignorespaces 
\sphinxstyleemphasis{Advanced use only.} Specify an
alternate timeout to use for
\sphinxcode{\sphinxupquote{list}} actions instead of the
value of \sphinxcode{\sphinxupquote{stonith-timeout}}. Some
devices need much more or less time to
complete than normal. Use this to
specify an alternate, device-specific
timeout.
\\
\hline
pcmk\_list\_retries
&
integer
&
2
&
\index{pcmk\_list\_retries}\ignorespaces 
\sphinxstyleemphasis{Advanced use only.} The maximum
number of times to retry the
\sphinxcode{\sphinxupquote{list}} command within the timeout
period. Some devices do not support
multiple connections, and operations
may fail if the device is busy with
another task, so Pacemaker will
automatically retry the operation, if
there is time remaining. Use this
option to alter the number of times
Pacemaker retries before giving up.
\\
\hline
pcmk\_monitor\_action
&
string
&
monitor
&
\index{pcmk\_monitor\_action}\ignorespaces 
\sphinxstyleemphasis{Advanced use only.} The command to
send to the resource agent in order to
report extended status. Some devices do
not support the standard commands or
may provide additional ones. Use this
to specify an alternate,
device-specific command.
\\
\hline
pcmk\_monitor\_timeout
&
time
&
60s
&
\index{pcmk\_monitor\_timeout}\ignorespaces 
\sphinxstyleemphasis{Advanced use only.} Specify an
alternate timeout to use for
\sphinxcode{\sphinxupquote{monitor}} actions instead of the
value of \sphinxcode{\sphinxupquote{stonith-timeout}}. Some
devices need much more or less time to
complete than normal. Use this to
specify an alternate, device-specific
timeout.
\\
\hline
pcmk\_monitor\_retries
&
integer
&
2
&
\index{pcmk\_monitor\_retries}\ignorespaces 
\sphinxstyleemphasis{Advanced use only.} The maximum
number of times to retry the
\sphinxcode{\sphinxupquote{monitor}} command within the timeout
period. Some devices do not support
multiple connections, and operations
may fail if the device is busy with
another task, so Pacemaker will
automatically retry the operation, if
there is time remaining. Use this
option to alter the number of times
Pacemaker retries before giving up.
\\
\hline
pcmk\_status\_action
&
string
&
status
&
\index{pcmk\_status\_action}\ignorespaces 
\sphinxstyleemphasis{Advanced use only.} The command to
send to the resource agent in order to
report status. Some devices do
not support the standard commands or
may provide additional ones. Use this
to specify an alternate,
device-specific command.
\\
\hline
pcmk\_status\_timeout
&
time
&
60s
&
\index{pcmk\_status\_timeout}\ignorespaces 
\sphinxstyleemphasis{Advanced use only.} Specify an
alternate timeout to use for
\sphinxcode{\sphinxupquote{status}} actions instead of the
value of \sphinxcode{\sphinxupquote{stonith-timeout}}. Some
devices need much more or less time to
complete than normal. Use this to
specify an alternate, device-specific
timeout.
\\
\hline
pcmk\_status\_retries
&
integer
&
2
&
\index{pcmk\_status\_retries}\ignorespaces 
\sphinxstyleemphasis{Advanced use only.} The maximum
number of times to retry the
\sphinxcode{\sphinxupquote{status}} command within the timeout
period. Some devices do not support
multiple connections, and operations
may fail if the device is busy with
another task, so Pacemaker will
automatically retry the operation, if
there is time remaining. Use this
option to alter the number of times
Pacemaker retries before giving up.
\\
\hline
\end{longtable}\sphinxatlongtableend\end{savenotes}


\subsection{Default Check Type}
\label{\detokenize{fencing:default-check-type}}
If the user does not explicitly configure \sphinxcode{\sphinxupquote{pcmk\_host\_check}} for a fence
device, a default value appropriate to other configured parameters will be
used:
\begin{itemize}
\item {} 
If either \sphinxcode{\sphinxupquote{pcmk\_host\_list}} or \sphinxcode{\sphinxupquote{pcmk\_host\_map}} is configured,
\sphinxcode{\sphinxupquote{static-list}} will be used;

\item {} 
otherwise, if the fence device supports the \sphinxcode{\sphinxupquote{list}} action, and the first
attempt at using \sphinxcode{\sphinxupquote{list}} succeeds, \sphinxcode{\sphinxupquote{dynamic-list}} will be used;

\item {} 
otherwise, if the fence device supports the \sphinxcode{\sphinxupquote{status}} action, \sphinxcode{\sphinxupquote{status}}
will be used;

\item {} 
otherwise, \sphinxcode{\sphinxupquote{none}} will be used.

\end{itemize}

\index{unfencing}\index{fencing!unfencing}\ignorespaces 

\subsection{Unfencing}
\label{\detokenize{fencing:unfencing}}\label{\detokenize{fencing:index-29}}\label{\detokenize{fencing:id2}}
With fabric fencing (such as cutting network or shared disk access rather than
power), it is expected that the cluster will fence the node, and then a system
administrator must manually investigate what went wrong, correct any issues
found, then reboot (or restart the cluster services on) the node.

Once the node reboots and rejoins the cluster, some fabric fencing devices
require an explicit command to restore the node’s access. This capability is
called \sphinxstyleemphasis{unfencing} and is typically implemented as the fence agent’s \sphinxcode{\sphinxupquote{on}}
command.

If any cluster resource has \sphinxcode{\sphinxupquote{requires}} set to \sphinxcode{\sphinxupquote{unfencing}}, then that
resource will not be probed or started on a node until that node has been
unfenced.


\subsection{Fencing and Quorum}
\label{\detokenize{fencing:fencing-and-quorum}}
In general, a cluster partition may execute fencing only if the partition has
quorum, and the \sphinxcode{\sphinxupquote{stonith-enabled}} cluster property is set to true. However,
there are exceptions:
\begin{itemize}
\item {} 
The requirements apply only to fencing initiated by Pacemaker. If an
administrator initiates fencing using the \sphinxcode{\sphinxupquote{stonith\_admin}} command, or an
external application such as DLM initiates fencing using Pacemaker’s C API,
the requirements do not apply.

\item {} 
A cluster partition without quorum is allowed to fence any active member of
that partition. As a corollary, this allows a \sphinxcode{\sphinxupquote{no-quorum-policy}} of
\sphinxcode{\sphinxupquote{suicide}} to work.

\item {} 
If the \sphinxcode{\sphinxupquote{no-quorum-policy}} cluster property is set to \sphinxcode{\sphinxupquote{ignore}}, then
quorum is not required to execute fencing of any node.

\end{itemize}


\subsection{Fencing Timeouts}
\label{\detokenize{fencing:fencing-timeouts}}
Fencing timeouts are complicated, since a single fencing operation can involve
many steps, each of which may have a separate timeout.

Fencing may be initiated in one of several ways:
\begin{itemize}
\item {} 
An administrator may initiate fencing using the \sphinxcode{\sphinxupquote{stonith\_admin}} tool,
which has a \sphinxcode{\sphinxupquote{-{-}timeout}} option (defaulting to 2 minutes) that will be used
as the fence operation timeout.

\item {} 
An external application such as DLM may initiate fencing using the Pacemaker
C API. The application will specify the fence operation timeout in this case,
which might or might not be configurable by the user.

\item {} 
The cluster may initiate fencing itself. In this case, the
\sphinxcode{\sphinxupquote{stonith-timeout}} cluster property (defaulting to 1 minute) will be used as
the fence operation timeout.

\end{itemize}

However fencing is initiated, the initiator contacts Pacemaker’s fencer
(\sphinxcode{\sphinxupquote{pacemaker-fenced}}) to request fencing. This connection and request has its
own timeout, separate from the fencing operation timeout, but usually happens
very quickly.

The fencer will contact all fencers in the cluster to ask what devices they
have available to fence the target node. The fence operation timeout will be
used as the timeout for each of these queries.

Once a fencing device has been selected, the fencer will check whether any
action-specific timeout has been configured for the device, to use instead of
the fence operation timeout. For example, if \sphinxcode{\sphinxupquote{stonith-timeout}} is 60 seconds,
but the fencing device has \sphinxcode{\sphinxupquote{pcmk\_reboot\_timeout}} configured as 90 seconds,
then a timeout of 90 seconds will be used for reboot actions using that device.

A device may have retries configured, in which case the timeout applies across
all attempts. For example, if a device has \sphinxcode{\sphinxupquote{pcmk\_reboot\_retries}} configured
as 2, and the first reboot attempt fails, the second attempt will only have
whatever time is remaining in the action timeout after subtracting how much
time the first attempt used. This means that if the first attempt fails due to
using the entire timeout, no further attempts will be made. There is currently
no way to configure a per-attempt timeout.

If more than one device is required to fence a target, whether due to failure
of the first device or a fencing topology with multiple devices configured for
the target, each device will have its own separate action timeout.

For all of the above timeouts, the fencer will generally multiply the
configured value by 1.2 to get an actual value to use, to account for time
needed by the fencer’s own processing.

Separate from the fencer’s timeouts, some fence agents have internal timeouts
for individual steps of their fencing process. These agents often have
parameters to configure these timeouts, such as \sphinxcode{\sphinxupquote{login-timeout}},
\sphinxcode{\sphinxupquote{shell-timeout}}, or \sphinxcode{\sphinxupquote{power-timeout}}. Many such agents also have a
\sphinxcode{\sphinxupquote{disable-timeout}} parameter to ignore their internal timeouts and just let
Pacemaker handle the timeout. This causes a difference in retry behavior.
If \sphinxcode{\sphinxupquote{disable-timeout}} is not set, and the agent hits one of its internal
timeouts, it will report that as a failure to Pacemaker, which can then retry.
If \sphinxcode{\sphinxupquote{disable-timeout}} is set, and Pacemaker hits a timeout for the agent, then
there will be no time remaining, and no retry will be done.


\subsection{Fence Devices Dependent on Other Resources}
\label{\detokenize{fencing:fence-devices-dependent-on-other-resources}}
In some cases, a fence device may require some other cluster resource (such as
an IP address) to be active in order to function properly.

This is obviously undesirable in general: fencing may be required when the
depended-on resource is not active, or fencing may be required because the node
running the depended-on resource is no longer responding.

However, this may be acceptable under certain conditions:
\begin{itemize}
\item {} 
The dependent fence device should not be able to target any node that is
allowed to run the depended-on resource.

\item {} 
The depended-on resource should not be disabled during production operation.

\item {} 
The \sphinxcode{\sphinxupquote{concurrent-fencing}} cluster property should be set to \sphinxcode{\sphinxupquote{true}}.
Otherwise, if both the node running the depended-on resource and some node
targeted by the dependent fence device need to be fenced, the fencing of the
node running the depended-on resource might be ordered first, making the
second fencing impossible and blocking further recovery. With concurrent
fencing, the dependent fence device might fail at first due to the
depended-on resource being unavailable, but it will be retried and eventually
succeed once the resource is brought back up.

\end{itemize}

Even under those conditions, there is one unlikely problem scenario. The DC
always schedules fencing of itself after any other fencing needed, to avoid
unnecessary repeated DC elections. If the dependent fence device targets the
DC, and both the DC and a different node running the depended-on resource need
to be fenced, the DC fencing will always fail and block further recovery. Note,
however, that losing a DC node entirely causes some other node to become DC and
schedule the fencing, so this is only a risk when a stop or other operation
with \sphinxcode{\sphinxupquote{on-fail}} set to \sphinxcode{\sphinxupquote{fencing}} fails on the DC.

\index{fencing!configuration}\ignorespaces 

\subsection{Configuring Fencing}
\label{\detokenize{fencing:configuring-fencing}}\label{\detokenize{fencing:index-30}}
Higher-level tools can provide simpler interfaces to this process, but using
Pacemaker command-line tools, this is how you could configure a fence device.
\begin{enumerate}
\item {} 
Find the correct driver:

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYGZsh{} stonith\PYGZus{}admin \PYGZhy{}\PYGZhy{}list\PYGZhy{}installed
\end{sphinxVerbatim}

\begin{sphinxadmonition}{note}{Note:}
You may have to install packages to make fence agents available on your
host. Searching your available packages for \sphinxcode{\sphinxupquote{fence-}} is usually
helpful. Ensure the packages providing the fence agents you require are
installed on every cluster node.
\end{sphinxadmonition}

\item {} 
Find the required parameters associated with the device
(replacing \sphinxcode{\sphinxupquote{\$AGENT\_NAME}} with the name obtained from the previous step):

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYGZsh{} stonith\PYGZus{}admin \PYGZhy{}\PYGZhy{}metadata \PYGZhy{}\PYGZhy{}agent \PYGZdl{}AGENT\PYGZus{}NAME
\end{sphinxVerbatim}

\item {} 
Create a file called \sphinxcode{\sphinxupquote{stonith.xml}} containing a primitive resource
with a class of \sphinxcode{\sphinxupquote{stonith}}, a type equal to the agent name obtained earlier,
and a parameter for each of the values returned in the previous step.

\item {} 
If the device does not know how to fence nodes based on their uname,
you may also need to set the special \sphinxcode{\sphinxupquote{pcmk\_host\_map}} parameter.  See
{\hyperref[\detokenize{fencing:fencing-attributes}]{\sphinxcrossref{\DUrole{std,std-ref}{Special Meta-Attributes for Fencing Resources}}}} for details.

\item {} 
If the device does not support the \sphinxcode{\sphinxupquote{list}} command, you may also need
to set the special \sphinxcode{\sphinxupquote{pcmk\_host\_list}} and/or \sphinxcode{\sphinxupquote{pcmk\_host\_check}}
parameters.  See {\hyperref[\detokenize{fencing:fencing-attributes}]{\sphinxcrossref{\DUrole{std,std-ref}{Special Meta-Attributes for Fencing Resources}}}} for details.

\item {} 
If the device does not expect the target to be specified with the
\sphinxcode{\sphinxupquote{port}} parameter, you may also need to set the special
\sphinxcode{\sphinxupquote{pcmk\_host\_argument}} parameter. See {\hyperref[\detokenize{fencing:fencing-attributes}]{\sphinxcrossref{\DUrole{std,std-ref}{Special Meta-Attributes for Fencing Resources}}}} for details.

\item {} 
Upload it into the CIB using cibadmin:

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYGZsh{} cibadmin \PYGZhy{}\PYGZhy{}create \PYGZhy{}\PYGZhy{}scope resources \PYGZhy{}\PYGZhy{}xml\PYGZhy{}file stonith.xml
\end{sphinxVerbatim}

\item {} 
Set \sphinxcode{\sphinxupquote{stonith-enabled}} to true:

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYGZsh{} crm\PYGZus{}attribute \PYGZhy{}\PYGZhy{}type crm\PYGZus{}config \PYGZhy{}\PYGZhy{}name stonith\PYGZhy{}enabled \PYGZhy{}\PYGZhy{}update true
\end{sphinxVerbatim}

\item {} 
Once the stonith resource is running, you can test it by executing the
following, replacing \sphinxcode{\sphinxupquote{\$NODE\_NAME}} with the name of the node to fence
(although you might want to stop the cluster on that machine first):

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYGZsh{} stonith\PYGZus{}admin \PYGZhy{}\PYGZhy{}reboot \PYGZdl{}NODE\PYGZus{}NAME
\end{sphinxVerbatim}

\end{enumerate}


\subsubsection{Example Fencing Configuration}
\label{\detokenize{fencing:example-fencing-configuration}}
For this example, we assume we have a cluster node, \sphinxcode{\sphinxupquote{pcmk-1}}, whose IPMI
controller is reachable at the IP address 192.0.2.1. The IPMI controller uses
the username \sphinxcode{\sphinxupquote{testuser}} and the password \sphinxcode{\sphinxupquote{abc123}}.
\begin{enumerate}
\item {} 
Looking at what’s installed, we may see a variety of available agents:

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYGZsh{} stonith\PYGZus{}admin \PYGZhy{}\PYGZhy{}list\PYGZhy{}installed
\end{sphinxVerbatim}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
(... some output omitted ...)
fence\PYGZus{}idrac
fence\PYGZus{}ilo3
fence\PYGZus{}ilo4
fence\PYGZus{}ilo5
fence\PYGZus{}imm
fence\PYGZus{}ipmilan
(... some output omitted ...)
\end{sphinxVerbatim}

Perhaps after some reading some man pages and doing some Internet searches,
we might decide \sphinxcode{\sphinxupquote{fence\_ipmilan}} is our best choice.

\item {} 
Next, we would check what parameters \sphinxcode{\sphinxupquote{fence\_ipmilan}} provides:

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYGZsh{} stonith\PYGZus{}admin \PYGZhy{}\PYGZhy{}metadata \PYGZhy{}a fence\PYGZus{}ipmilan
\end{sphinxVerbatim}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}resource\PYGZhy{}agent} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}fence\PYGZus{}ipmilan\PYGZdq{}} \PYG{n+na}{shortdesc=}\PYG{l+s}{\PYGZdq{}Fence agent for IPMI\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}symlink} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}fence\PYGZus{}ilo3\PYGZdq{}} \PYG{n+na}{shortdesc=}\PYG{l+s}{\PYGZdq{}Fence agent for HP iLO3\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}symlink} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}fence\PYGZus{}ilo4\PYGZdq{}} \PYG{n+na}{shortdesc=}\PYG{l+s}{\PYGZdq{}Fence agent for HP iLO4\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}symlink} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}fence\PYGZus{}ilo5\PYGZdq{}} \PYG{n+na}{shortdesc=}\PYG{l+s}{\PYGZdq{}Fence agent for HP iLO5\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}symlink} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}fence\PYGZus{}imm\PYGZdq{}} \PYG{n+na}{shortdesc=}\PYG{l+s}{\PYGZdq{}Fence agent for IBM Integrated Management Module\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}symlink} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}fence\PYGZus{}idrac\PYGZdq{}} \PYG{n+na}{shortdesc=}\PYG{l+s}{\PYGZdq{}Fence agent for Dell iDRAC\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}longdesc}\PYG{n+nt}{\PYGZgt{}}fence\PYGZus{}ipmilan is an I/O Fencing agentwhich can be used with machines controlled by IPMI.This agent calls support software ipmitool (http://ipmitool.sf.net/). WARNING! This fence agent might report success before the node is powered off. You should use \PYGZhy{}m/method onoff if your fence device works correctly with that option.\PYG{n+nt}{\PYGZlt{}/longdesc\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}vendor\PYGZhy{}url}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}parameters}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}parameter} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}action\PYGZdq{}} \PYG{n+na}{unique=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{required=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}getopt} \PYG{n+na}{mixed=}\PYG{l+s}{\PYGZdq{}\PYGZhy{}o, \PYGZhy{}\PYGZhy{}action=[action]\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}content} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}string\PYGZdq{}} \PYG{n+na}{default=}\PYG{l+s}{\PYGZdq{}reboot\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}shortdesc} \PYG{n+na}{lang=}\PYG{l+s}{\PYGZdq{}en\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}Fencing action\PYG{n+nt}{\PYGZlt{}/shortdesc\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/parameter\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}parameter} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}auth\PYGZdq{}} \PYG{n+na}{unique=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{required=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}getopt} \PYG{n+na}{mixed=}\PYG{l+s}{\PYGZdq{}\PYGZhy{}A, \PYGZhy{}\PYGZhy{}auth=[auth]\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}content} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}select\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}option} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}md5\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}option} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}password\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}option} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}none\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}/content\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}shortdesc} \PYG{n+na}{lang=}\PYG{l+s}{\PYGZdq{}en\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}IPMI Lan Auth type.\PYG{n+nt}{\PYGZlt{}/shortdesc\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/parameter\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}parameter} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}cipher\PYGZdq{}} \PYG{n+na}{unique=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{required=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}getopt} \PYG{n+na}{mixed=}\PYG{l+s}{\PYGZdq{}\PYGZhy{}C, \PYGZhy{}\PYGZhy{}cipher=[cipher]\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}content} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}string\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}shortdesc} \PYG{n+na}{lang=}\PYG{l+s}{\PYGZdq{}en\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}Ciphersuite to use (same as ipmitool \PYGZhy{}C parameter)\PYG{n+nt}{\PYGZlt{}/shortdesc\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/parameter\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}parameter} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}hexadecimal\PYGZus{}kg\PYGZdq{}} \PYG{n+na}{unique=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{required=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}getopt} \PYG{n+na}{mixed=}\PYG{l+s}{\PYGZdq{}\PYGZhy{}\PYGZhy{}hexadecimal\PYGZhy{}kg=[key]\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}content} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}string\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}shortdesc} \PYG{n+na}{lang=}\PYG{l+s}{\PYGZdq{}en\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}Hexadecimal\PYGZhy{}encoded Kg key for IPMIv2 authentication\PYG{n+nt}{\PYGZlt{}/shortdesc\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/parameter\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}parameter} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}ip\PYGZdq{}} \PYG{n+na}{unique=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{required=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{obsoletes=}\PYG{l+s}{\PYGZdq{}ipaddr\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}getopt} \PYG{n+na}{mixed=}\PYG{l+s}{\PYGZdq{}\PYGZhy{}a, \PYGZhy{}\PYGZhy{}ip=[ip]\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}content} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}string\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}shortdesc} \PYG{n+na}{lang=}\PYG{l+s}{\PYGZdq{}en\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}IP address or hostname of fencing device\PYG{n+nt}{\PYGZlt{}/shortdesc\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/parameter\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}parameter} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}ipaddr\PYGZdq{}} \PYG{n+na}{unique=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{required=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{deprecated=}\PYG{l+s}{\PYGZdq{}1\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}getopt} \PYG{n+na}{mixed=}\PYG{l+s}{\PYGZdq{}\PYGZhy{}a, \PYGZhy{}\PYGZhy{}ip=[ip]\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}content} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}string\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}shortdesc} \PYG{n+na}{lang=}\PYG{l+s}{\PYGZdq{}en\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}IP address or hostname of fencing device\PYG{n+nt}{\PYGZlt{}/shortdesc\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/parameter\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}parameter} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}ipport\PYGZdq{}} \PYG{n+na}{unique=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{required=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}getopt} \PYG{n+na}{mixed=}\PYG{l+s}{\PYGZdq{}\PYGZhy{}u, \PYGZhy{}\PYGZhy{}ipport=[port]\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}content} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}integer\PYGZdq{}} \PYG{n+na}{default=}\PYG{l+s}{\PYGZdq{}623\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}shortdesc} \PYG{n+na}{lang=}\PYG{l+s}{\PYGZdq{}en\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}TCP/UDP port to use for connection with device\PYG{n+nt}{\PYGZlt{}/shortdesc\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/parameter\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}parameter} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}lanplus\PYGZdq{}} \PYG{n+na}{unique=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{required=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}getopt} \PYG{n+na}{mixed=}\PYG{l+s}{\PYGZdq{}\PYGZhy{}P, \PYGZhy{}\PYGZhy{}lanplus\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}content} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}boolean\PYGZdq{}} \PYG{n+na}{default=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}shortdesc} \PYG{n+na}{lang=}\PYG{l+s}{\PYGZdq{}en\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}Use Lanplus to improve security of connection\PYG{n+nt}{\PYGZlt{}/shortdesc\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/parameter\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}parameter} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}login\PYGZdq{}} \PYG{n+na}{unique=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{required=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{deprecated=}\PYG{l+s}{\PYGZdq{}1\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}getopt} \PYG{n+na}{mixed=}\PYG{l+s}{\PYGZdq{}\PYGZhy{}l, \PYGZhy{}\PYGZhy{}username=[name]\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}content} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}string\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}shortdesc} \PYG{n+na}{lang=}\PYG{l+s}{\PYGZdq{}en\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}Login name\PYG{n+nt}{\PYGZlt{}/shortdesc\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/parameter\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}parameter} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}method\PYGZdq{}} \PYG{n+na}{unique=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{required=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}getopt} \PYG{n+na}{mixed=}\PYG{l+s}{\PYGZdq{}\PYGZhy{}m, \PYGZhy{}\PYGZhy{}method=[method]\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}content} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}select\PYGZdq{}} \PYG{n+na}{default=}\PYG{l+s}{\PYGZdq{}onoff\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}option} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}onoff\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}option} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}cycle\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}/content\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}shortdesc} \PYG{n+na}{lang=}\PYG{l+s}{\PYGZdq{}en\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}Method to fence\PYG{n+nt}{\PYGZlt{}/shortdesc\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/parameter\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}parameter} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}passwd\PYGZdq{}} \PYG{n+na}{unique=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{required=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{deprecated=}\PYG{l+s}{\PYGZdq{}1\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}getopt} \PYG{n+na}{mixed=}\PYG{l+s}{\PYGZdq{}\PYGZhy{}p, \PYGZhy{}\PYGZhy{}password=[password]\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}content} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}string\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}shortdesc} \PYG{n+na}{lang=}\PYG{l+s}{\PYGZdq{}en\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}Login password or passphrase\PYG{n+nt}{\PYGZlt{}/shortdesc\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/parameter\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}parameter} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}passwd\PYGZus{}script\PYGZdq{}} \PYG{n+na}{unique=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{required=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{deprecated=}\PYG{l+s}{\PYGZdq{}1\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}getopt} \PYG{n+na}{mixed=}\PYG{l+s}{\PYGZdq{}\PYGZhy{}S, \PYGZhy{}\PYGZhy{}password\PYGZhy{}script=[script]\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}content} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}string\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}shortdesc} \PYG{n+na}{lang=}\PYG{l+s}{\PYGZdq{}en\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}Script to run to retrieve password\PYG{n+nt}{\PYGZlt{}/shortdesc\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/parameter\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}parameter} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}password\PYGZdq{}} \PYG{n+na}{unique=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{required=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{obsoletes=}\PYG{l+s}{\PYGZdq{}passwd\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}getopt} \PYG{n+na}{mixed=}\PYG{l+s}{\PYGZdq{}\PYGZhy{}p, \PYGZhy{}\PYGZhy{}password=[password]\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}content} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}string\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}shortdesc} \PYG{n+na}{lang=}\PYG{l+s}{\PYGZdq{}en\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}Login password or passphrase\PYG{n+nt}{\PYGZlt{}/shortdesc\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/parameter\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}parameter} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}password\PYGZus{}script\PYGZdq{}} \PYG{n+na}{unique=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{required=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{obsoletes=}\PYG{l+s}{\PYGZdq{}passwd\PYGZus{}script\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}getopt} \PYG{n+na}{mixed=}\PYG{l+s}{\PYGZdq{}\PYGZhy{}S, \PYGZhy{}\PYGZhy{}password\PYGZhy{}script=[script]\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}content} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}string\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}shortdesc} \PYG{n+na}{lang=}\PYG{l+s}{\PYGZdq{}en\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}Script to run to retrieve password\PYG{n+nt}{\PYGZlt{}/shortdesc\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/parameter\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}parameter} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}plug\PYGZdq{}} \PYG{n+na}{unique=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{required=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{obsoletes=}\PYG{l+s}{\PYGZdq{}port\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}getopt} \PYG{n+na}{mixed=}\PYG{l+s}{\PYGZdq{}\PYGZhy{}n, \PYGZhy{}\PYGZhy{}plug=[ip]\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}content} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}string\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}shortdesc} \PYG{n+na}{lang=}\PYG{l+s}{\PYGZdq{}en\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}IP address or hostname of fencing device (together with \PYGZhy{}\PYGZhy{}port\PYGZhy{}as\PYGZhy{}ip)\PYG{n+nt}{\PYGZlt{}/shortdesc\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/parameter\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}parameter} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}port\PYGZdq{}} \PYG{n+na}{unique=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{required=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{deprecated=}\PYG{l+s}{\PYGZdq{}1\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}getopt} \PYG{n+na}{mixed=}\PYG{l+s}{\PYGZdq{}\PYGZhy{}n, \PYGZhy{}\PYGZhy{}plug=[ip]\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}content} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}string\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}shortdesc} \PYG{n+na}{lang=}\PYG{l+s}{\PYGZdq{}en\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}IP address or hostname of fencing device (together with \PYGZhy{}\PYGZhy{}port\PYGZhy{}as\PYGZhy{}ip)\PYG{n+nt}{\PYGZlt{}/shortdesc\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/parameter\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}parameter} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}privlvl\PYGZdq{}} \PYG{n+na}{unique=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{required=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}getopt} \PYG{n+na}{mixed=}\PYG{l+s}{\PYGZdq{}\PYGZhy{}L, \PYGZhy{}\PYGZhy{}privlvl=[level]\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}content} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}select\PYGZdq{}} \PYG{n+na}{default=}\PYG{l+s}{\PYGZdq{}administrator\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}option} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}callback\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}option} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}user\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}option} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}operator\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}option} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}administrator\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}/content\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}shortdesc} \PYG{n+na}{lang=}\PYG{l+s}{\PYGZdq{}en\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}Privilege level on IPMI device\PYG{n+nt}{\PYGZlt{}/shortdesc\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/parameter\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}parameter} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}target\PYGZdq{}} \PYG{n+na}{unique=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{required=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}getopt} \PYG{n+na}{mixed=}\PYG{l+s}{\PYGZdq{}\PYGZhy{}\PYGZhy{}target=[targetaddress]\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}content} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}string\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}shortdesc} \PYG{n+na}{lang=}\PYG{l+s}{\PYGZdq{}en\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}Bridge IPMI requests to the remote target address\PYG{n+nt}{\PYGZlt{}/shortdesc\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/parameter\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}parameter} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}username\PYGZdq{}} \PYG{n+na}{unique=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{required=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{obsoletes=}\PYG{l+s}{\PYGZdq{}login\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}getopt} \PYG{n+na}{mixed=}\PYG{l+s}{\PYGZdq{}\PYGZhy{}l, \PYGZhy{}\PYGZhy{}username=[name]\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}content} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}string\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}shortdesc} \PYG{n+na}{lang=}\PYG{l+s}{\PYGZdq{}en\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}Login name\PYG{n+nt}{\PYGZlt{}/shortdesc\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/parameter\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}parameter} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}quiet\PYGZdq{}} \PYG{n+na}{unique=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{required=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}getopt} \PYG{n+na}{mixed=}\PYG{l+s}{\PYGZdq{}\PYGZhy{}q, \PYGZhy{}\PYGZhy{}quiet\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}content} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}boolean\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}shortdesc} \PYG{n+na}{lang=}\PYG{l+s}{\PYGZdq{}en\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}Disable logging to stderr. Does not affect \PYGZhy{}\PYGZhy{}verbose or \PYGZhy{}\PYGZhy{}debug\PYGZhy{}file or logging to syslog.\PYG{n+nt}{\PYGZlt{}/shortdesc\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/parameter\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}parameter} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}verbose\PYGZdq{}} \PYG{n+na}{unique=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{required=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}getopt} \PYG{n+na}{mixed=}\PYG{l+s}{\PYGZdq{}\PYGZhy{}v, \PYGZhy{}\PYGZhy{}verbose\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}content} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}boolean\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}shortdesc} \PYG{n+na}{lang=}\PYG{l+s}{\PYGZdq{}en\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}Verbose mode\PYG{n+nt}{\PYGZlt{}/shortdesc\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/parameter\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}parameter} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}debug\PYGZdq{}} \PYG{n+na}{unique=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{required=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{deprecated=}\PYG{l+s}{\PYGZdq{}1\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}getopt} \PYG{n+na}{mixed=}\PYG{l+s}{\PYGZdq{}\PYGZhy{}D, \PYGZhy{}\PYGZhy{}debug\PYGZhy{}file=[debugfile]\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}content} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}string\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}shortdesc} \PYG{n+na}{lang=}\PYG{l+s}{\PYGZdq{}en\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}Write debug information to given file\PYG{n+nt}{\PYGZlt{}/shortdesc\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/parameter\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}parameter} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}debug\PYGZus{}file\PYGZdq{}} \PYG{n+na}{unique=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{required=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{obsoletes=}\PYG{l+s}{\PYGZdq{}debug\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}getopt} \PYG{n+na}{mixed=}\PYG{l+s}{\PYGZdq{}\PYGZhy{}D, \PYGZhy{}\PYGZhy{}debug\PYGZhy{}file=[debugfile]\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}content} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}string\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}shortdesc} \PYG{n+na}{lang=}\PYG{l+s}{\PYGZdq{}en\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}Write debug information to given file\PYG{n+nt}{\PYGZlt{}/shortdesc\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/parameter\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}parameter} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}version\PYGZdq{}} \PYG{n+na}{unique=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{required=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}getopt} \PYG{n+na}{mixed=}\PYG{l+s}{\PYGZdq{}\PYGZhy{}V, \PYGZhy{}\PYGZhy{}version\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}content} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}boolean\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}shortdesc} \PYG{n+na}{lang=}\PYG{l+s}{\PYGZdq{}en\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}Display version information and exit\PYG{n+nt}{\PYGZlt{}/shortdesc\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/parameter\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}parameter} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}help\PYGZdq{}} \PYG{n+na}{unique=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{required=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}getopt} \PYG{n+na}{mixed=}\PYG{l+s}{\PYGZdq{}\PYGZhy{}h, \PYGZhy{}\PYGZhy{}help\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}content} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}boolean\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}shortdesc} \PYG{n+na}{lang=}\PYG{l+s}{\PYGZdq{}en\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}Display help and exit\PYG{n+nt}{\PYGZlt{}/shortdesc\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/parameter\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}parameter} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}delay\PYGZdq{}} \PYG{n+na}{unique=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{required=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}getopt} \PYG{n+na}{mixed=}\PYG{l+s}{\PYGZdq{}\PYGZhy{}\PYGZhy{}delay=[seconds]\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}content} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}second\PYGZdq{}} \PYG{n+na}{default=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}shortdesc} \PYG{n+na}{lang=}\PYG{l+s}{\PYGZdq{}en\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}Wait X seconds before fencing is started\PYG{n+nt}{\PYGZlt{}/shortdesc\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/parameter\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}parameter} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}ipmitool\PYGZus{}path\PYGZdq{}} \PYG{n+na}{unique=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{required=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}getopt} \PYG{n+na}{mixed=}\PYG{l+s}{\PYGZdq{}\PYGZhy{}\PYGZhy{}ipmitool\PYGZhy{}path=[path]\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}content} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}string\PYGZdq{}} \PYG{n+na}{default=}\PYG{l+s}{\PYGZdq{}/usr/bin/ipmitool\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}shortdesc} \PYG{n+na}{lang=}\PYG{l+s}{\PYGZdq{}en\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}Path to ipmitool binary\PYG{n+nt}{\PYGZlt{}/shortdesc\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/parameter\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}parameter} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}login\PYGZus{}timeout\PYGZdq{}} \PYG{n+na}{unique=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{required=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}getopt} \PYG{n+na}{mixed=}\PYG{l+s}{\PYGZdq{}\PYGZhy{}\PYGZhy{}login\PYGZhy{}timeout=[seconds]\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}content} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}second\PYGZdq{}} \PYG{n+na}{default=}\PYG{l+s}{\PYGZdq{}5\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}shortdesc} \PYG{n+na}{lang=}\PYG{l+s}{\PYGZdq{}en\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}Wait X seconds for cmd prompt after login\PYG{n+nt}{\PYGZlt{}/shortdesc\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/parameter\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}parameter} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}port\PYGZus{}as\PYGZus{}ip\PYGZdq{}} \PYG{n+na}{unique=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{required=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}getopt} \PYG{n+na}{mixed=}\PYG{l+s}{\PYGZdq{}\PYGZhy{}\PYGZhy{}port\PYGZhy{}as\PYGZhy{}ip\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}content} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}boolean\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}shortdesc} \PYG{n+na}{lang=}\PYG{l+s}{\PYGZdq{}en\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}Make \PYGZdq{}port/plug\PYGZdq{} to be an alias to IP address\PYG{n+nt}{\PYGZlt{}/shortdesc\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/parameter\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}parameter} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}power\PYGZus{}timeout\PYGZdq{}} \PYG{n+na}{unique=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{required=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}getopt} \PYG{n+na}{mixed=}\PYG{l+s}{\PYGZdq{}\PYGZhy{}\PYGZhy{}power\PYGZhy{}timeout=[seconds]\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}content} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}second\PYGZdq{}} \PYG{n+na}{default=}\PYG{l+s}{\PYGZdq{}20\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}shortdesc} \PYG{n+na}{lang=}\PYG{l+s}{\PYGZdq{}en\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}Test X seconds for status change after ON/OFF\PYG{n+nt}{\PYGZlt{}/shortdesc\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/parameter\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}parameter} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}power\PYGZus{}wait\PYGZdq{}} \PYG{n+na}{unique=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{required=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}getopt} \PYG{n+na}{mixed=}\PYG{l+s}{\PYGZdq{}\PYGZhy{}\PYGZhy{}power\PYGZhy{}wait=[seconds]\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}content} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}second\PYGZdq{}} \PYG{n+na}{default=}\PYG{l+s}{\PYGZdq{}2\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}shortdesc} \PYG{n+na}{lang=}\PYG{l+s}{\PYGZdq{}en\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}Wait X seconds after issuing ON/OFF\PYG{n+nt}{\PYGZlt{}/shortdesc\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/parameter\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}parameter} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}shell\PYGZus{}timeout\PYGZdq{}} \PYG{n+na}{unique=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{required=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}getopt} \PYG{n+na}{mixed=}\PYG{l+s}{\PYGZdq{}\PYGZhy{}\PYGZhy{}shell\PYGZhy{}timeout=[seconds]\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}content} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}second\PYGZdq{}} \PYG{n+na}{default=}\PYG{l+s}{\PYGZdq{}3\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}shortdesc} \PYG{n+na}{lang=}\PYG{l+s}{\PYGZdq{}en\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}Wait X seconds for cmd prompt after issuing command\PYG{n+nt}{\PYGZlt{}/shortdesc\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/parameter\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}parameter} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}retry\PYGZus{}on\PYGZdq{}} \PYG{n+na}{unique=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{required=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}getopt} \PYG{n+na}{mixed=}\PYG{l+s}{\PYGZdq{}\PYGZhy{}\PYGZhy{}retry\PYGZhy{}on=[attempts]\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}content} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}integer\PYGZdq{}} \PYG{n+na}{default=}\PYG{l+s}{\PYGZdq{}1\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}shortdesc} \PYG{n+na}{lang=}\PYG{l+s}{\PYGZdq{}en\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}Count of attempts to retry power on\PYG{n+nt}{\PYGZlt{}/shortdesc\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/parameter\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}parameter} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}sudo\PYGZdq{}} \PYG{n+na}{unique=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{required=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{deprecated=}\PYG{l+s}{\PYGZdq{}1\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}getopt} \PYG{n+na}{mixed=}\PYG{l+s}{\PYGZdq{}\PYGZhy{}\PYGZhy{}use\PYGZhy{}sudo\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}content} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}boolean\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}shortdesc} \PYG{n+na}{lang=}\PYG{l+s}{\PYGZdq{}en\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}Use sudo (without password) when calling 3rd party software\PYG{n+nt}{\PYGZlt{}/shortdesc\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/parameter\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}parameter} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}use\PYGZus{}sudo\PYGZdq{}} \PYG{n+na}{unique=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{required=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{obsoletes=}\PYG{l+s}{\PYGZdq{}sudo\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}getopt} \PYG{n+na}{mixed=}\PYG{l+s}{\PYGZdq{}\PYGZhy{}\PYGZhy{}use\PYGZhy{}sudo\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}content} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}boolean\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}shortdesc} \PYG{n+na}{lang=}\PYG{l+s}{\PYGZdq{}en\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}Use sudo (without password) when calling 3rd party software\PYG{n+nt}{\PYGZlt{}/shortdesc\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/parameter\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}parameter} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}sudo\PYGZus{}path\PYGZdq{}} \PYG{n+na}{unique=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{required=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}getopt} \PYG{n+na}{mixed=}\PYG{l+s}{\PYGZdq{}\PYGZhy{}\PYGZhy{}sudo\PYGZhy{}path=[path]\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}content} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}string\PYGZdq{}} \PYG{n+na}{default=}\PYG{l+s}{\PYGZdq{}/usr/bin/sudo\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}shortdesc} \PYG{n+na}{lang=}\PYG{l+s}{\PYGZdq{}en\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}Path to sudo binary\PYG{n+nt}{\PYGZlt{}/shortdesc\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/parameter\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}/parameters\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}actions}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}action} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}on\PYGZdq{}} \PYG{n+na}{automatic=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}action} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}off\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}action} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}reboot\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}action} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}status\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}action} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}monitor\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}action} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}metadata\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}action} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}manpage\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}action} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}validate\PYGZhy{}all\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}action} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}diag\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}action} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}stop\PYGZdq{}} \PYG{n+na}{timeout=}\PYG{l+s}{\PYGZdq{}20s\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}action} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}start\PYGZdq{}} \PYG{n+na}{timeout=}\PYG{l+s}{\PYGZdq{}20s\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}/actions\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/resource\PYGZhy{}agent\PYGZgt{}}
\end{sphinxVerbatim}

Once we’ve decided what parameter values we think we need, it is a good idea
to run the fence agent’s status action manually, to verify that our values
work correctly:

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYGZsh{} fence\PYGZus{}ipmilan \PYGZhy{}\PYGZhy{}lanplus \PYGZhy{}a 192.0.2.1 \PYGZhy{}l testuser \PYGZhy{}p abc123 \PYGZhy{}o status

Chassis Power is on
\end{sphinxVerbatim}

\item {} 
Based on that, we might create a fencing resource configuration like this in
\sphinxcode{\sphinxupquote{stonith.xml}} (or any file name, just use the same name with \sphinxcode{\sphinxupquote{cibadmin}}
later):

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}primitive} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}Fencing\PYGZhy{}pcmk\PYGZhy{}1\PYGZdq{}} \PYG{n+na}{class=}\PYG{l+s}{\PYGZdq{}stonith\PYGZdq{}} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}fence\PYGZus{}ipmilan\PYGZdq{}} \PYG{n+nt}{\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}instance\PYGZus{}attributes} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}Fencing\PYGZhy{}params\PYGZdq{}} \PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}Fencing\PYGZhy{}lanplus\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}lanplus\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}1\PYGZdq{}} \PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}Fencing\PYGZhy{}ip\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}ip\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}192.0.2.1\PYGZdq{}} \PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}Fencing\PYGZhy{}password\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}password\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}testuser\PYGZdq{}} \PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}Fencing\PYGZhy{}username\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}username\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}abc123\PYGZdq{}} \PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}/instance\PYGZus{}attributes\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}operations} \PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}op} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}Fencing\PYGZhy{}monitor\PYGZhy{}10m\PYGZdq{}} \PYG{n+na}{interval=}\PYG{l+s}{\PYGZdq{}10m\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}monitor\PYGZdq{}} \PYG{n+na}{timeout=}\PYG{l+s}{\PYGZdq{}300s\PYGZdq{}} \PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}/operations\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/primitive\PYGZgt{}}
\end{sphinxVerbatim}

\begin{sphinxadmonition}{note}{Note:}
Even though the man page shows that the \sphinxcode{\sphinxupquote{action}} parameter is
supported, we do not provide that in the resource configuration.
Pacemaker will supply an appropriate action whenever the fence device
must be used.
\end{sphinxadmonition}

\item {} 
In this case, we don’t need to configure \sphinxcode{\sphinxupquote{pcmk\_host\_map}} because
\sphinxcode{\sphinxupquote{fence\_ipmilan}} ignores the target node name and instead uses its
\sphinxcode{\sphinxupquote{ip}} parameter to know how to contact the IPMI controller.

\item {} 
We do need to let Pacemaker know which cluster node can be fenced by this
device, since \sphinxcode{\sphinxupquote{fence\_ipmilan}} doesn’t support the \sphinxcode{\sphinxupquote{list}} action. Add
a line like this to the agent’s instance attributes:

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}Fencing\PYGZhy{}pcmk\PYGZus{}host\PYGZus{}list\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}pcmk\PYGZus{}host\PYGZus{}list\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}pcmk\PYGZhy{}1\PYGZdq{}} \PYG{n+nt}{/\PYGZgt{}}
\end{sphinxVerbatim}

\item {} 
We don’t need to configure \sphinxcode{\sphinxupquote{pcmk\_host\_argument}} since \sphinxcode{\sphinxupquote{ip}} is all the
fence agent needs (it ignores the target name).

\item {} 
Make the configuration active:

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYGZsh{} cibadmin \PYGZhy{}\PYGZhy{}create \PYGZhy{}\PYGZhy{}scope resources \PYGZhy{}\PYGZhy{}xml\PYGZhy{}file stonith.xml
\end{sphinxVerbatim}

\item {} 
Set \sphinxcode{\sphinxupquote{stonith-enabled}} to true (this only has to be done once):

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYGZsh{} crm\PYGZus{}attribute \PYGZhy{}\PYGZhy{}type crm\PYGZus{}config \PYGZhy{}\PYGZhy{}name stonith\PYGZhy{}enabled \PYGZhy{}\PYGZhy{}update true
\end{sphinxVerbatim}

\item {} 
Since our cluster is still in testing, we can reboot \sphinxcode{\sphinxupquote{pcmk-1}} without
bothering anyone, so we’ll test our fencing configuration by running this
from one of the other cluster nodes:

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYGZsh{} stonith\PYGZus{}admin \PYGZhy{}\PYGZhy{}reboot pcmk\PYGZhy{}1
\end{sphinxVerbatim}

Then we will verify that the node did, in fact, reboot.

\end{enumerate}

We can repeat that process to create a separate fencing resource for each node.

With some other fence device types, a single fencing resource is able to be
used for all nodes. In fact, we could do that with \sphinxcode{\sphinxupquote{fence\_ipmilan}}, using the
\sphinxcode{\sphinxupquote{port-as-ip}} parameter along with \sphinxcode{\sphinxupquote{pcmk\_host\_map}}. Either approach is
fine.

\index{fencing!topology}\index{fencing-topology}\index{fencing-level}\ignorespaces 

\subsection{Fencing Topologies}
\label{\detokenize{fencing:fencing-topologies}}\label{\detokenize{fencing:index-31}}
Pacemaker supports fencing nodes with multiple devices through a feature called
\sphinxstyleemphasis{fencing topologies}. Fencing topologies may be used to provide alternative
devices in case one fails, or to require multiple devices to all be executed
successfully in order to consider the node successfully fenced, or even a
combination of the two.

Create the individual devices as you normally would, then define one or more
\sphinxcode{\sphinxupquote{fencing-level}} entries in the \sphinxcode{\sphinxupquote{fencing-topology}} section of the
configuration.
\begin{itemize}
\item {} 
Each fencing level is attempted in order of ascending \sphinxcode{\sphinxupquote{index}}. Allowed
values are 1 through 9.

\item {} 
If a device fails, processing terminates for the current level. No further
devices in that level are exercised, and the next level is attempted instead.

\item {} 
If the operation succeeds for all the listed devices in a level, the level is
deemed to have passed.

\item {} 
The operation is finished when a level has passed (success), or all levels
have been attempted (failed).

\item {} 
If the operation failed, the next step is determined by the scheduler and/or
the controller.

\end{itemize}

Some possible uses of topologies include:
\begin{itemize}
\item {} 
Try on-board IPMI, then an intelligent power switch if that fails

\item {} 
Try fabric fencing of both disk and network, then fall back to power fencing
if either fails

\item {} 
Wait up to a certain time for a kernel dump to complete, then cut power to
the node

\end{itemize}


\begin{savenotes}\sphinxatlongtablestart\begin{longtable}{|\X{1}{5}|\X{4}{5}|}
\caption{\sphinxstylestrong{Attributes of a fencing-level Element}\strut}\label{\detokenize{fencing:id5}}\\*[\sphinxlongtablecapskipadjust]
\hline
\sphinxstyletheadfamily 
Attribute
&\sphinxstyletheadfamily 
Description
\\
\hline
\endfirsthead

\multicolumn{2}{c}%
{\makebox[0pt]{\sphinxtablecontinued{\tablename\ \thetable{} -- continued from previous page}}}\\
\hline
\sphinxstyletheadfamily 
Attribute
&\sphinxstyletheadfamily 
Description
\\
\hline
\endhead

\hline
\multicolumn{2}{r}{\makebox[0pt][r]{\sphinxtablecontinued{Continued on next page}}}\\
\endfoot

\endlastfoot

id
&
\index{fencing-level!id}\index{id!fencing-level}\ignorespaces 
A unique name for this element (required)
\\
\hline
target
&
\index{fencing-level!target}\index{target!fencing-level}\ignorespaces 
The name of a single node to which this level applies
\\
\hline
target-pattern
&
\index{fencing-level!target-pattern}\index{target-pattern!fencing-level}\ignorespaces 
An extended regular expression (as defined in \sphinxhref{https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1\_chap09.html\#tag\_09\_04}{POSIX})
matching the names of nodes to which this level applies
\\
\hline
target-attribute
&
\index{fencing-level!target-attribute}\index{target-attribute!fencing-level}\ignorespaces 
The name of a node attribute that is set (to \sphinxcode{\sphinxupquote{target-value}}) for nodes to which this
level applies
\\
\hline
target-value
&
\index{fencing-level!target-value}\index{target-value!fencing-level}\ignorespaces 
The node attribute value (of \sphinxcode{\sphinxupquote{target-attribute}}) that is set for nodes to which this
level applies
\\
\hline
index
&
\index{fencing-level!index}\index{index!fencing-level}\ignorespaces 
The order in which to attempt the levels. Levels are attempted in ascending order
\sphinxstyleemphasis{until one succeeds}. Valid values are 1 through 9.
\\
\hline
devices
&
\index{fencing-level!devices}\index{devices!fencing-level}\ignorespaces 
A comma-separated list of devices that must all be tried for this level
\\
\hline
\end{longtable}\sphinxatlongtableend\end{savenotes}

\begin{sphinxadmonition}{note}{Note:}
\sphinxstylestrong{Fencing topology with different devices for different nodes}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}cib} \PYG{n+na}{crm\PYGZus{}feature\PYGZus{}set=}\PYG{l+s}{\PYGZdq{}3.6.0\PYGZdq{}} \PYG{n+na}{validate\PYGZhy{}with=}\PYG{l+s}{\PYGZdq{}pacemaker\PYGZhy{}3.5\PYGZdq{}} \PYG{n+na}{admin\PYGZus{}epoch=}\PYG{l+s}{\PYGZdq{}1\PYGZdq{}} \PYG{n+na}{epoch=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{num\PYGZus{}updates=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}configuration}\PYG{n+nt}{\PYGZgt{}}
    ...
    \PYG{n+nt}{\PYGZlt{}fencing\PYGZhy{}topology}\PYG{n+nt}{\PYGZgt{}}
      \PYG{c}{\PYGZlt{}!\PYGZhy{}\PYGZhy{}}\PYG{c}{ For pcmk}\PYG{c}{\PYGZhy{}}\PYG{c}{1, try poison}\PYG{c}{\PYGZhy{}}\PYG{c}{pill and fail back to power }\PYG{c}{\PYGZhy{}\PYGZhy{}\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}fencing\PYGZhy{}level} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}f\PYGZhy{}p1.1\PYGZdq{}} \PYG{n+na}{target=}\PYG{l+s}{\PYGZdq{}pcmk\PYGZhy{}1\PYGZdq{}} \PYG{n+na}{index=}\PYG{l+s}{\PYGZdq{}1\PYGZdq{}} \PYG{n+na}{devices=}\PYG{l+s}{\PYGZdq{}poison\PYGZhy{}pill\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}fencing\PYGZhy{}level} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}f\PYGZhy{}p1.2\PYGZdq{}} \PYG{n+na}{target=}\PYG{l+s}{\PYGZdq{}pcmk\PYGZhy{}1\PYGZdq{}} \PYG{n+na}{index=}\PYG{l+s}{\PYGZdq{}2\PYGZdq{}} \PYG{n+na}{devices=}\PYG{l+s}{\PYGZdq{}power\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}

      \PYG{c}{\PYGZlt{}!\PYGZhy{}\PYGZhy{}}\PYG{c}{ For pcmk}\PYG{c}{\PYGZhy{}}\PYG{c}{2, try disk and network, and fail back to power }\PYG{c}{\PYGZhy{}\PYGZhy{}\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}fencing\PYGZhy{}level} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}f\PYGZhy{}p2.1\PYGZdq{}} \PYG{n+na}{target=}\PYG{l+s}{\PYGZdq{}pcmk\PYGZhy{}2\PYGZdq{}} \PYG{n+na}{index=}\PYG{l+s}{\PYGZdq{}1\PYGZdq{}} \PYG{n+na}{devices=}\PYG{l+s}{\PYGZdq{}disk,network\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}fencing\PYGZhy{}level} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}f\PYGZhy{}p2.2\PYGZdq{}} \PYG{n+na}{target=}\PYG{l+s}{\PYGZdq{}pcmk\PYGZhy{}2\PYGZdq{}} \PYG{n+na}{index=}\PYG{l+s}{\PYGZdq{}2\PYGZdq{}} \PYG{n+na}{devices=}\PYG{l+s}{\PYGZdq{}power\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/fencing\PYGZhy{}topology\PYGZgt{}}
    ...
  \PYG{n+nt}{\PYGZlt{}configuration}\PYG{n+nt}{\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}status}\PYG{n+nt}{/\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/cib\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxadmonition}


\subsubsection{Example Dual-Layer, Dual-Device Fencing Topologies}
\label{\detokenize{fencing:example-dual-layer-dual-device-fencing-topologies}}
The following example illustrates an advanced use of \sphinxcode{\sphinxupquote{fencing-topology}} in a
cluster with the following properties:
\begin{itemize}
\item {} 
2 nodes (prod-mysql1 and prod-mysql2)

\item {} 
the nodes have IPMI controllers reachable at 192.0.2.1 and 192.0.2.2

\item {} 
the nodes each have two independent Power Supply Units (PSUs) connected to
two independent Power Distribution Units (PDUs) reachable at 198.51.100.1
(port 10 and port 11) and 203.0.113.1 (port 10 and port 11)

\item {} 
fencing via the IPMI controller uses the \sphinxcode{\sphinxupquote{fence\_ipmilan}} agent (1 fence device
per controller, with each device targeting a separate node)

\item {} 
fencing via the PDUs uses the \sphinxcode{\sphinxupquote{fence\_apc\_snmp}} agent (1 fence device per
PDU, with both devices targeting both nodes)

\item {} 
a random delay is used to lessen the chance of a “death match”

\item {} 
fencing topology is set to try IPMI fencing first then dual PDU fencing if
that fails

\end{itemize}

In a node failure scenario, Pacemaker will first select \sphinxcode{\sphinxupquote{fence\_ipmilan}} to
try to kill the faulty node. Using the fencing topology, if that method fails,
it will then move on to selecting \sphinxcode{\sphinxupquote{fence\_apc\_snmp}} twice (once for the first
PDU, then again for the second PDU).

The fence action is considered successful only if both PDUs report the required
status. If any of them fails, fencing loops back to the first fencing method,
\sphinxcode{\sphinxupquote{fence\_ipmilan}}, and so on, until the node is fenced or the fencing action is
cancelled.

\begin{sphinxadmonition}{note}{Note:}
\sphinxstylestrong{First fencing method: single IPMI device per target}

Each cluster node has it own dedicated IPMI controller that can be contacted
for fencing using the following primitives:

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}primitive} \PYG{n+na}{class=}\PYG{l+s}{\PYGZdq{}stonith\PYGZdq{}} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}fence\PYGZus{}prod\PYGZhy{}mysql1\PYGZus{}ipmi\PYGZdq{}} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}fence\PYGZus{}ipmilan\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}instance\PYGZus{}attributes} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}fence\PYGZus{}prod\PYGZhy{}mysql1\PYGZus{}ipmi\PYGZhy{}instance\PYGZus{}attributes\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}fence\PYGZus{}prod\PYGZhy{}mysql1\PYGZus{}ipmi\PYGZhy{}instance\PYGZus{}attributes\PYGZhy{}ipaddr\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}ipaddr\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}192.0.2.1\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}fence\PYGZus{}prod\PYGZhy{}mysql1\PYGZus{}ipmi\PYGZhy{}instance\PYGZus{}attributes\PYGZhy{}login\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}login\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}fencing\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}fence\PYGZus{}prod\PYGZhy{}mysql1\PYGZus{}ipmi\PYGZhy{}instance\PYGZus{}attributes\PYGZhy{}passwd\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}passwd\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}finishme\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}fence\PYGZus{}prod\PYGZhy{}mysql1\PYGZus{}ipmi\PYGZhy{}instance\PYGZus{}attributes\PYGZhy{}lanplus\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}lanplus\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}true\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}fence\PYGZus{}prod\PYGZhy{}mysql1\PYGZus{}ipmi\PYGZhy{}instance\PYGZus{}attributes\PYGZhy{}pcmk\PYGZus{}host\PYGZus{}list\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}pcmk\PYGZus{}host\PYGZus{}list\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}prod\PYGZhy{}mysql1\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}fence\PYGZus{}prod\PYGZhy{}mysql1\PYGZus{}ipmi\PYGZhy{}instance\PYGZus{}attributes\PYGZhy{}pcmk\PYGZus{}delay\PYGZus{}max\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}pcmk\PYGZus{}delay\PYGZus{}max\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}8s\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}/instance\PYGZus{}attributes\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/primitive\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}primitive} \PYG{n+na}{class=}\PYG{l+s}{\PYGZdq{}stonith\PYGZdq{}} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}fence\PYGZus{}prod\PYGZhy{}mysql2\PYGZus{}ipmi\PYGZdq{}} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}fence\PYGZus{}ipmilan\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}instance\PYGZus{}attributes} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}fence\PYGZus{}prod\PYGZhy{}mysql2\PYGZus{}ipmi\PYGZhy{}instance\PYGZus{}attributes\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}fence\PYGZus{}prod\PYGZhy{}mysql2\PYGZus{}ipmi\PYGZhy{}instance\PYGZus{}attributes\PYGZhy{}ipaddr\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}ipaddr\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}192.0.2.2\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}fence\PYGZus{}prod\PYGZhy{}mysql2\PYGZus{}ipmi\PYGZhy{}instance\PYGZus{}attributes\PYGZhy{}login\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}login\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}fencing\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}fence\PYGZus{}prod\PYGZhy{}mysql2\PYGZus{}ipmi\PYGZhy{}instance\PYGZus{}attributes\PYGZhy{}passwd\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}passwd\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}finishme\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}fence\PYGZus{}prod\PYGZhy{}mysql2\PYGZus{}ipmi\PYGZhy{}instance\PYGZus{}attributes\PYGZhy{}lanplus\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}lanplus\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}true\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}fence\PYGZus{}prod\PYGZhy{}mysql2\PYGZus{}ipmi\PYGZhy{}instance\PYGZus{}attributes\PYGZhy{}pcmk\PYGZus{}host\PYGZus{}list\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}pcmk\PYGZus{}host\PYGZus{}list\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}prod\PYGZhy{}mysql2\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}fence\PYGZus{}prod\PYGZhy{}mysql2\PYGZus{}ipmi\PYGZhy{}instance\PYGZus{}attributes\PYGZhy{}pcmk\PYGZus{}delay\PYGZus{}max\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}pcmk\PYGZus{}delay\PYGZus{}max\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}8s\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}/instance\PYGZus{}attributes\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/primitive\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxadmonition}

\begin{sphinxadmonition}{note}{Note:}
\sphinxstylestrong{Second fencing method: dual PDU devices}

Each cluster node also has 2 distinct power supplies controlled by 2
distinct PDUs:
\begin{itemize}
\item {} 
Node 1: PDU 1 port 10 and PDU 2 port 10

\item {} 
Node 2: PDU 1 port 11 and PDU 2 port 11

\end{itemize}

The matching fencing agents are configured as follows:

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}primitive} \PYG{n+na}{class=}\PYG{l+s}{\PYGZdq{}stonith\PYGZdq{}} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}fence\PYGZus{}apc1\PYGZdq{}} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}fence\PYGZus{}apc\PYGZus{}snmp\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}instance\PYGZus{}attributes} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}fence\PYGZus{}apc1\PYGZhy{}instance\PYGZus{}attributes\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}fence\PYGZus{}apc1\PYGZhy{}instance\PYGZus{}attributes\PYGZhy{}ipaddr\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}ipaddr\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}198.51.100.1\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}fence\PYGZus{}apc1\PYGZhy{}instance\PYGZus{}attributes\PYGZhy{}login\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}login\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}fencing\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}fence\PYGZus{}apc1\PYGZhy{}instance\PYGZus{}attributes\PYGZhy{}passwd\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}passwd\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}fencing\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}fence\PYGZus{}apc1\PYGZhy{}instance\PYGZus{}attributes\PYGZhy{}pcmk\PYGZus{}host\PYGZus{}list\PYGZdq{}}
       \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}pcmk\PYGZus{}host\PYGZus{}map\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}prod\PYGZhy{}mysql1:10;prod\PYGZhy{}mysql2:11\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}fence\PYGZus{}apc1\PYGZhy{}instance\PYGZus{}attributes\PYGZhy{}pcmk\PYGZus{}delay\PYGZus{}max\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}pcmk\PYGZus{}delay\PYGZus{}max\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}8s\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}/instance\PYGZus{}attributes\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/primitive\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}primitive} \PYG{n+na}{class=}\PYG{l+s}{\PYGZdq{}stonith\PYGZdq{}} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}fence\PYGZus{}apc2\PYGZdq{}} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}fence\PYGZus{}apc\PYGZus{}snmp\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}instance\PYGZus{}attributes} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}fence\PYGZus{}apc2\PYGZhy{}instance\PYGZus{}attributes\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}fence\PYGZus{}apc2\PYGZhy{}instance\PYGZus{}attributes\PYGZhy{}ipaddr\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}ipaddr\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}203.0.113.1\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}fence\PYGZus{}apc2\PYGZhy{}instance\PYGZus{}attributes\PYGZhy{}login\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}login\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}fencing\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}fence\PYGZus{}apc2\PYGZhy{}instance\PYGZus{}attributes\PYGZhy{}passwd\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}passwd\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}fencing\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}fence\PYGZus{}apc2\PYGZhy{}instance\PYGZus{}attributes\PYGZhy{}pcmk\PYGZus{}host\PYGZus{}list\PYGZdq{}}
       \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}pcmk\PYGZus{}host\PYGZus{}map\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}prod\PYGZhy{}mysql1:10;prod\PYGZhy{}mysql2:11\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}fence\PYGZus{}apc2\PYGZhy{}instance\PYGZus{}attributes\PYGZhy{}pcmk\PYGZus{}delay\PYGZus{}max\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}pcmk\PYGZus{}delay\PYGZus{}max\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}8s\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}/instance\PYGZus{}attributes\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/primitive\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxadmonition}

\begin{sphinxadmonition}{note}{Note:}
\sphinxstylestrong{Fencing topology}

Now that all the fencing resources are defined, it’s time to create the
right topology. We want to first fence using IPMI and if that does not work,
fence both PDUs to effectively and surely kill the node.

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}fencing\PYGZhy{}topology}\PYG{n+nt}{\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}fencing\PYGZhy{}level} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}level\PYGZhy{}1\PYGZhy{}1\PYGZdq{}} \PYG{n+na}{target=}\PYG{l+s}{\PYGZdq{}prod\PYGZhy{}mysql1\PYGZdq{}} \PYG{n+na}{index=}\PYG{l+s}{\PYGZdq{}1\PYGZdq{}} \PYG{n+na}{devices=}\PYG{l+s}{\PYGZdq{}fence\PYGZus{}prod\PYGZhy{}mysql1\PYGZus{}ipmi\PYGZdq{}} \PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}fencing\PYGZhy{}level} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}level\PYGZhy{}1\PYGZhy{}2\PYGZdq{}} \PYG{n+na}{target=}\PYG{l+s}{\PYGZdq{}prod\PYGZhy{}mysql1\PYGZdq{}} \PYG{n+na}{index=}\PYG{l+s}{\PYGZdq{}2\PYGZdq{}} \PYG{n+na}{devices=}\PYG{l+s}{\PYGZdq{}fence\PYGZus{}apc1,fence\PYGZus{}apc2\PYGZdq{}}  \PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}fencing\PYGZhy{}level} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}level\PYGZhy{}2\PYGZhy{}1\PYGZdq{}} \PYG{n+na}{target=}\PYG{l+s}{\PYGZdq{}prod\PYGZhy{}mysql2\PYGZdq{}} \PYG{n+na}{index=}\PYG{l+s}{\PYGZdq{}1\PYGZdq{}} \PYG{n+na}{devices=}\PYG{l+s}{\PYGZdq{}fence\PYGZus{}prod\PYGZhy{}mysql2\PYGZus{}ipmi\PYGZdq{}} \PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}fencing\PYGZhy{}level} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}level\PYGZhy{}2\PYGZhy{}2\PYGZdq{}} \PYG{n+na}{target=}\PYG{l+s}{\PYGZdq{}prod\PYGZhy{}mysql2\PYGZdq{}} \PYG{n+na}{index=}\PYG{l+s}{\PYGZdq{}2\PYGZdq{}} \PYG{n+na}{devices=}\PYG{l+s}{\PYGZdq{}fence\PYGZus{}apc1,fence\PYGZus{}apc2\PYGZdq{}}  \PYG{n+nt}{/\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/fencing\PYGZhy{}topology\PYGZgt{}}
\end{sphinxVerbatim}

In \sphinxcode{\sphinxupquote{fencing-topology}}, the lowest \sphinxcode{\sphinxupquote{index}} value for a target determines
its first fencing method.
\end{sphinxadmonition}


\subsection{Remapping Reboots}
\label{\detokenize{fencing:remapping-reboots}}
When the cluster needs to reboot a node, whether because \sphinxcode{\sphinxupquote{stonith-action}} is
\sphinxcode{\sphinxupquote{reboot}} or because a reboot was requested externally (such as by
\sphinxcode{\sphinxupquote{stonith\_admin -{-}reboot}}), it will remap that to other commands in two cases:
\begin{itemize}
\item {} 
If the chosen fencing device does not support the \sphinxcode{\sphinxupquote{reboot}} command, the
cluster will ask it to perform \sphinxcode{\sphinxupquote{off}} instead.

\item {} 
If a fencing topology level with multiple devices must be executed, the
cluster will ask all the devices to perform \sphinxcode{\sphinxupquote{off}}, then ask the devices to
perform \sphinxcode{\sphinxupquote{on}}.

\end{itemize}

To understand the second case, consider the example of a node with redundant
power supplies connected to intelligent power switches. Rebooting one switch
and then the other would have no effect on the node. Turning both switches off,
and then on, actually reboots the node.

In such a case, the fencing operation will be treated as successful as long as
the \sphinxcode{\sphinxupquote{off}} commands succeed, because then it is safe for the cluster to
recover any resources that were on the node. Timeouts and errors in the \sphinxcode{\sphinxupquote{on}}
phase will be logged but ignored.

When a reboot operation is remapped, any action-specific timeout for the
remapped action will be used (for example, \sphinxcode{\sphinxupquote{pcmk\_off\_timeout}} will be used
when executing the \sphinxcode{\sphinxupquote{off}} command, not \sphinxcode{\sphinxupquote{pcmk\_reboot\_timeout}}).

\index{alert}\index{resource!alert}\index{node!alert}\index{fencing!alert}\index{XML element!alert}\index{alert!XML element}\index{XML element!alerts}\index{alerts!XML element}\ignorespaces 

\section{Alerts}
\label{\detokenize{alerts:alerts}}\label{\detokenize{alerts:index-0}}\label{\detokenize{alerts::doc}}
\sphinxstyleemphasis{Alerts} may be configured to take some external action when a cluster event
occurs (node failure, resource starting or stopping, etc.).

\index{alert!agent}\index{agent!alert}\ignorespaces 

\subsection{Alert Agents}
\label{\detokenize{alerts:alert-agents}}\label{\detokenize{alerts:index-1}}
As with resource agents, the cluster calls an external program (an
\sphinxstyleemphasis{alert agent}) to handle alerts. The cluster passes information about the event
to the agent via environment variables. Agents can do anything desired with
this information (send an e-mail, log to a file, update a monitoring system,
etc.).

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Simple alert configuration}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}configuration}\PYG{n+nt}{\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}alerts}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}alert} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}my\PYGZhy{}alert\PYGZdq{}} \PYG{n+na}{path=}\PYG{l+s}{\PYGZdq{}/path/to/my\PYGZhy{}script.sh\PYGZdq{}} \PYG{n+nt}{/\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}/alerts\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/configuration\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

In the example above, the cluster will call \sphinxcode{\sphinxupquote{my-script.sh}} for each event.

Multiple alert agents may be configured; the cluster will call all of them for
each event.

Alert agents will be called only on cluster nodes. They will be called for
events involving Pacemaker Remote nodes, but they will never be called \sphinxstyleemphasis{on}
those nodes.

For more information about sample alert agents provided by Pacemaker and about
developing custom alert agents, see the \sphinxstyleemphasis{Pacemaker Administration} document.

\index{alert!recipient}\index{XML element!recipient}\index{recipient!XML element}\ignorespaces 

\subsection{Alert Recipients}
\label{\detokenize{alerts:alert-recipients}}\label{\detokenize{alerts:index-2}}
Usually, alerts are directed towards a recipient. Thus, each alert may be
additionally configured with one or more recipients. The cluster will call the
agent separately for each recipient.

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Alert configuration with recipient}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}configuration}\PYG{n+nt}{\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}alerts}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}alert} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}my\PYGZhy{}alert\PYGZdq{}} \PYG{n+na}{path=}\PYG{l+s}{\PYGZdq{}/path/to/my\PYGZhy{}script.sh\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
          \PYG{n+nt}{\PYGZlt{}recipient} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}my\PYGZhy{}alert\PYGZhy{}recipient\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}some\PYGZhy{}address\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}/alert\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}/alerts\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/configuration\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

In the above example, the cluster will call \sphinxcode{\sphinxupquote{my-script.sh}} for each event,
passing the recipient \sphinxcode{\sphinxupquote{some-address}} as an environment variable.

The recipient may be anything the alert agent can recognize \textendash{} an IP address,
an e-mail address, a file name, whatever the particular agent supports.

\index{alert!meta-attributes}\index{meta-attribute!alert meta-attributes}\ignorespaces 

\subsection{Alert Meta-Attributes}
\label{\detokenize{alerts:alert-meta-attributes}}\label{\detokenize{alerts:index-3}}
As with resources, meta-attributes can be configured for alerts to change
whether and how Pacemaker calls them.


\begin{savenotes}\sphinxatlongtablestart\begin{longtable}{|\X{1}{5}|\X{1}{5}|\X{3}{5}|}
\caption{\sphinxstylestrong{Meta-Attributes of an Alert}\strut}\label{\detokenize{alerts:id1}}\\*[\sphinxlongtablecapskipadjust]
\hline
\sphinxstyletheadfamily 
Meta-Attribute
&\sphinxstyletheadfamily 
Default
&\sphinxstyletheadfamily 
Description
\\
\hline
\endfirsthead

\multicolumn{3}{c}%
{\makebox[0pt]{\sphinxtablecontinued{\tablename\ \thetable{} -- continued from previous page}}}\\
\hline
\sphinxstyletheadfamily 
Meta-Attribute
&\sphinxstyletheadfamily 
Default
&\sphinxstyletheadfamily 
Description
\\
\hline
\endhead

\hline
\multicolumn{3}{r}{\makebox[0pt][r]{\sphinxtablecontinued{Continued on next page}}}\\
\endfoot

\endlastfoot

enabled
&
true
&
\index{alert!meta-attribute, enabled}\index{meta-attribute!enabled (alert)}\index{enabled!alert meta-attribute}\ignorespaces 
If false for an alert, the alert will not be used.
If true for an alert and false for a particular
recipient of that alert, that recipient will not be
used. \sphinxstyleemphasis{(since 2.1.6)}
\\
\hline
timestamp-format
&
\%H:\%M:\%S.\%06N
&
\index{alert!meta-attribute, timestamp-format}\index{meta-attribute!timestamp-format (alert)}\index{timestamp-format!alert meta-attribute}\ignorespaces 
Format the cluster will use when sending the
event’s timestamp to the agent. This is a string as
used with the \sphinxcode{\sphinxupquote{date(1)}} command.
\\
\hline
timeout
&
30s
&
\index{alert!meta-attribute, timeout}\index{meta-attribute!timeout (alert)}\index{timeout!alert meta-attribute}\ignorespaces 
If the alert agent does not complete within this
amount of time, it will be terminated.
\\
\hline
\end{longtable}\sphinxatlongtableend\end{savenotes}

Meta-attributes can be configured per alert and/or per recipient.

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Alert configuration with meta-attributes}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}configuration}\PYG{n+nt}{\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}alerts}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}alert} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}my\PYGZhy{}alert\PYGZdq{}} \PYG{n+na}{path=}\PYG{l+s}{\PYGZdq{}/path/to/my\PYGZhy{}script.sh\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
         \PYG{n+nt}{\PYGZlt{}meta\PYGZus{}attributes} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}my\PYGZhy{}alert\PYGZhy{}attributes\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
            \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}my\PYGZhy{}alert\PYGZhy{}attributes\PYGZhy{}timeout\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}timeout\PYGZdq{}}
                    \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}15s\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
         \PYG{n+nt}{\PYGZlt{}/meta\PYGZus{}attributes\PYGZgt{}}
         \PYG{n+nt}{\PYGZlt{}recipient} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}my\PYGZhy{}alert\PYGZhy{}recipient1\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}someuser@example.com\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
            \PYG{n+nt}{\PYGZlt{}meta\PYGZus{}attributes} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}my\PYGZhy{}alert\PYGZhy{}recipient1\PYGZhy{}attributes\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
               \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}my\PYGZhy{}alert\PYGZhy{}recipient1\PYGZhy{}timestamp\PYGZhy{}format\PYGZdq{}}
                       \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}timestamp\PYGZhy{}format\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}\PYGZpc{}D \PYGZpc{}H:\PYGZpc{}M\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
            \PYG{n+nt}{\PYGZlt{}/meta\PYGZus{}attributes\PYGZgt{}}
         \PYG{n+nt}{\PYGZlt{}/recipient\PYGZgt{}}
         \PYG{n+nt}{\PYGZlt{}recipient} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}my\PYGZhy{}alert\PYGZhy{}recipient2\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}otheruser@example.com\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
            \PYG{n+nt}{\PYGZlt{}meta\PYGZus{}attributes} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}my\PYGZhy{}alert\PYGZhy{}recipient2\PYGZhy{}attributes\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
               \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}my\PYGZhy{}alert\PYGZhy{}recipient2\PYGZhy{}timestamp\PYGZhy{}format\PYGZdq{}}
                       \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}timestamp\PYGZhy{}format\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}\PYGZpc{}c\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
            \PYG{n+nt}{\PYGZlt{}/meta\PYGZus{}attributes\PYGZgt{}}
         \PYG{n+nt}{\PYGZlt{}/recipient\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}/alert\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}/alerts\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/configuration\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

In the above example, the \sphinxcode{\sphinxupquote{my-script.sh}} will get called twice for each
event, with each call using a 15-second timeout. One call will be passed the
recipient \sphinxcode{\sphinxupquote{someuser@example.com}} and a timestamp in the format \sphinxcode{\sphinxupquote{\%D \%H:\%M}},
while the other call will be passed the recipient \sphinxcode{\sphinxupquote{otheruser@example.com}} and
a timestamp in the format \sphinxcode{\sphinxupquote{\%c}}.

\index{alert!instance attributes}\index{instance attribute!alert instance attributes}\ignorespaces 

\subsection{Alert Instance Attributes}
\label{\detokenize{alerts:alert-instance-attributes}}\label{\detokenize{alerts:index-7}}
As with resource agents, agent-specific configuration values may be configured
as instance attributes. These will be passed to the agent as additional
environment variables. The number, names and allowed values of these instance
attributes are completely up to the particular agent.

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Alert configuration with instance attributes}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}configuration}\PYG{n+nt}{\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}alerts}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}alert} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}my\PYGZhy{}alert\PYGZdq{}} \PYG{n+na}{path=}\PYG{l+s}{\PYGZdq{}/path/to/my\PYGZhy{}script.sh\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
         \PYG{n+nt}{\PYGZlt{}meta\PYGZus{}attributes} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}my\PYGZhy{}alert\PYGZhy{}attributes\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
            \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}my\PYGZhy{}alert\PYGZhy{}attributes\PYGZhy{}timeout\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}timeout\PYGZdq{}}
                    \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}15s\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
         \PYG{n+nt}{\PYGZlt{}/meta\PYGZus{}attributes\PYGZgt{}}
         \PYG{n+nt}{\PYGZlt{}instance\PYGZus{}attributes} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}my\PYGZhy{}alert\PYGZhy{}options\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
             \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}my\PYGZhy{}alert\PYGZhy{}options\PYGZhy{}debug\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}debug\PYGZdq{}}
                     \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}false\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
         \PYG{n+nt}{\PYGZlt{}/instance\PYGZus{}attributes\PYGZgt{}}
         \PYG{n+nt}{\PYGZlt{}recipient} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}my\PYGZhy{}alert\PYGZhy{}recipient1\PYGZdq{}}
                    \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}someuser@example.com\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}/alert\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}/alerts\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/configuration\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

\index{alert!filters}\index{XML element!select}\index{select!XML element}\index{XML element!select\_nodes}\index{select\_nodes!XML element}\index{XML element!select\_fencing}\index{select\_fencing!XML element}\index{XML element!select\_resources}\index{select\_resources!XML element}\index{XML element!select\_attributes}\index{select\_attributes!XML element}\index{XML element!attribute}\index{attribute!XML element}\ignorespaces 

\subsection{Alert Filters}
\label{\detokenize{alerts:alert-filters}}\label{\detokenize{alerts:index-8}}
By default, an alert agent will be called for node events, fencing events, and
resource events. An agent may choose to ignore certain types of events, but
there is still the overhead of calling it for those events. To eliminate that
overhead, you may select which types of events the agent should receive.

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Alert configuration to receive only node events and fencing events}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}configuration}\PYG{n+nt}{\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}alerts}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}alert} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}my\PYGZhy{}alert\PYGZdq{}} \PYG{n+na}{path=}\PYG{l+s}{\PYGZdq{}/path/to/my\PYGZhy{}script.sh\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
         \PYG{n+nt}{\PYGZlt{}select}\PYG{n+nt}{\PYGZgt{}}
            \PYG{n+nt}{\PYGZlt{}select\PYGZus{}nodes} \PYG{n+nt}{/\PYGZgt{}}
            \PYG{n+nt}{\PYGZlt{}select\PYGZus{}fencing} \PYG{n+nt}{/\PYGZgt{}}
         \PYG{n+nt}{\PYGZlt{}/select\PYGZgt{}}
         \PYG{n+nt}{\PYGZlt{}recipient} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}my\PYGZhy{}alert\PYGZhy{}recipient1\PYGZdq{}}
                    \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}someuser@example.com\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}/alert\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}/alerts\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/configuration\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

The possible options within \sphinxcode{\sphinxupquote{\textless{}select\textgreater{}}} are \sphinxcode{\sphinxupquote{\textless{}select\_nodes\textgreater{}}},
\sphinxcode{\sphinxupquote{\textless{}select\_fencing\textgreater{}}}, \sphinxcode{\sphinxupquote{\textless{}select\_resources\textgreater{}}}, and \sphinxcode{\sphinxupquote{\textless{}select\_attributes\textgreater{}}}.

With \sphinxcode{\sphinxupquote{\textless{}select\_attributes\textgreater{}}} (the only event type not enabled by default), the
agent will receive alerts when a node attribute changes. If you wish the agent
to be called only when certain attributes change, you can configure that as well.

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Alert configuration to be called when certain node attributes change}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}configuration}\PYG{n+nt}{\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}alerts}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}alert} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}my\PYGZhy{}alert\PYGZdq{}} \PYG{n+na}{path=}\PYG{l+s}{\PYGZdq{}/path/to/my\PYGZhy{}script.sh\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
         \PYG{n+nt}{\PYGZlt{}select}\PYG{n+nt}{\PYGZgt{}}
            \PYG{n+nt}{\PYGZlt{}select\PYGZus{}attributes}\PYG{n+nt}{\PYGZgt{}}
               \PYG{n+nt}{\PYGZlt{}attribute} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}alert\PYGZhy{}standby\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}standby\PYGZdq{}} \PYG{n+nt}{/\PYGZgt{}}
               \PYG{n+nt}{\PYGZlt{}attribute} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}alert\PYGZhy{}shutdown\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}shutdown\PYGZdq{}} \PYG{n+nt}{/\PYGZgt{}}
            \PYG{n+nt}{\PYGZlt{}/select\PYGZus{}attributes\PYGZgt{}}
         \PYG{n+nt}{\PYGZlt{}/select\PYGZgt{}}
         \PYG{n+nt}{\PYGZlt{}recipient} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}my\PYGZhy{}alert\PYGZhy{}recipient1\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}someuser@example.com\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}/alert\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}/alerts\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/configuration\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

Node attribute alerts are currently considered experimental. Alerts may be
limited to attributes set via \sphinxcode{\sphinxupquote{attrd\_updater}}, and agents may be called
multiple times with the same attribute value.

\index{rule}\ignorespaces 

\section{Rules}
\label{\detokenize{rules:rules}}\label{\detokenize{rules:index-0}}\label{\detokenize{rules:id1}}\label{\detokenize{rules::doc}}
Rules can be used to make your configuration more dynamic, allowing values to
change depending on the time or the value of a node attribute. Examples of
things rules are useful for:
\begin{itemize}
\item {} 
Set a higher value for {\hyperref[\detokenize{resources:resource-stickiness}]{\sphinxcrossref{\DUrole{std,std-ref}{resource-stickiness}}}}
during working hours, to minimize downtime, and a lower value on weekends, to
allow resources to move to their most preferred locations when people aren’t
around to notice.

\item {} 
Automatically place the cluster into maintenance mode during a scheduled
maintenance window.

\item {} 
Assign certain nodes and resources to a particular department via custom
node attributes and meta-attributes, and add a single location constraint
that restricts the department’s resources to run only on those nodes.

\end{itemize}

Each constraint type or property set that supports rules may contain one or more
\sphinxcode{\sphinxupquote{rule}} elements specifying conditions under which the constraint or properties
take effect. Examples later in this chapter will make this clearer.

\index{XML element!rule}\index{rule!XML element}\ignorespaces 

\subsection{Rule Properties}
\label{\detokenize{rules:rule-properties}}\label{\detokenize{rules:index-1}}

\begin{savenotes}\sphinxattablestart
\centering
\sphinxcapstartof{table}
\sphinxcaption{\sphinxstylestrong{Attributes of a rule Element}}\label{\detokenize{rules:id4}}
\sphinxaftercaption
\begin{tabular}[t]{|\X{1}{5}|\X{1}{5}|\X{3}{5}|}
\hline
\sphinxstyletheadfamily 
Attribute
&\sphinxstyletheadfamily 
Default
&\sphinxstyletheadfamily 
Description
\\
\hline
id
&&
\index{rule!id}\index{id!rule}\ignorespaces 
A unique name for this element (required)
\\
\hline
role
&
\sphinxcode{\sphinxupquote{Started}}
&
\index{rule!role}\index{role!rule}\ignorespaces 
The rule is in effect only when the
resource is in the specified role.
Allowed values are \sphinxcode{\sphinxupquote{Started}},
\sphinxcode{\sphinxupquote{Unpromoted}}, and \sphinxcode{\sphinxupquote{Promoted}}. A rule
with a \sphinxcode{\sphinxupquote{role}} of \sphinxcode{\sphinxupquote{Promoted}} cannot
determine the initial location of a clone
instance and will only affect which of
the active instances will be promoted.
\\
\hline
score
&&
\index{rule!score}\index{score!rule}\ignorespaces 
If this rule is used in a location
constraint and evaluates to true, apply
this score to the constraint. Only one of
\sphinxcode{\sphinxupquote{score}} and \sphinxcode{\sphinxupquote{score-attribute}} may be
used.
\\
\hline
score-attribute
&&
\index{rule!score-attribute}\index{score-attribute!rule}\ignorespaces 
If this rule is used in a location
constraint and evaluates to true, use the
value of this node attribute as the score
to apply to the constraint. Only one of
\sphinxcode{\sphinxupquote{score}} and \sphinxcode{\sphinxupquote{score-attribute}} may be
used.
\\
\hline
boolean-op
&
\sphinxcode{\sphinxupquote{and}}
&
\index{rule!boolean-op}\index{boolean-op!rule}\ignorespaces 
If this rule contains more than one
condition, a value of \sphinxcode{\sphinxupquote{and}} specifies
that the rule evaluates to true only if
all conditions are true, and a value of
\sphinxcode{\sphinxupquote{or}} specifies that the rule evaluates
to true if any condition is true.
\\
\hline
\end{tabular}
\par
\sphinxattableend\end{savenotes}

A \sphinxcode{\sphinxupquote{rule}} element must contain one or more conditions. A condition may be an
\sphinxcode{\sphinxupquote{expression}} element, a \sphinxcode{\sphinxupquote{date\_expression}} element, or another \sphinxcode{\sphinxupquote{rule}} element.

\index{rule!node attribute expression}\index{node attribute!rule expression}\index{XML element!expression}\index{expression!XML element}\ignorespaces 

\subsection{Node Attribute Expressions}
\label{\detokenize{rules:node-attribute-expressions}}\label{\detokenize{rules:index-7}}\label{\detokenize{rules:id2}}
Expressions are rule conditions based on the values of node attributes.


\begin{savenotes}\sphinxatlongtablestart\begin{longtable}{|\X{1}{6}|\X{2}{6}|\X{3}{6}|}
\caption{\sphinxstylestrong{Attributes of an expression Element}\strut}\label{\detokenize{rules:id5}}\\*[\sphinxlongtablecapskipadjust]
\hline
\sphinxstyletheadfamily 
Attribute
&\sphinxstyletheadfamily 
Default
&\sphinxstyletheadfamily 
Description
\\
\hline
\endfirsthead

\multicolumn{3}{c}%
{\makebox[0pt]{\sphinxtablecontinued{\tablename\ \thetable{} -- continued from previous page}}}\\
\hline
\sphinxstyletheadfamily 
Attribute
&\sphinxstyletheadfamily 
Default
&\sphinxstyletheadfamily 
Description
\\
\hline
\endhead

\hline
\multicolumn{3}{r}{\makebox[0pt][r]{\sphinxtablecontinued{Continued on next page}}}\\
\endfoot

\endlastfoot

id
&&
\index{expression!id}\index{id!expression}\ignorespaces 
A unique name for this element (required)
\\
\hline
attribute
&&
\index{expression!attribute}\index{attribute!expression}\ignorespaces 
The node attribute to test (required)
\\
\hline
type
&
The default type for
\sphinxcode{\sphinxupquote{lt}}, \sphinxcode{\sphinxupquote{gt}}, \sphinxcode{\sphinxupquote{lte}}, and
\sphinxcode{\sphinxupquote{gte}} operations is \sphinxcode{\sphinxupquote{number}}
if either value contains a
decimal point character, or
\sphinxcode{\sphinxupquote{integer}} otherwise. The
default type for all other
operations is \sphinxcode{\sphinxupquote{string}}. If a
numeric parse fails for either
value, then the values are
compared as type \sphinxcode{\sphinxupquote{string}}.
&
\index{expression!type}\index{type!expression}\ignorespaces 
How the node attributes should be
compared. Allowed values are \sphinxcode{\sphinxupquote{string}},
\sphinxcode{\sphinxupquote{integer}} \sphinxstyleemphasis{(since 2.0.5)}, \sphinxcode{\sphinxupquote{number}},
and \sphinxcode{\sphinxupquote{version}}. \sphinxcode{\sphinxupquote{integer}} truncates
floating-point values if necessary before
performing a 64-bit integer comparison.
\sphinxcode{\sphinxupquote{number}} performs a double-precision
floating-point comparison
\sphinxstyleemphasis{(32-bit integer before 2.0.5)}.
\\
\hline
operation
&&
\index{expression!operation}\index{operation!expression}\ignorespaces 
The comparison to perform (required).
Allowed values:
\begin{itemize}
\item {} \begin{description}
\item[{\sphinxcode{\sphinxupquote{lt:}} True if the node attribute value}] \leavevmode
is less than the comparison value

\end{description}

\item {} \begin{description}
\item[{\sphinxcode{\sphinxupquote{gt:}} True if the node attribute value}] \leavevmode
is greater than the comparison value

\end{description}

\item {} \begin{description}
\item[{\sphinxcode{\sphinxupquote{lte:}} True if the node attribute}] \leavevmode
value is less than or equal to the
comparison value

\end{description}

\item {} \begin{description}
\item[{\sphinxcode{\sphinxupquote{gte:}} True if the node attribute}] \leavevmode
value is greater than or equal to the
comparison value

\end{description}

\item {} \begin{description}
\item[{\sphinxcode{\sphinxupquote{eq:}} True if the node attribute value}] \leavevmode
is equal to the comparison value

\end{description}

\item {} \begin{description}
\item[{\sphinxcode{\sphinxupquote{ne:}} True if the node attribute value}] \leavevmode
is not equal to the comparison value

\end{description}

\item {} \begin{description}
\item[{\sphinxcode{\sphinxupquote{defined:}} True if the node has the}] \leavevmode
named attribute

\end{description}

\item {} \begin{description}
\item[{\sphinxcode{\sphinxupquote{not\_defined:}} True if the node does}] \leavevmode
not have the named attribute

\end{description}

\end{itemize}
\\
\hline
value
&&
\index{expression!value}\index{value!expression}\ignorespaces 
User-supplied value for comparison
(required for operations other than
\sphinxcode{\sphinxupquote{defined}} and \sphinxcode{\sphinxupquote{not\_defined}})
\\
\hline
value-source
&
\sphinxcode{\sphinxupquote{literal}}
&
\index{expression!value-source}\index{value-source!expression}\ignorespaces 
How the \sphinxcode{\sphinxupquote{value}} is derived. Allowed
values:
\begin{itemize}
\item {} 
\sphinxcode{\sphinxupquote{literal}}: \sphinxcode{\sphinxupquote{value}} is a literal
string to compare against

\item {} 
\sphinxcode{\sphinxupquote{param}}: \sphinxcode{\sphinxupquote{value}} is the name of a
resource parameter to compare against
(only valid in location constraints)

\item {} 
\sphinxcode{\sphinxupquote{meta}}: \sphinxcode{\sphinxupquote{value}} is the name of a
resource meta-attribute to compare
against (only valid in location
constraints)

\end{itemize}
\\
\hline
\end{longtable}\sphinxatlongtableend\end{savenotes}
\phantomsection\label{\detokenize{rules:node-attribute-expressions-special}}
In addition to custom node attributes defined by the administrator, the cluster
defines special, built-in node attributes for each node that can also be used
in rule expressions.


\begin{savenotes}\sphinxattablestart
\centering
\sphinxcapstartof{table}
\sphinxcaption{\sphinxstylestrong{Built-in Node Attributes}}\label{\detokenize{rules:id6}}
\sphinxaftercaption
\begin{tabular}[t]{|\X{1}{5}|\X{4}{5}|}
\hline
\sphinxstyletheadfamily 
Name
&\sphinxstyletheadfamily 
Value
\\
\hline
\#uname
&
{\hyperref[\detokenize{nodes:node-name}]{\sphinxcrossref{\DUrole{std,std-ref}{Node name}}}}
\\
\hline
\#id
&
Node ID
\\
\hline
\#kind
&
Node type. Possible values are \sphinxcode{\sphinxupquote{cluster}}, \sphinxcode{\sphinxupquote{remote}},
and \sphinxcode{\sphinxupquote{container}}. Kind is \sphinxcode{\sphinxupquote{remote}} for Pacemaker Remote
nodes created with the \sphinxcode{\sphinxupquote{ocf:pacemaker:remote}} resource,
and \sphinxcode{\sphinxupquote{container}} for Pacemaker Remote guest nodes and
bundle nodes
\\
\hline
\#is\_dc
&
\sphinxcode{\sphinxupquote{true}} if this node is the cluster’s Designated
Controller (DC), \sphinxcode{\sphinxupquote{false}} otherwise
\\
\hline
\#cluster-name
&
The value of the \sphinxcode{\sphinxupquote{cluster-name}} cluster property, if set
\\
\hline
\#site-name
&
The value of the \sphinxcode{\sphinxupquote{site-name}} node attribute, if set,
otherwise identical to \sphinxcode{\sphinxupquote{\#cluster-name}}
\\
\hline
\#role
&
The role the relevant promotable clone resource has on
this node. Valid only within a rule for a location
constraint for a promotable clone resource.
\\
\hline
\end{tabular}
\par
\sphinxattableend\end{savenotes}

\index{rule!date/time expression}\index{XML element!date\_expression}\index{date\_expression!XML element}\ignorespaces 

\subsection{Date/Time Expressions}
\label{\detokenize{rules:date-time-expressions}}\label{\detokenize{rules:index-14}}
Date/time expressions are rule conditions based (as the name suggests) on the
current date and time.

A \sphinxcode{\sphinxupquote{date\_expression}} element may optionally contain a \sphinxcode{\sphinxupquote{date\_spec}} or
\sphinxcode{\sphinxupquote{duration}} element depending on the context.


\begin{savenotes}\sphinxattablestart
\centering
\sphinxcapstartof{table}
\sphinxcaption{\sphinxstylestrong{Attributes of a date\_expression Element}}\label{\detokenize{rules:id7}}
\sphinxaftercaption
\begin{tabular}[t]{|\X{1}{5}|\X{4}{5}|}
\hline
\sphinxstyletheadfamily 
Attribute
&\sphinxstyletheadfamily 
Description
\\
\hline
id
&
\index{id!date\_expression}\index{date\_expression!id}\ignorespaces 
A unique name for this element (required)
\\
\hline
start
&
\index{start!date\_expression}\index{date\_expression!start}\ignorespaces 
A date/time conforming to the
\sphinxhref{https://en.wikipedia.org/wiki/ISO\_8601}{ISO8601}
specification. May be used when \sphinxcode{\sphinxupquote{operation}} is
\sphinxcode{\sphinxupquote{in\_range}} (in which case at least one of \sphinxcode{\sphinxupquote{start}} or
\sphinxcode{\sphinxupquote{end}} must be specified) or \sphinxcode{\sphinxupquote{gt}} (in which case
\sphinxcode{\sphinxupquote{start}} is required).
\\
\hline
end
&
\index{end!date\_expression}\index{date\_expression!end}\ignorespaces 
A date/time conforming to the
\sphinxhref{https://en.wikipedia.org/wiki/ISO\_8601}{ISO8601}
specification. May be used when \sphinxcode{\sphinxupquote{operation}} is
\sphinxcode{\sphinxupquote{in\_range}} (in which case at least one of \sphinxcode{\sphinxupquote{start}} or
\sphinxcode{\sphinxupquote{end}} must be specified) or \sphinxcode{\sphinxupquote{lt}} (in which case
\sphinxcode{\sphinxupquote{end}} is required).
\\
\hline
operation
&
\index{operation!date\_expression}\index{date\_expression!operation}\ignorespaces 
Compares the current date/time with the start and/or end
date, depending on the context. Allowed values:
\begin{itemize}
\item {} 
\sphinxcode{\sphinxupquote{gt:}} True if the current date/time is after \sphinxcode{\sphinxupquote{start}}

\item {} 
\sphinxcode{\sphinxupquote{lt:}} True if the current date/time is before \sphinxcode{\sphinxupquote{end}}

\item {} 
\sphinxcode{\sphinxupquote{in\_range:}} True if the current date/time is after
\sphinxcode{\sphinxupquote{start}} (if specified) and before either \sphinxcode{\sphinxupquote{end}} (if
specified) or \sphinxcode{\sphinxupquote{start}} plus the value of the
\sphinxcode{\sphinxupquote{duration}} element (if one is contained in the
\sphinxcode{\sphinxupquote{date\_expression}}). If both \sphinxcode{\sphinxupquote{end}} and \sphinxcode{\sphinxupquote{duration}}
are specified, \sphinxcode{\sphinxupquote{duration}} is ignored.

\item {} 
\sphinxcode{\sphinxupquote{date\_spec:}} True if the current date/time matches
the specification given in the contained \sphinxcode{\sphinxupquote{date\_spec}}
element (described below)

\end{itemize}
\\
\hline
\end{tabular}
\par
\sphinxattableend\end{savenotes}

\begin{sphinxadmonition}{note}{Note:}
There is no \sphinxcode{\sphinxupquote{eq}}, \sphinxcode{\sphinxupquote{neq}}, \sphinxcode{\sphinxupquote{gte}}, or \sphinxcode{\sphinxupquote{lte}} operation, since
they would be valid only for a single second.
\end{sphinxadmonition}

\index{date specification}\index{XML element!date\_spec}\index{date\_spec!XML element}\ignorespaces 

\subsubsection{Date Specifications}
\label{\detokenize{rules:date-specifications}}\label{\detokenize{rules:index-19}}
A \sphinxcode{\sphinxupquote{date\_spec}} element is used to create a cron-like expression relating
to time. Each field can contain a single number or range. Any field not
supplied is ignored.


\begin{savenotes}\sphinxattablestart
\centering
\sphinxcapstartof{table}
\sphinxcaption{\sphinxstylestrong{Attributes of a date\_spec Element}}\label{\detokenize{rules:id8}}
\sphinxaftercaption
\begin{tabular}[t]{|\X{1}{4}|\X{3}{4}|}
\hline
\sphinxstyletheadfamily 
Attribute
&\sphinxstyletheadfamily 
Description
\\
\hline
id
&
\index{id!date\_spec}\index{date\_spec!id}\ignorespaces 
A unique name for this element (required)
\\
\hline
seconds
&
\index{seconds!date\_spec}\index{date\_spec!seconds}\ignorespaces 
Allowed values: 0-59
\\
\hline
minutes
&
\index{minutes!date\_spec}\index{date\_spec!minutes}\ignorespaces 
Allowed values: 0-59
\\
\hline
hours
&
\index{hours!date\_spec}\index{date\_spec!hours}\ignorespaces 
Allowed values: 0-23 (where 0 is midnight and 23 is
11 p.m.)
\\
\hline
monthdays
&
\index{monthdays!date\_spec}\index{date\_spec!monthdays}\ignorespaces 
Allowed values: 1-31 (depending on month and year)
\\
\hline
weekdays
&
\index{weekdays!date\_spec}\index{date\_spec!weekdays}\ignorespaces 
Allowed values: 1-7 (where 1 is Monday and  7 is Sunday)
\\
\hline
yeardays
&
\index{yeardays!date\_spec}\index{date\_spec!yeardays}\ignorespaces 
Allowed values: 1-366 (depending on the year)
\\
\hline
months
&
\index{months!date\_spec}\index{date\_spec!months}\ignorespaces 
Allowed values: 1-12
\\
\hline
weeks
&
\index{weeks!date\_spec}\index{date\_spec!weeks}\ignorespaces 
Allowed values: 1-53 (depending on weekyear)
\\
\hline
years
&
\index{years!date\_spec}\index{date\_spec!years}\ignorespaces 
Year according to the Gregorian calendar
\\
\hline
weekyears
&
\index{weekyears!date\_spec}\index{date\_spec!weekyears}\ignorespaces 
Year in which the week started; for example, 1 January
2005 can be specified in ISO 8601 as “2005-001 Ordinal”,
“2005-01-01 Gregorian” or “2004-W53-6 Weekly” and thus
would match \sphinxcode{\sphinxupquote{years="2005"}} or \sphinxcode{\sphinxupquote{weekyears="2004"}}
\\
\hline
moon
&
\index{moon!date\_spec}\index{date\_spec!moon}\ignorespaces 
Allowed values are 0-7 (where 0 is the new moon and 4 is
full moon). \sphinxstyleemphasis{(deprecated since 2.1.6)}
\\
\hline
\end{tabular}
\par
\sphinxattableend\end{savenotes}

For example, \sphinxcode{\sphinxupquote{monthdays="1"}} matches the first day of every month, and
\sphinxcode{\sphinxupquote{hours="09-17"}} matches the hours between 9 a.m. and 5 p.m. (inclusive).

At this time, multiple ranges (e.g. \sphinxcode{\sphinxupquote{weekdays="1,2"}} or \sphinxcode{\sphinxupquote{weekdays="1-2,5-6"}})
are not supported.

\begin{sphinxadmonition}{note}{Note:}
Pacemaker can calculate when evaluation of a \sphinxcode{\sphinxupquote{date\_expression}} with
an \sphinxcode{\sphinxupquote{operation}} of \sphinxcode{\sphinxupquote{gt}}, \sphinxcode{\sphinxupquote{lt}}, or \sphinxcode{\sphinxupquote{in\_range}} will next change,
and schedule a cluster re-check for that time. However, it does not
do this for \sphinxcode{\sphinxupquote{date\_spec}}.  Instead, it evaluates the \sphinxcode{\sphinxupquote{date\_spec}}
whenever a cluster re-check naturally happens via a cluster event or
the \sphinxcode{\sphinxupquote{cluster-recheck-interval}} cluster option.

For example, if you have a \sphinxcode{\sphinxupquote{date\_spec}} enabling a resource from 9
a.m. to 5 p.m., and \sphinxcode{\sphinxupquote{cluster-recheck-interval}} has been set to 5
minutes, then sometime between 9 a.m. and 9:05 a.m. the cluster would
notice that it needs to start the resource, and sometime between 5
p.m. and 5:05 p.m. it would realize that it needs to stop the
resource. The timing of the actual start and stop actions will
further depend on factors such as any other actions the cluster may
need to perform first, and the load of the machine.
\end{sphinxadmonition}

\index{duration}\index{XML element!duration}\index{duration!XML element}\ignorespaces 

\subsubsection{Durations}
\label{\detokenize{rules:durations}}\label{\detokenize{rules:index-32}}
A \sphinxcode{\sphinxupquote{duration}} is used to calculate a value for \sphinxcode{\sphinxupquote{end}} when one is not
supplied to \sphinxcode{\sphinxupquote{in\_range}} operations. It contains one or more attributes each
containing a single number. Any attribute not supplied is ignored.


\begin{savenotes}\sphinxattablestart
\centering
\sphinxcapstartof{table}
\sphinxcaption{\sphinxstylestrong{Attributes of a duration Element}}\label{\detokenize{rules:id9}}
\sphinxaftercaption
\begin{tabular}[t]{|\X{1}{4}|\X{3}{4}|}
\hline
\sphinxstyletheadfamily 
Attribute
&\sphinxstyletheadfamily 
Description
\\
\hline
id
&
\index{id!duration}\index{duration!id}\ignorespaces 
A unique name for this element (required)
\\
\hline
seconds
&
\index{seconds!duration}\index{duration!seconds}\ignorespaces 
This many seconds will be added to the total duration
\\
\hline
minutes
&
\index{minutes!duration}\index{duration!minutes}\ignorespaces 
This many minutes will be added to the total duration
\\
\hline
hours
&
\index{hours!duration}\index{duration!hours}\ignorespaces 
This many hours will be added to the total duration
\\
\hline
days
&
\index{days!duration}\index{duration!days}\ignorespaces 
This many days will be added to the total duration
\\
\hline
weeks
&
\index{weeks!duration}\index{duration!weeks}\ignorespaces 
This many weeks will be added to the total duration
\\
\hline
months
&
\index{months!duration}\index{duration!months}\ignorespaces 
This many months will be added to the total duration
\\
\hline
years
&
\index{years!duration}\index{duration!years}\ignorespaces 
This many years will be added to the total duration
\\
\hline
\end{tabular}
\par
\sphinxattableend\end{savenotes}


\subsubsection{Example Time-Based Expressions}
\label{\detokenize{rules:example-time-based-expressions}}
A small sample of how time-based expressions can be used:

\begin{sphinxShadowBox}
\sphinxstyletopictitle{True if now is any time in the year 2005}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}rule} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}rule1\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}INFINITY\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}date\PYGZus{}expression} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}date\PYGZus{}expr1\PYGZdq{}} \PYG{n+na}{start=}\PYG{l+s}{\PYGZdq{}2005\PYGZhy{}001\PYGZdq{}} \PYG{n+na}{operation=}\PYG{l+s}{\PYGZdq{}in\PYGZus{}range\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}duration} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}duration1\PYGZdq{}} \PYG{n+na}{years=}\PYG{l+s}{\PYGZdq{}1\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}/date\PYGZus{}expression\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/rule\PYGZgt{}}
\end{sphinxVerbatim}

or equivalently:

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}rule} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}rule2\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}INFINITY\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}date\PYGZus{}expression} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}date\PYGZus{}expr2\PYGZdq{}} \PYG{n+na}{operation=}\PYG{l+s}{\PYGZdq{}date\PYGZus{}spec\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}date\PYGZus{}spec} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}date\PYGZus{}spec2\PYGZdq{}} \PYG{n+na}{years=}\PYG{l+s}{\PYGZdq{}2005\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}/date\PYGZus{}expression\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/rule\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

\begin{sphinxShadowBox}
\sphinxstyletopictitle{9 a.m. to 5 p.m. Monday through Friday}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}rule} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}rule3\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}INFINITY\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}date\PYGZus{}expression} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}date\PYGZus{}expr3\PYGZdq{}} \PYG{n+na}{operation=}\PYG{l+s}{\PYGZdq{}date\PYGZus{}spec\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}date\PYGZus{}spec} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}date\PYGZus{}spec3\PYGZdq{}} \PYG{n+na}{hours=}\PYG{l+s}{\PYGZdq{}9\PYGZhy{}16\PYGZdq{}} \PYG{n+na}{weekdays=}\PYG{l+s}{\PYGZdq{}1\PYGZhy{}5\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}/date\PYGZus{}expression\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/rule\PYGZgt{}}
\end{sphinxVerbatim}

Note that the \sphinxcode{\sphinxupquote{16}} matches all the way through \sphinxcode{\sphinxupquote{16:59:59}}, because the
numeric value of the hour still matches.
\end{sphinxShadowBox}

\begin{sphinxShadowBox}
\sphinxstyletopictitle{9 a.m. to 6 p.m. Monday through Friday or anytime Saturday}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}rule} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}rule4\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}INFINITY\PYGZdq{}} \PYG{n+na}{boolean\PYGZhy{}op=}\PYG{l+s}{\PYGZdq{}or\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}date\PYGZus{}expression} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}date\PYGZus{}expr4\PYGZhy{}1\PYGZdq{}} \PYG{n+na}{operation=}\PYG{l+s}{\PYGZdq{}date\PYGZus{}spec\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}date\PYGZus{}spec} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}date\PYGZus{}spec4\PYGZhy{}1\PYGZdq{}} \PYG{n+na}{hours=}\PYG{l+s}{\PYGZdq{}9\PYGZhy{}16\PYGZdq{}} \PYG{n+na}{weekdays=}\PYG{l+s}{\PYGZdq{}1\PYGZhy{}5\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}/date\PYGZus{}expression\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}date\PYGZus{}expression} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}date\PYGZus{}expr4\PYGZhy{}2\PYGZdq{}} \PYG{n+na}{operation=}\PYG{l+s}{\PYGZdq{}date\PYGZus{}spec\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}date\PYGZus{}spec} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}date\PYGZus{}spec4\PYGZhy{}2\PYGZdq{}} \PYG{n+na}{weekdays=}\PYG{l+s}{\PYGZdq{}6\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}/date\PYGZus{}expression\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/rule\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

\begin{sphinxShadowBox}
\sphinxstyletopictitle{9 a.m. to 5 p.m. or 9 p.m. to 12 a.m. Monday through Friday}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}rule} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}rule5\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}INFINITY\PYGZdq{}} \PYG{n+na}{boolean\PYGZhy{}op=}\PYG{l+s}{\PYGZdq{}and\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}rule} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}rule5\PYGZhy{}nested1\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}INFINITY\PYGZdq{}} \PYG{n+na}{boolean\PYGZhy{}op=}\PYG{l+s}{\PYGZdq{}or\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}date\PYGZus{}expression} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}date\PYGZus{}expr5\PYGZhy{}1\PYGZdq{}} \PYG{n+na}{operation=}\PYG{l+s}{\PYGZdq{}date\PYGZus{}spec\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
     \PYG{n+nt}{\PYGZlt{}date\PYGZus{}spec} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}date\PYGZus{}spec5\PYGZhy{}1\PYGZdq{}} \PYG{n+na}{hours=}\PYG{l+s}{\PYGZdq{}9\PYGZhy{}16\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/date\PYGZus{}expression\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}date\PYGZus{}expression} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}date\PYGZus{}expr5\PYGZhy{}2\PYGZdq{}} \PYG{n+na}{operation=}\PYG{l+s}{\PYGZdq{}date\PYGZus{}spec\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
     \PYG{n+nt}{\PYGZlt{}date\PYGZus{}spec} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}date\PYGZus{}spec5\PYGZhy{}2\PYGZdq{}} \PYG{n+na}{hours=}\PYG{l+s}{\PYGZdq{}21\PYGZhy{}23\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/date\PYGZus{}expression\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}/rule\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}date\PYGZus{}expression} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}date\PYGZus{}expr5\PYGZhy{}3\PYGZdq{}} \PYG{n+na}{operation=}\PYG{l+s}{\PYGZdq{}date\PYGZus{}spec\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}date\PYGZus{}spec} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}date\PYGZus{}spec5\PYGZhy{}3\PYGZdq{}} \PYG{n+na}{weekdays=}\PYG{l+s}{\PYGZdq{}1\PYGZhy{}5\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}/date\PYGZus{}expression\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/rule\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Mondays in March 2005}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}rule} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}rule6\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}INFINITY\PYGZdq{}} \PYG{n+na}{boolean\PYGZhy{}op=}\PYG{l+s}{\PYGZdq{}and\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}date\PYGZus{}expression} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}date\PYGZus{}expr6\PYGZhy{}1\PYGZdq{}} \PYG{n+na}{operation=}\PYG{l+s}{\PYGZdq{}date\PYGZus{}spec\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}date\PYGZus{}spec} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}date\PYGZus{}spec6\PYGZdq{}} \PYG{n+na}{weekdays=}\PYG{l+s}{\PYGZdq{}1\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}/date\PYGZus{}expression\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}date\PYGZus{}expression} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}date\PYGZus{}expr6\PYGZhy{}2\PYGZdq{}} \PYG{n+na}{operation=}\PYG{l+s}{\PYGZdq{}in\PYGZus{}range\PYGZdq{}}
     \PYG{n+na}{start=}\PYG{l+s}{\PYGZdq{}2005\PYGZhy{}03\PYGZhy{}01\PYGZdq{}} \PYG{n+na}{end=}\PYG{l+s}{\PYGZdq{}2005\PYGZhy{}04\PYGZhy{}01\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/rule\PYGZgt{}}
\end{sphinxVerbatim}

\begin{sphinxadmonition}{note}{Note:}
Because no time is specified with the above dates, 00:00:00 is
implied. This means that the range includes all of 2005-03-01 but
none of 2005-04-01. You may wish to write \sphinxcode{\sphinxupquote{end}} as
\sphinxcode{\sphinxupquote{"2005-03-31T23:59:59"}} to avoid confusion.
\end{sphinxadmonition}
\end{sphinxShadowBox}

\index{rule!resource expression}\index{resource!rule expression}\index{XML element!rsc\_expression}\index{rsc\_expression!XML element}\ignorespaces 

\subsection{Resource Expressions}
\label{\detokenize{rules:resource-expressions}}\label{\detokenize{rules:index-41}}
An \sphinxcode{\sphinxupquote{rsc\_expression}} \sphinxstyleemphasis{(since 2.0.5)} is a rule condition based on a resource
agent’s properties. This rule is only valid within an \sphinxcode{\sphinxupquote{rsc\_defaults}} or
\sphinxcode{\sphinxupquote{op\_defaults}} context. None of the matching attributes of \sphinxcode{\sphinxupquote{class}},
\sphinxcode{\sphinxupquote{provider}}, and \sphinxcode{\sphinxupquote{type}} are required. If one is omitted, all values of that
attribute will match.  For instance, omitting \sphinxcode{\sphinxupquote{type}} means every type will
match.


\begin{savenotes}\sphinxattablestart
\centering
\sphinxcapstartof{table}
\sphinxcaption{\sphinxstylestrong{Attributes of a rsc\_expression Element}}\label{\detokenize{rules:id10}}
\sphinxaftercaption
\begin{tabular}[t]{|\X{1}{4}|\X{3}{4}|}
\hline
\sphinxstyletheadfamily 
Attribute
&\sphinxstyletheadfamily 
Description
\\
\hline
id
&
\index{id!rsc\_expression}\index{rsc\_expression!id}\ignorespaces 
A unique name for this element (required)
\\
\hline
class
&
\index{class!rsc\_expression}\index{rsc\_expression!class}\ignorespaces 
The standard name to be matched against resource agents
\\
\hline
provider
&
\index{provider!rsc\_expression}\index{rsc\_expression!provider}\ignorespaces 
If given, the vendor to be matched against resource
agents (only relevant when \sphinxcode{\sphinxupquote{class}} is \sphinxcode{\sphinxupquote{ocf}})
\\
\hline
type
&
\index{type!rsc\_expression}\index{rsc\_expression!type}\ignorespaces 
The name of the resource agent to be matched
\\
\hline
\end{tabular}
\par
\sphinxattableend\end{savenotes}


\subsubsection{Example Resource-Based Expressions}
\label{\detokenize{rules:example-resource-based-expressions}}
A small sample of how resource-based expressions can be used:

\begin{sphinxShadowBox}
\sphinxstyletopictitle{True for all \sphinxstyleliteralintitle{\sphinxupquote{ocf:heartbeat:IPaddr2}} resources}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}rule} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}rule1\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}INFINITY\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}expression} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}rule\PYGZus{}expr1\PYGZdq{}} \PYG{n+na}{class=}\PYG{l+s}{\PYGZdq{}ocf\PYGZdq{}} \PYG{n+na}{provider=}\PYG{l+s}{\PYGZdq{}heartbeat\PYGZdq{}} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}IPaddr2\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/rule\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Provider doesn’t apply to non-OCF resources}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}rule} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}rule2\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}INFINITY\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}expression} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}rule\PYGZus{}expr2\PYGZdq{}} \PYG{n+na}{class=}\PYG{l+s}{\PYGZdq{}stonith\PYGZdq{}} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}fence\PYGZus{}xvm\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/rule\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

\index{rule!operation expression}\index{operation!rule expression}\index{XML element!op\_expression}\index{op\_expression!XML element}\ignorespaces 

\subsection{Operation Expressions}
\label{\detokenize{rules:operation-expressions}}\label{\detokenize{rules:index-46}}
An \sphinxcode{\sphinxupquote{op\_expression}} \sphinxstyleemphasis{(since 2.0.5)} is a rule condition based on an action of
some resource agent. This rule is only valid within an \sphinxcode{\sphinxupquote{op\_defaults}} context.


\begin{savenotes}\sphinxattablestart
\centering
\sphinxcapstartof{table}
\sphinxcaption{\sphinxstylestrong{Attributes of an op\_expression Element}}\label{\detokenize{rules:id11}}
\sphinxaftercaption
\begin{tabular}[t]{|\X{1}{4}|\X{3}{4}|}
\hline
\sphinxstyletheadfamily 
Attribute
&\sphinxstyletheadfamily 
Description
\\
\hline
id
&
\index{id!op\_expression}\index{op\_expression!id}\ignorespaces 
A unique name for this element (required)
\\
\hline
name
&
\index{name!op\_expression}\index{op\_expression!name}\ignorespaces 
The action name to match against. This can be any action
supported by the resource agent; common values include
\sphinxcode{\sphinxupquote{monitor}}, \sphinxcode{\sphinxupquote{start}}, and \sphinxcode{\sphinxupquote{stop}} (required).
\\
\hline
interval
&
\index{interval!op\_expression}\index{op\_expression!interval}\ignorespaces 
The interval of the action to match against. If not given,
only the name attribute will be used to match.
\\
\hline
\end{tabular}
\par
\sphinxattableend\end{savenotes}


\subsubsection{Example Operation-Based Expressions}
\label{\detokenize{rules:example-operation-based-expressions}}
A small sample of how operation-based expressions can be used:

\begin{sphinxShadowBox}
\sphinxstyletopictitle{True for all monitor actions}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}rule} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}rule1\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}INFINITY\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}op\PYGZus{}expression} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}rule\PYGZus{}expr1\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}monitor\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/rule\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

\begin{sphinxShadowBox}
\sphinxstyletopictitle{True for all monitor actions with a 10 second interval}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}rule} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}rule2\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}INFINITY\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}op\PYGZus{}expression} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}rule\PYGZus{}expr2\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}monitor\PYGZdq{}} \PYG{n+na}{interval=}\PYG{l+s}{\PYGZdq{}10s\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/rule\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

\index{location constraint!rule}\index{rule!location constraint}\ignorespaces 

\subsection{Using Rules to Determine Resource Location}
\label{\detokenize{rules:using-rules-to-determine-resource-location}}\label{\detokenize{rules:index-50}}
A location constraint may contain one or more top-level rules. The cluster will
act as if there is a separate location constraint for each rule that evaluates
as true.

Consider the following simple location constraint:

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Prevent resource \sphinxstyleliteralintitle{\sphinxupquote{webserver}} from running on node \sphinxstyleliteralintitle{\sphinxupquote{node3}}}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}location} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}ban\PYGZhy{}apache\PYGZhy{}on\PYGZhy{}node3\PYGZdq{}} \PYG{n+na}{rsc=}\PYG{l+s}{\PYGZdq{}webserver\PYGZdq{}}
              \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}\PYGZhy{}INFINITY\PYGZdq{}} \PYG{n+na}{node=}\PYG{l+s}{\PYGZdq{}node3\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

The same constraint can be more verbosely written using a rule:

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Prevent resource \sphinxstyleliteralintitle{\sphinxupquote{webserver}} from running on node \sphinxstyleliteralintitle{\sphinxupquote{node3}} using a rule}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}location} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}ban\PYGZhy{}apache\PYGZhy{}on\PYGZhy{}node3\PYGZdq{}} \PYG{n+na}{rsc=}\PYG{l+s}{\PYGZdq{}webserver\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}rule} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}ban\PYGZhy{}apache\PYGZhy{}rule\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}\PYGZhy{}INFINITY\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}expression} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}ban\PYGZhy{}apache\PYGZhy{}expr\PYGZdq{}} \PYG{n+na}{attribute=}\PYG{l+s}{\PYGZdq{}\PYGZsh{}uname\PYGZdq{}}
        \PYG{n+na}{operation=}\PYG{l+s}{\PYGZdq{}eq\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}node3\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/rule\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/rsc\PYGZus{}location\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

The advantage of using the expanded form is that one could add more expressions
(for example, limiting the constraint to certain days of the week), or activate
the constraint by some node attribute other than node name.


\subsubsection{Location Rules Based on Other Node Properties}
\label{\detokenize{rules:location-rules-based-on-other-node-properties}}
The expanded form allows us to match on node properties other than its name.
If we rated each machine’s CPU power such that the cluster had the following
nodes section:

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Sample node section with node attributes}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}nodes}\PYG{n+nt}{\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}node} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}uuid1\PYGZdq{}} \PYG{n+na}{uname=}\PYG{l+s}{\PYGZdq{}c001n01\PYGZdq{}} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}normal\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}instance\PYGZus{}attributes} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}uuid1\PYGZhy{}custom\PYGZus{}attrs\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}uuid1\PYGZhy{}cpu\PYGZus{}mips\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}cpu\PYGZus{}mips\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}1234\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}/instance\PYGZus{}attributes\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}/node\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}node} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}uuid2\PYGZdq{}} \PYG{n+na}{uname=}\PYG{l+s}{\PYGZdq{}c001n02\PYGZdq{}} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}normal\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}instance\PYGZus{}attributes} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}uuid2\PYGZhy{}custom\PYGZus{}attrs\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}uuid2\PYGZhy{}cpu\PYGZus{}mips\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}cpu\PYGZus{}mips\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}5678\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}/instance\PYGZus{}attributes\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}/node\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/nodes\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

then we could prevent resources from running on underpowered machines with this
rule:

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Rule using a node attribute (to be used inside a location constraint)}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}rule} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}need\PYGZhy{}more\PYGZhy{}power\PYGZhy{}rule\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}\PYGZhy{}INFINITY\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}expression} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}need\PYGZhy{}more\PYGZhy{}power\PYGZhy{}expr\PYGZdq{}} \PYG{n+na}{attribute=}\PYG{l+s}{\PYGZdq{}cpu\PYGZus{}mips\PYGZdq{}}
               \PYG{n+na}{operation=}\PYG{l+s}{\PYGZdq{}lt\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}3000\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/rule\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}


\subsubsection{Using \sphinxstyleliteralintitle{\sphinxupquote{score-attribute}} Instead of \sphinxstyleliteralintitle{\sphinxupquote{score}}}
\label{\detokenize{rules:using-score-attribute-instead-of-score}}
When using \sphinxcode{\sphinxupquote{score-attribute}} instead of \sphinxcode{\sphinxupquote{score}}, each node matched by the
rule has its score adjusted differently, according to its value for the named
node attribute. Thus, in the previous example, if a rule inside a location
constraint for a resource used \sphinxcode{\sphinxupquote{score-attribute="cpu\_mips"}}, \sphinxcode{\sphinxupquote{c001n01}}
would have its preference to run the resource increased by \sphinxcode{\sphinxupquote{1234}} whereas
\sphinxcode{\sphinxupquote{c001n02}} would have its preference increased by \sphinxcode{\sphinxupquote{5678}}.


\subsubsection{Specifying location scores using pattern submatches}
\label{\detokenize{rules:specifying-location-scores-using-pattern-submatches}}\label{\detokenize{rules:s-rsc-pattern-rules}}
Location constraints may use \sphinxcode{\sphinxupquote{rsc-pattern}} to apply the constraint to all
resources whose IDs match the given pattern (see {\hyperref[\detokenize{constraints:s-rsc-pattern}]{\sphinxcrossref{\DUrole{std,std-ref}{Specifying locations using pattern matching}}}}). The
pattern may contain up to 9 submatches in parentheses, whose values may be used
as \sphinxcode{\sphinxupquote{\%1}} through \sphinxcode{\sphinxupquote{\%9}} in a rule’s \sphinxcode{\sphinxupquote{score-attribute}} or a rule expression’s
\sphinxcode{\sphinxupquote{attribute}}.

As an example, the following configuration (only relevant parts are shown)
gives the resources \sphinxstylestrong{server-httpd} and \sphinxstylestrong{ip-httpd} a preference of 100 on
\sphinxstylestrong{node1} and 50 on \sphinxstylestrong{node2}, and \sphinxstylestrong{ip-gateway} a preference of -100 on
\sphinxstylestrong{node1} and 200 on \sphinxstylestrong{node2}.

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Location constraint using submatches}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}nodes}\PYG{n+nt}{\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}node} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}1\PYGZdq{}} \PYG{n+na}{uname=}\PYG{l+s}{\PYGZdq{}node1\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}instance\PYGZus{}attributes} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}node1\PYGZhy{}attrs\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
         \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}node1\PYGZhy{}prefer\PYGZhy{}httpd\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}prefer\PYGZhy{}httpd\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}100\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
         \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}node1\PYGZhy{}prefer\PYGZhy{}gateway\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}prefer\PYGZhy{}gateway\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}\PYGZhy{}100\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}/instance\PYGZus{}attributes\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}/node\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}node} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}2\PYGZdq{}} \PYG{n+na}{uname=}\PYG{l+s}{\PYGZdq{}node2\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}instance\PYGZus{}attributes} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}node2\PYGZhy{}attrs\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
         \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}node2\PYGZhy{}prefer\PYGZhy{}httpd\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}prefer\PYGZhy{}httpd\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}50\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
         \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}node2\PYGZhy{}prefer\PYGZhy{}gateway\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}prefer\PYGZhy{}gateway\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}200\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}/instance\PYGZus{}attributes\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}/node\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/nodes\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}resources}\PYG{n+nt}{\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}primitive} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}server\PYGZhy{}httpd\PYGZdq{}} \PYG{n+na}{class=}\PYG{l+s}{\PYGZdq{}ocf\PYGZdq{}} \PYG{n+na}{provider=}\PYG{l+s}{\PYGZdq{}heartbeat\PYGZdq{}} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}apache\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}primitive} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}ip\PYGZhy{}httpd\PYGZdq{}} \PYG{n+na}{class=}\PYG{l+s}{\PYGZdq{}ocf\PYGZdq{}} \PYG{n+na}{provider=}\PYG{l+s}{\PYGZdq{}heartbeat\PYGZdq{}} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}IPaddr2\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}primitive} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}ip\PYGZhy{}gateway\PYGZdq{}} \PYG{n+na}{class=}\PYG{l+s}{\PYGZdq{}ocf\PYGZdq{}} \PYG{n+na}{provider=}\PYG{l+s}{\PYGZdq{}heartbeat\PYGZdq{}} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}IPaddr2\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/resources\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}constraints}\PYG{n+nt}{\PYGZgt{}}
   \PYG{c}{\PYGZlt{}!\PYGZhy{}\PYGZhy{}}\PYG{c}{ The following constraint says that for any resource whose name}
\PYG{c}{        starts with \PYGZdq{}server}\PYG{c}{\PYGZhy{}}\PYG{c}{\PYGZdq{} or \PYGZdq{}ip}\PYG{c}{\PYGZhy{}}\PYG{c}{\PYGZdq{}, that resource\PYGZsq{}s preference for a}
\PYG{c}{        node is the value of the node attribute named \PYGZdq{}prefer}\PYG{c}{\PYGZhy{}}\PYG{c}{\PYGZdq{} followed}
\PYG{c}{        by the part of the resource name after \PYGZdq{}server}\PYG{c}{\PYGZhy{}}\PYG{c}{\PYGZdq{} or \PYGZdq{}ip}\PYG{c}{\PYGZhy{}}\PYG{c}{\PYGZdq{},}
\PYG{c}{        wherever such a node attribute is defined.}
\PYG{c}{     }\PYG{c}{\PYGZhy{}\PYGZhy{}\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}location} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}location1\PYGZdq{}} \PYG{n+na}{rsc\PYGZhy{}pattern=}\PYG{l+s}{\PYGZdq{}(server\textbar{}ip)\PYGZhy{}(.*)\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}rule} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}location1\PYGZhy{}rule1\PYGZdq{}} \PYG{n+na}{score\PYGZhy{}attribute=}\PYG{l+s}{\PYGZdq{}prefer\PYGZhy{}\PYGZpc{}2\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
         \PYG{n+nt}{\PYGZlt{}expression} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}location1\PYGZhy{}rule1\PYGZhy{}expression1\PYGZdq{}} \PYG{n+na}{attribute=}\PYG{l+s}{\PYGZdq{}prefer\PYGZhy{}\PYGZpc{}2\PYGZdq{}} \PYG{n+na}{operation=}\PYG{l+s}{\PYGZdq{}defined\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}/rule\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}/rsc\PYGZus{}location\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/constraints\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

\index{cluster option!rule}\index{rule!cluster option}\index{instance attribute!rule}\index{rule!instance attribute}\index{meta-attribute!rule}\index{rule!meta-attribute}\index{resource defaults!rule}\index{rule!resource defaults}\index{operation defaults!rule}\index{rule!operation defaults}\index{node attribute!rule}\index{rule!node attribute}\ignorespaces 

\subsection{Using Rules to Define Options}
\label{\detokenize{rules:using-rules-to-define-options}}\label{\detokenize{rules:index-51}}
Rules may be used to control a variety of options:
\begin{itemize}
\item {} 
{\hyperref[\detokenize{cluster-options:cluster-options}]{\sphinxcrossref{\DUrole{std,std-ref}{Cluster options}}}} (\sphinxcode{\sphinxupquote{cluster\_property\_set}} elements)

\item {} 
{\hyperref[\detokenize{nodes:node-attributes}]{\sphinxcrossref{\DUrole{std,std-ref}{Node attributes}}}} (\sphinxcode{\sphinxupquote{instance\_attributes}} or
\sphinxcode{\sphinxupquote{utilization}} elements inside a \sphinxcode{\sphinxupquote{node}} element)

\item {} 
{\hyperref[\detokenize{resources:resource-options}]{\sphinxcrossref{\DUrole{std,std-ref}{Resource options}}}} (\sphinxcode{\sphinxupquote{utilization}},
\sphinxcode{\sphinxupquote{meta\_attributes}}, or \sphinxcode{\sphinxupquote{instance\_attributes}} elements inside a resource
definition element or \sphinxcode{\sphinxupquote{op}} , \sphinxcode{\sphinxupquote{rsc\_defaults}}, \sphinxcode{\sphinxupquote{op\_defaults}}, or
\sphinxcode{\sphinxupquote{template}} element)

\item {} 
{\hyperref[\detokenize{operations:operation-properties}]{\sphinxcrossref{\DUrole{std,std-ref}{Operation properties}}}} (\sphinxcode{\sphinxupquote{meta\_attributes}}
elements inside an \sphinxcode{\sphinxupquote{op}} or \sphinxcode{\sphinxupquote{op\_defaults}} element)

\end{itemize}

\begin{sphinxadmonition}{note}{Note:}
Attribute-based expressions for meta-attributes can only be used within
\sphinxcode{\sphinxupquote{operations}} and \sphinxcode{\sphinxupquote{op\_defaults}}.  They will not work with resource
configuration or \sphinxcode{\sphinxupquote{rsc\_defaults}}.  Additionally, attribute-based
expressions cannot be used with cluster options.
\end{sphinxadmonition}


\subsubsection{Using Rules to Control Resource Options}
\label{\detokenize{rules:using-rules-to-control-resource-options}}
Often some cluster nodes will be different from their peers. Sometimes,
these differences \textendash{} e.g. the location of a binary or the names of network
interfaces \textendash{} require resources to be configured differently depending
on the machine they’re hosted on.

By defining multiple \sphinxcode{\sphinxupquote{instance\_attributes}} objects for the resource and
adding a rule to each, we can easily handle these special cases.

In the example below, \sphinxcode{\sphinxupquote{mySpecialRsc}} will use eth1 and port 9999 when run on
\sphinxcode{\sphinxupquote{node1}}, eth2 and port 8888 on \sphinxcode{\sphinxupquote{node2}} and default to eth0 and port 9999
for all other nodes.

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Defining different resource options based on the node name}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}primitive} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}mySpecialRsc\PYGZdq{}} \PYG{n+na}{class=}\PYG{l+s}{\PYGZdq{}ocf\PYGZdq{}} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}Special\PYGZdq{}} \PYG{n+na}{provider=}\PYG{l+s}{\PYGZdq{}me\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}instance\PYGZus{}attributes} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}special\PYGZhy{}node1\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}3\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}rule} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}node1\PYGZhy{}special\PYGZhy{}case\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}INFINITY\PYGZdq{}} \PYG{n+nt}{\PYGZgt{}}
     \PYG{n+nt}{\PYGZlt{}expression} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}node1\PYGZhy{}special\PYGZhy{}case\PYGZhy{}expr\PYGZdq{}} \PYG{n+na}{attribute=}\PYG{l+s}{\PYGZdq{}\PYGZsh{}uname\PYGZdq{}}
       \PYG{n+na}{operation=}\PYG{l+s}{\PYGZdq{}eq\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}node1\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/rule\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}node1\PYGZhy{}interface\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}interface\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}eth1\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}/instance\PYGZus{}attributes\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}instance\PYGZus{}attributes} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}special\PYGZhy{}node2\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}2\PYGZdq{}} \PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}rule} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}node2\PYGZhy{}special\PYGZhy{}case\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}INFINITY\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
     \PYG{n+nt}{\PYGZlt{}expression} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}node2\PYGZhy{}special\PYGZhy{}case\PYGZhy{}expr\PYGZdq{}} \PYG{n+na}{attribute=}\PYG{l+s}{\PYGZdq{}\PYGZsh{}uname\PYGZdq{}}
       \PYG{n+na}{operation=}\PYG{l+s}{\PYGZdq{}eq\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}node2\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/rule\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}node2\PYGZhy{}interface\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}interface\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}eth2\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}node2\PYGZhy{}port\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}port\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}8888\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}/instance\PYGZus{}attributes\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}instance\PYGZus{}attributes} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}defaults\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}1\PYGZdq{}} \PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}default\PYGZhy{}interface\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}interface\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}eth0\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}default\PYGZhy{}port\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}port\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}9999\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}/instance\PYGZus{}attributes\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/primitive\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

The order in which \sphinxcode{\sphinxupquote{instance\_attributes}} objects are evaluated is determined
by their score (highest to lowest). If not supplied, the score defaults to
zero. Objects with an equal score are processed in their listed order. If the
\sphinxcode{\sphinxupquote{instance\_attributes}} object has no rule, or a \sphinxcode{\sphinxupquote{rule}} that evaluates to
\sphinxcode{\sphinxupquote{true}}, then for any parameter the resource does not yet have a value for,
the resource will use the parameter values defined by the \sphinxcode{\sphinxupquote{instance\_attributes}}.

For example, given the configuration above, if the resource is placed on
\sphinxcode{\sphinxupquote{node1}}:
\begin{itemize}
\item {} 
\sphinxcode{\sphinxupquote{special-node1}} has the highest score (3) and so is evaluated first; its
rule evaluates to \sphinxcode{\sphinxupquote{true}}, so \sphinxcode{\sphinxupquote{interface}} is set to \sphinxcode{\sphinxupquote{eth1}}.

\item {} 
\sphinxcode{\sphinxupquote{special-node2}} is evaluated next with score 2, but its rule evaluates to
\sphinxcode{\sphinxupquote{false}}, so it is ignored.

\item {} 
\sphinxcode{\sphinxupquote{defaults}} is evaluated last with score 1, and has no rule, so its values
are examined; \sphinxcode{\sphinxupquote{interface}} is already defined, so the value here is not
used, but \sphinxcode{\sphinxupquote{port}} is not yet defined, so \sphinxcode{\sphinxupquote{port}} is set to \sphinxcode{\sphinxupquote{9999}}.

\end{itemize}


\subsubsection{Using Rules to Control Resource Defaults}
\label{\detokenize{rules:using-rules-to-control-resource-defaults}}
Rules can be used for resource and operation defaults. The following example
illustrates how to set a different \sphinxcode{\sphinxupquote{resource-stickiness}} value during and
outside work hours. This allows resources to automatically move back to their
most preferred hosts, but at a time that (in theory) does not interfere with
business activities.

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Change \sphinxstyleliteralintitle{\sphinxupquote{resource-stickiness}} during working hours}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}defaults}\PYG{n+nt}{\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}meta\PYGZus{}attributes} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}core\PYGZhy{}hours\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}2\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}rule} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}core\PYGZhy{}hour\PYGZhy{}rule\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}date\PYGZus{}expression} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}nine\PYGZhy{}to\PYGZhy{}five\PYGZhy{}Mon\PYGZhy{}to\PYGZhy{}Fri\PYGZdq{}} \PYG{n+na}{operation=}\PYG{l+s}{\PYGZdq{}date\PYGZus{}spec\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
          \PYG{n+nt}{\PYGZlt{}date\PYGZus{}spec} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}nine\PYGZhy{}to\PYGZhy{}five\PYGZhy{}Mon\PYGZhy{}to\PYGZhy{}Fri\PYGZhy{}spec\PYGZdq{}} \PYG{n+na}{hours=}\PYG{l+s}{\PYGZdq{}9\PYGZhy{}16\PYGZdq{}} \PYG{n+na}{weekdays=}\PYG{l+s}{\PYGZdq{}1\PYGZhy{}5\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}/date\PYGZus{}expression\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}/rule\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}core\PYGZhy{}stickiness\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}resource\PYGZhy{}stickiness\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}INFINITY\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}/meta\PYGZus{}attributes\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}meta\PYGZus{}attributes} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}after\PYGZhy{}hours\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}1\PYGZdq{}} \PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}after\PYGZhy{}stickiness\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}resource\PYGZhy{}stickiness\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}/meta\PYGZus{}attributes\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/rsc\PYGZus{}defaults\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

Rules may be used similarly in \sphinxcode{\sphinxupquote{instance\_attributes}} or \sphinxcode{\sphinxupquote{utilization}}
blocks.

Any single block may directly contain only a single rule, but that rule may
itself contain any number of rules.

\sphinxcode{\sphinxupquote{rsc\_expression}} and \sphinxcode{\sphinxupquote{op\_expression}} blocks may additionally be used to
set defaults on either a single resource or across an entire class of resources
with a single rule. \sphinxcode{\sphinxupquote{rsc\_expression}} may be used to select resource agents
within both \sphinxcode{\sphinxupquote{rsc\_defaults}} and \sphinxcode{\sphinxupquote{op\_defaults}}, while \sphinxcode{\sphinxupquote{op\_expression}} may
only be used within \sphinxcode{\sphinxupquote{op\_defaults}}. If multiple rules succeed for a given
resource agent, the last one specified will be the one that takes effect. As
with any other rule, boolean operations may be used to make more complicated
expressions.

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Default all IPaddr2 resources to stopped}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}defaults}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}meta\PYGZus{}attributes} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}op\PYGZhy{}target\PYGZhy{}role\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}rule} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}op\PYGZhy{}target\PYGZhy{}role\PYGZhy{}rule\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}INFINITY\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
            \PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}expression} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}op\PYGZhy{}target\PYGZhy{}role\PYGZhy{}expr\PYGZdq{}} \PYG{n+na}{class=}\PYG{l+s}{\PYGZdq{}ocf\PYGZdq{}} \PYG{n+na}{provider=}\PYG{l+s}{\PYGZdq{}heartbeat\PYGZdq{}}
              \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}IPaddr2\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}/rule\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}op\PYGZhy{}target\PYGZhy{}role\PYGZhy{}nvpair\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}target\PYGZhy{}role\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}Stopped\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/meta\PYGZus{}attributes\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/rsc\PYGZus{}defaults\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Default all monitor action timeouts to 7 seconds}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}op\PYGZus{}defaults}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}meta\PYGZus{}attributes} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}op\PYGZhy{}monitor\PYGZhy{}defaults\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}rule} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}op\PYGZhy{}monitor\PYGZhy{}default\PYGZhy{}rule\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}INFINITY\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
            \PYG{n+nt}{\PYGZlt{}op\PYGZus{}expression} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}op\PYGZhy{}monitor\PYGZhy{}default\PYGZhy{}expr\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}monitor\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}/rule\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}op\PYGZhy{}monitor\PYGZhy{}timeout\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}timeout\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}7s\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/meta\PYGZus{}attributes\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/op\PYGZus{}defaults\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Default the timeout on all 10-second-interval monitor actions on \sphinxstyleliteralintitle{\sphinxupquote{IPaddr2}} resources to 8 seconds}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}op\PYGZus{}defaults}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}meta\PYGZus{}attributes} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}op\PYGZhy{}monitor\PYGZhy{}and\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}rule} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}op\PYGZhy{}monitor\PYGZhy{}and\PYGZhy{}rule\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}INFINITY\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
            \PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}expression} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}op\PYGZhy{}monitor\PYGZhy{}and\PYGZhy{}rsc\PYGZhy{}expr\PYGZdq{}} \PYG{n+na}{class=}\PYG{l+s}{\PYGZdq{}ocf\PYGZdq{}} \PYG{n+na}{provider=}\PYG{l+s}{\PYGZdq{}heartbeat\PYGZdq{}}
              \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}IPaddr2\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
            \PYG{n+nt}{\PYGZlt{}op\PYGZus{}expression} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}op\PYGZhy{}monitor\PYGZhy{}and\PYGZhy{}op\PYGZhy{}expr\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}monitor\PYGZdq{}} \PYG{n+na}{interval=}\PYG{l+s}{\PYGZdq{}10s\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}/rule\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}op\PYGZhy{}monitor\PYGZhy{}and\PYGZhy{}timeout\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}timeout\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}8s\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/meta\PYGZus{}attributes\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/op\PYGZus{}defaults\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

\index{rule!cluster option}\index{cluster option!rule}\ignorespaces 

\subsubsection{Using Rules to Control Cluster Options}
\label{\detokenize{rules:using-rules-to-control-cluster-options}}\label{\detokenize{rules:index-52}}
Controlling cluster options is achieved in much the same manner as specifying
different resource options on different nodes.

The following example illustrates how to set \sphinxcode{\sphinxupquote{maintenance\_mode}} during a
scheduled maintenance window. This will keep the cluster running but not
monitor, start, or stop resources during this time.

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Schedule a maintenance window for 9 to 11 p.m. CDT Sept. 20, 2019}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}crm\PYGZus{}config}\PYG{n+nt}{\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}cluster\PYGZus{}property\PYGZus{}set} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}cib\PYGZhy{}bootstrap\PYGZhy{}options\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
     \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}bootstrap\PYGZhy{}stonith\PYGZhy{}enabled\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}stonith\PYGZhy{}enabled\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}1\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}/cluster\PYGZus{}property\PYGZus{}set\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}cluster\PYGZus{}property\PYGZus{}set} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}normal\PYGZhy{}set\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}10\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
     \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}normal\PYGZhy{}maintenance\PYGZhy{}mode\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}maintenance\PYGZhy{}mode\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}false\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}/cluster\PYGZus{}property\PYGZus{}set\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}cluster\PYGZus{}property\PYGZus{}set} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}maintenance\PYGZhy{}window\PYGZhy{}set\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}1000\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
     \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}maintenance\PYGZhy{}nvpair1\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}maintenance\PYGZhy{}mode\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}true\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
     \PYG{n+nt}{\PYGZlt{}rule} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}maintenance\PYGZhy{}rule1\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}INFINITY\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
       \PYG{n+nt}{\PYGZlt{}date\PYGZus{}expression} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}maintenance\PYGZhy{}date1\PYGZdq{}} \PYG{n+na}{operation=}\PYG{l+s}{\PYGZdq{}in\PYGZus{}range\PYGZdq{}}
         \PYG{n+na}{start=}\PYG{l+s}{\PYGZdq{}2019\PYGZhy{}09\PYGZhy{}20 21:00:00 \PYGZhy{}05:00\PYGZdq{}} \PYG{n+na}{end=}\PYG{l+s}{\PYGZdq{}2019\PYGZhy{}09\PYGZhy{}20 23:00:00 \PYGZhy{}05:00\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
     \PYG{n+nt}{\PYGZlt{}/rule\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}/cluster\PYGZus{}property\PYGZus{}set\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/crm\PYGZus{}config\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

\begin{sphinxadmonition}{important}{Important:}
The \sphinxcode{\sphinxupquote{cluster\_property\_set}} with an \sphinxcode{\sphinxupquote{id}} set to
“cib-bootstrap-options” will \sphinxstyleemphasis{always} have the highest priority,
regardless of any scores. Therefore, rules in another
\sphinxcode{\sphinxupquote{cluster\_property\_set}} can never take effect for any
properties listed in the bootstrap set.
\end{sphinxadmonition}


\section{Collective Resources}
\label{\detokenize{collective:collective-resources}}\label{\detokenize{collective::doc}}
Pacemaker supports several types of \sphinxstyleemphasis{collective} resources, which consist of
multiple, related resource instances.


\subsection{Groups - A Syntactic Shortcut}
\label{\detokenize{collective:groups-a-syntactic-shortcut}}\label{\detokenize{collective:group-resources}}
One of the most common elements of a cluster is a set of resources
that need to be located together, start sequentially, and stop in the
reverse order.  To simplify this configuration, we support the concept
of groups.

\begin{sphinxShadowBox}
\sphinxstyletopictitle{A group of two primitive resources}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}group} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}shortcut\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}primitive} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}Public\PYGZhy{}IP\PYGZdq{}} \PYG{n+na}{class=}\PYG{l+s}{\PYGZdq{}ocf\PYGZdq{}} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}IPaddr\PYGZdq{}} \PYG{n+na}{provider=}\PYG{l+s}{\PYGZdq{}heartbeat\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}instance\PYGZus{}attributes} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}params\PYGZhy{}public\PYGZhy{}ip\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
       \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}public\PYGZhy{}ip\PYGZhy{}addr\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}ip\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}192.0.2.2\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/instance\PYGZus{}attributes\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}/primitive\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}primitive} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}Email\PYGZdq{}} \PYG{n+na}{class=}\PYG{l+s}{\PYGZdq{}lsb\PYGZdq{}} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}exim\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/group\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

Although the example above contains only two resources, there is no
limit to the number of resources a group can contain.  The example is
also sufficient to explain the fundamental properties of a group:
\begin{itemize}
\item {} 
Resources are started in the order they appear in (\sphinxstylestrong{Public-IP} first,
then \sphinxstylestrong{Email})

\item {} 
Resources are stopped in the reverse order to which they appear in
(\sphinxstylestrong{Email} first, then \sphinxstylestrong{Public-IP})

\end{itemize}

If a resource in the group can’t run anywhere, then nothing after that
is allowed to run, too.
\begin{itemize}
\item {} 
If \sphinxstylestrong{Public-IP} can’t run anywhere, neither can \sphinxstylestrong{Email};

\item {} 
but if \sphinxstylestrong{Email} can’t run anywhere, this does not affect \sphinxstylestrong{Public-IP}
in any way

\end{itemize}

The group above is logically equivalent to writing:

\begin{sphinxShadowBox}
\sphinxstyletopictitle{How the cluster sees a group resource}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}configuration}\PYG{n+nt}{\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}resources}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}primitive} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}Public\PYGZhy{}IP\PYGZdq{}} \PYG{n+na}{class=}\PYG{l+s}{\PYGZdq{}ocf\PYGZdq{}} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}IPaddr\PYGZdq{}} \PYG{n+na}{provider=}\PYG{l+s}{\PYGZdq{}heartbeat\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
     \PYG{n+nt}{\PYGZlt{}instance\PYGZus{}attributes} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}params\PYGZhy{}public\PYGZhy{}ip\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}public\PYGZhy{}ip\PYGZhy{}addr\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}ip\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}192.0.2.2\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
     \PYG{n+nt}{\PYGZlt{}/instance\PYGZus{}attributes\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/primitive\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}primitive} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}Email\PYGZdq{}} \PYG{n+na}{class=}\PYG{l+s}{\PYGZdq{}lsb\PYGZdq{}} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}exim\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}/resources\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}constraints}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}colocation} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}xxx\PYGZdq{}} \PYG{n+na}{rsc=}\PYG{l+s}{\PYGZdq{}Email\PYGZdq{}} \PYG{n+na}{with\PYGZhy{}rsc=}\PYG{l+s}{\PYGZdq{}Public\PYGZhy{}IP\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}INFINITY\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}order} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}yyy\PYGZdq{}} \PYG{n+na}{first=}\PYG{l+s}{\PYGZdq{}Public\PYGZhy{}IP\PYGZdq{}} \PYG{n+na}{then=}\PYG{l+s}{\PYGZdq{}Email\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}/constraints\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/configuration\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

Obviously as the group grows bigger, the reduced configuration effort
can become significant.

Another (typical) example of a group is a DRBD volume, the filesystem
mount, an IP address, and an application that uses them.

\index{XML element!group}\index{group!XML element}\ignorespaces 

\subsubsection{Group Properties}
\label{\detokenize{collective:group-properties}}\label{\detokenize{collective:index-0}}

\begin{savenotes}\sphinxattablestart
\centering
\sphinxcapstartof{table}
\sphinxcaption{\sphinxstylestrong{Properties of a Group Resource}}\label{\detokenize{collective:id6}}
\sphinxaftercaption
\begin{tabular}[t]{|\X{1}{5}|\X{4}{5}|}
\hline
\sphinxstyletheadfamily 
Field
&\sphinxstyletheadfamily 
Description
\\
\hline
id
&
\index{group!property, id}\index{property!id (group)}\index{id!group property}\ignorespaces 
A unique name for the group
\\
\hline
description
&
\index{group!attribute, description}\index{attribute!description (group)}\index{description!group attribute}\ignorespaces 
An optional description of the group, for the user’s own
purposes.
E.g. \sphinxcode{\sphinxupquote{resources needed for website}}
\\
\hline
\end{tabular}
\par
\sphinxattableend\end{savenotes}


\subsubsection{Group Options}
\label{\detokenize{collective:group-options}}
Groups inherit the \sphinxcode{\sphinxupquote{priority}}, \sphinxcode{\sphinxupquote{target-role}}, and \sphinxcode{\sphinxupquote{is-managed}} properties
from primitive resources. See {\hyperref[\detokenize{resources:resource-options}]{\sphinxcrossref{\DUrole{std,std-ref}{Resource Options}}}} for information about
those properties.


\subsubsection{Group Instance Attributes}
\label{\detokenize{collective:group-instance-attributes}}
Groups have no instance attributes. However, any that are set for the group
object will be inherited by the group’s children.


\subsubsection{Group Contents}
\label{\detokenize{collective:group-contents}}
Groups may only contain a collection of cluster resources (see
{\hyperref[\detokenize{resources:primitive-resource}]{\sphinxcrossref{\DUrole{std,std-ref}{Resource Properties}}}}).  To refer to a child of a group resource, just use
the child’s \sphinxcode{\sphinxupquote{id}} instead of the group’s.


\subsubsection{Group Constraints}
\label{\detokenize{collective:group-constraints}}
Although it is possible to reference a group’s children in
constraints, it is usually preferable to reference the group itself.

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Some constraints involving groups}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}constraints}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}location} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}group\PYGZhy{}prefers\PYGZhy{}node1\PYGZdq{}} \PYG{n+na}{rsc=}\PYG{l+s}{\PYGZdq{}shortcut\PYGZdq{}} \PYG{n+na}{node=}\PYG{l+s}{\PYGZdq{}node1\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}500\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}colocation} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}webserver\PYGZhy{}with\PYGZhy{}group\PYGZdq{}} \PYG{n+na}{rsc=}\PYG{l+s}{\PYGZdq{}Webserver\PYGZdq{}} \PYG{n+na}{with\PYGZhy{}rsc=}\PYG{l+s}{\PYGZdq{}shortcut\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}order} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}start\PYGZhy{}group\PYGZhy{}then\PYGZhy{}webserver\PYGZdq{}} \PYG{n+na}{first=}\PYG{l+s}{\PYGZdq{}Webserver\PYGZdq{}} \PYG{n+na}{then=}\PYG{l+s}{\PYGZdq{}shortcut\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/constraints\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

\index{resource-stickiness!group}\index{group!resource-stickiness}\ignorespaces 

\subsubsection{Group Stickiness}
\label{\detokenize{collective:group-stickiness}}\label{\detokenize{collective:index-3}}
Stickiness, the measure of how much a resource wants to stay where it
is, is additive in groups.  Every active resource of the group will
contribute its stickiness value to the group’s total.  So if the
default \sphinxcode{\sphinxupquote{resource-stickiness}} is 100, and a group has seven members,
five of which are active, then the group as a whole will prefer its
current location with a score of 500.

\index{clone}\index{resource!clone}\ignorespaces 

\subsection{Clones - Resources That Can Have Multiple Active Instances}
\label{\detokenize{collective:clones-resources-that-can-have-multiple-active-instances}}\label{\detokenize{collective:s-resource-clone}}\label{\detokenize{collective:index-4}}
\sphinxstyleemphasis{Clone} resources are resources that can have more than one copy active at the
same time. This allows you, for example, to run a copy of a daemon on every
node. You can clone any primitive or group resource %
\begin{footnote}[1]\sphinxAtStartFootnote
Of course, the service must support running multiple instances.
%
\end{footnote}.


\subsubsection{Anonymous versus Unique Clones}
\label{\detokenize{collective:anonymous-versus-unique-clones}}
A clone resource is configured to be either \sphinxstyleemphasis{anonymous} or \sphinxstyleemphasis{globally unique}.

Anonymous clones are the simplest. These behave completely identically
everywhere they are running. Because of this, there can be only one instance of
an anonymous clone active per node.

The instances of globally unique clones are distinct entities. All instances
are launched identically, but one instance of the clone is not identical to any
other instance, whether running on the same node or a different node. As an
example, a cloned IP address can use special kernel functionality such that
each instance handles a subset of requests for the same IP address.

\index{promotable clone}\index{resource!promotable}\ignorespaces 

\subsubsection{Promotable clones}
\label{\detokenize{collective:promotable-clones}}\label{\detokenize{collective:s-resource-promotable}}\label{\detokenize{collective:index-5}}
If a clone is \sphinxstyleemphasis{promotable}, its instances can perform a special role that
Pacemaker will manage via the \sphinxcode{\sphinxupquote{promote}} and \sphinxcode{\sphinxupquote{demote}} actions of the resource
agent.

Services that support such a special role have various terms for the special
role and the default role: primary and secondary, master and replica,
controller and worker, etc. Pacemaker uses the terms \sphinxstyleemphasis{promoted} and
\sphinxstyleemphasis{unpromoted} to be agnostic to what the service calls them or what they do.

All that Pacemaker cares about is that an instance comes up in the unpromoted role
when started, and the resource agent supports the \sphinxcode{\sphinxupquote{promote}} and \sphinxcode{\sphinxupquote{demote}} actions
to manage entering and exiting the promoted role.

\index{XML element!clone}\index{clone!XML element}\ignorespaces 

\subsubsection{Clone Properties}
\label{\detokenize{collective:clone-properties}}\label{\detokenize{collective:index-6}}

\begin{savenotes}\sphinxattablestart
\centering
\sphinxcapstartof{table}
\sphinxcaption{\sphinxstylestrong{Properties of a Clone Resource}}\label{\detokenize{collective:id7}}
\sphinxaftercaption
\begin{tabular}[t]{|\X{1}{5}|\X{4}{5}|}
\hline
\sphinxstyletheadfamily 
Field
&\sphinxstyletheadfamily 
Description
\\
\hline
id
&
\index{clone!property, id}\index{property!id (clone)}\index{id!clone property}\ignorespaces 
A unique name for the clone
\\
\hline
description
&
\index{clone!attribute, description}\index{attribute!description (clone)}\index{description!clone attribute}\ignorespaces 
An optional description of the clone, for the user’s own
purposes.
E.g. \sphinxcode{\sphinxupquote{IP address for website}}
\\
\hline
\end{tabular}
\par
\sphinxattableend\end{savenotes}

\index{options!clone}\index{clone!options}\ignorespaces 

\subsubsection{Clone Options}
\label{\detokenize{collective:clone-options}}\label{\detokenize{collective:index-9}}
{\hyperref[\detokenize{resources:resource-options}]{\sphinxcrossref{\DUrole{std,std-ref}{Options}}}} inherited from primitive resources:
\sphinxcode{\sphinxupquote{priority, target-role, is-managed}}


\begin{savenotes}\sphinxatlongtablestart\begin{longtable}{|\X{1}{5}|\X{1}{5}|\X{3}{5}|}
\caption{\sphinxstylestrong{Clone-specific configuration options}\strut}\label{\detokenize{collective:id8}}\\*[\sphinxlongtablecapskipadjust]
\hline
\sphinxstyletheadfamily 
Field
&\sphinxstyletheadfamily 
Default
&\sphinxstyletheadfamily 
Description
\\
\hline
\endfirsthead

\multicolumn{3}{c}%
{\makebox[0pt]{\sphinxtablecontinued{\tablename\ \thetable{} -- continued from previous page}}}\\
\hline
\sphinxstyletheadfamily 
Field
&\sphinxstyletheadfamily 
Default
&\sphinxstyletheadfamily 
Description
\\
\hline
\endhead

\hline
\multicolumn{3}{r}{\makebox[0pt][r]{\sphinxtablecontinued{Continued on next page}}}\\
\endfoot

\endlastfoot

globally-unique
&
false
&\begin{quote}

\index{clone!option, globally-unique}\index{option!globally-unique (clone)}\index{globally-unique!clone option}\ignorespaces \end{quote}

If \sphinxstylestrong{true}, each clone instance performs a
distinct function
\\
\hline
clone-max
&
0
&
\index{clone!option, clone-max}\index{option!clone-max (clone)}\index{clone-max!clone option}\ignorespaces 
The maximum number of clone instances that can
be started across the entire cluster. If 0, the
number of nodes in the cluster will be used.
\\
\hline
clone-node-max
&
1
&
\index{clone!option, clone-node-max}\index{option!clone-node-max (clone)}\index{clone-node-max!clone option}\ignorespaces 
If \sphinxcode{\sphinxupquote{globally-unique}} is \sphinxstylestrong{true}, the maximum
number of clone instances that can be started
on a single node
\\
\hline
clone-min
&
0
&
\index{clone!option, clone-min}\index{option!clone-min (clone)}\index{clone-min!clone option}\ignorespaces 
Require at least this number of clone instances
to be runnable before allowing resources
depending on the clone to be runnable. A value
of 0 means require all clone instances to be
runnable.
\\
\hline
notify
&
false
&
\index{clone!option, notify}\index{option!notify (clone)}\index{notify!clone option}\ignorespaces 
Call the resource agent’s \sphinxstylestrong{notify} action for
all active instances, before and after starting
or stopping any clone instance. The resource
agent must support this action.
Allowed values: \sphinxstylestrong{false}, \sphinxstylestrong{true}
\\
\hline
ordered
&
false
&
\index{clone!option, ordered}\index{option!ordered (clone)}\index{ordered!clone option}\ignorespaces 
If \sphinxstylestrong{true}, clone instances must be started
sequentially instead of in parallel.
Allowed values: \sphinxstylestrong{false}, \sphinxstylestrong{true}
\\
\hline
interleave
&
false
&
\index{clone!option, interleave}\index{option!interleave (clone)}\index{interleave!clone option}\ignorespaces 
When this clone is ordered relative to another
clone, if this option is \sphinxstylestrong{false} (the default),
the ordering is relative to \sphinxstyleemphasis{all} instances of
the other clone, whereas if this option is
\sphinxstylestrong{true}, the ordering is relative only to
instances on the same node.
Allowed values: \sphinxstylestrong{false}, \sphinxstylestrong{true}
\\
\hline
promotable
&
false
&
\index{clone!option, promotable}\index{option!promotable (clone)}\index{promotable!clone option}\ignorespaces 
If \sphinxstylestrong{true}, clone instances can perform a
special role that Pacemaker will manage via the
resource agent’s \sphinxstylestrong{promote} and \sphinxstylestrong{demote}
actions. The resource agent must support these
actions.
Allowed values: \sphinxstylestrong{false}, \sphinxstylestrong{true}
\\
\hline
promoted-max
&
1
&
\index{clone!option, promoted-max}\index{option!promoted-max (clone)}\index{promoted-max!clone option}\ignorespaces 
If \sphinxcode{\sphinxupquote{promotable}} is \sphinxstylestrong{true}, the number of
instances that can be promoted at one time
across the entire cluster
\\
\hline
promoted-node-max
&
1
&
\index{clone!option, promoted-node-max}\index{option!promoted-node-max (clone)}\index{promoted-node-max!clone option}\ignorespaces 
If \sphinxcode{\sphinxupquote{promotable}} is \sphinxstylestrong{true} and \sphinxcode{\sphinxupquote{globally-unique}}
is \sphinxstylestrong{false}, the number of clone instances can be
promoted at one time on a single node
\\
\hline
\end{longtable}\sphinxatlongtableend\end{savenotes}

\begin{sphinxadmonition}{note}{Note:}
\sphinxstylestrong{Deprecated Terminology}

In older documentation and online examples, you may see promotable clones
referred to as \sphinxstyleemphasis{multi-state}, \sphinxstyleemphasis{stateful}, or \sphinxstyleemphasis{master/slave}; these mean the
same thing as \sphinxstyleemphasis{promotable}. Certain syntax is supported for backward
compatibility, but is deprecated and will be removed in a future version:
\begin{itemize}
\item {} 
Using a \sphinxcode{\sphinxupquote{master}} tag, instead of a \sphinxcode{\sphinxupquote{clone}} tag with the \sphinxcode{\sphinxupquote{promotable}}
meta-attribute set to \sphinxcode{\sphinxupquote{true}}

\item {} 
Using the \sphinxcode{\sphinxupquote{master-max}} meta-attribute instead of \sphinxcode{\sphinxupquote{promoted-max}}

\item {} 
Using the \sphinxcode{\sphinxupquote{master-node-max}} meta-attribute instead of
\sphinxcode{\sphinxupquote{promoted-node-max}}

\item {} 
Using \sphinxcode{\sphinxupquote{Master}} as a role name instead of \sphinxcode{\sphinxupquote{Promoted}}

\item {} 
Using \sphinxcode{\sphinxupquote{Slave}} as a role name instead of \sphinxcode{\sphinxupquote{Unpromoted}}

\end{itemize}
\end{sphinxadmonition}


\subsubsection{Clone Contents}
\label{\detokenize{collective:clone-contents}}
Clones must contain exactly one primitive or group resource.

\begin{sphinxShadowBox}
\sphinxstyletopictitle{A clone that runs a web server on all nodes}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}clone} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}apache\PYGZhy{}clone\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}primitive} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}apache\PYGZdq{}} \PYG{n+na}{class=}\PYG{l+s}{\PYGZdq{}lsb\PYGZdq{}} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}apache\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}operations}\PYG{n+nt}{\PYGZgt{}}
           \PYG{n+nt}{\PYGZlt{}op} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}apache\PYGZhy{}monitor\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}monitor\PYGZdq{}} \PYG{n+na}{interval=}\PYG{l+s}{\PYGZdq{}30\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}/operations\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/primitive\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/clone\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

\begin{sphinxadmonition}{warning}{Warning:}
You should never reference the name of a clone’s child (the primitive or group
resource being cloned). If you think you need to do this, you probably need to
re-evaluate your design.
\end{sphinxadmonition}


\subsubsection{Clone Instance Attribute}
\label{\detokenize{collective:clone-instance-attribute}}
Clones have no instance attributes; however, any that are set here will be
inherited by the clone’s child.

\index{clone!constraint}\ignorespaces 

\subsubsection{Clone Constraints}
\label{\detokenize{collective:clone-constraints}}\label{\detokenize{collective:index-20}}
In most cases, a clone will have a single instance on each active cluster
node.  If this is not the case, you can indicate which nodes the
cluster should preferentially assign copies to with resource location
constraints.  These constraints are written no differently from those
for primitive resources except that the clone’s \sphinxstylestrong{id} is used.

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Some constraints involving clones}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}constraints}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}location} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}clone\PYGZhy{}prefers\PYGZhy{}node1\PYGZdq{}} \PYG{n+na}{rsc=}\PYG{l+s}{\PYGZdq{}apache\PYGZhy{}clone\PYGZdq{}} \PYG{n+na}{node=}\PYG{l+s}{\PYGZdq{}node1\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}500\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}colocation} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}stats\PYGZhy{}with\PYGZhy{}clone\PYGZdq{}} \PYG{n+na}{rsc=}\PYG{l+s}{\PYGZdq{}apache\PYGZhy{}stats\PYGZdq{}} \PYG{n+na}{with=}\PYG{l+s}{\PYGZdq{}apache\PYGZhy{}clone\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}order} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}start\PYGZhy{}clone\PYGZhy{}then\PYGZhy{}stats\PYGZdq{}} \PYG{n+na}{first=}\PYG{l+s}{\PYGZdq{}apache\PYGZhy{}clone\PYGZdq{}} \PYG{n+na}{then=}\PYG{l+s}{\PYGZdq{}apache\PYGZhy{}stats\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/constraints\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

Ordering constraints behave slightly differently for clones.  In the
example above, \sphinxcode{\sphinxupquote{apache-stats}} will wait until all copies of \sphinxcode{\sphinxupquote{apache-clone}}
that need to be started have done so before being started itself.
Only if \sphinxstyleemphasis{no} copies can be started will \sphinxcode{\sphinxupquote{apache-stats}} be prevented
from being active.  Additionally, the clone will wait for
\sphinxcode{\sphinxupquote{apache-stats}} to be stopped before stopping itself.

Colocation of a primitive or group resource with a clone means that
the resource can run on any node with an active instance of the clone.
The cluster will choose an instance based on where the clone is running and
the resource’s own location preferences.

Colocation between clones is also possible.  If one clone \sphinxstylestrong{A} is colocated
with another clone \sphinxstylestrong{B}, the set of allowed locations for \sphinxstylestrong{A} is limited to
nodes on which \sphinxstylestrong{B} is (or will be) active.  Placement is then performed
normally.

\index{promotable clone!constraint}\ignorespaces 

\paragraph{Promotable Clone Constraints}
\label{\detokenize{collective:promotable-clone-constraints}}\label{\detokenize{collective:index-21}}\label{\detokenize{collective:id2}}
For promotable clone resources, the \sphinxcode{\sphinxupquote{first-action}} and/or \sphinxcode{\sphinxupquote{then-action}} fields
for ordering constraints may be set to \sphinxcode{\sphinxupquote{promote}} or \sphinxcode{\sphinxupquote{demote}} to constrain the
promoted role, and colocation constraints may contain \sphinxcode{\sphinxupquote{rsc-role}} and/or
\sphinxcode{\sphinxupquote{with-rsc-role}} fields.

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Constraints involving promotable clone resources}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}constraints}\PYG{n+nt}{\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}location} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}db\PYGZhy{}prefers\PYGZhy{}node1\PYGZdq{}} \PYG{n+na}{rsc=}\PYG{l+s}{\PYGZdq{}database\PYGZdq{}} \PYG{n+na}{node=}\PYG{l+s}{\PYGZdq{}node1\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}500\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}colocation} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}backup\PYGZhy{}with\PYGZhy{}db\PYGZhy{}unpromoted\PYGZdq{}} \PYG{n+na}{rsc=}\PYG{l+s}{\PYGZdq{}backup\PYGZdq{}}
     \PYG{n+na}{with\PYGZhy{}rsc=}\PYG{l+s}{\PYGZdq{}database\PYGZdq{}} \PYG{n+na}{with\PYGZhy{}rsc\PYGZhy{}role=}\PYG{l+s}{\PYGZdq{}Unpromoted\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}colocation} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}myapp\PYGZhy{}with\PYGZhy{}db\PYGZhy{}promoted\PYGZdq{}} \PYG{n+na}{rsc=}\PYG{l+s}{\PYGZdq{}myApp\PYGZdq{}}
     \PYG{n+na}{with\PYGZhy{}rsc=}\PYG{l+s}{\PYGZdq{}database\PYGZdq{}} \PYG{n+na}{with\PYGZhy{}rsc\PYGZhy{}role=}\PYG{l+s}{\PYGZdq{}Promoted\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}order} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}start\PYGZhy{}db\PYGZhy{}before\PYGZhy{}backup\PYGZdq{}} \PYG{n+na}{first=}\PYG{l+s}{\PYGZdq{}database\PYGZdq{}} \PYG{n+na}{then=}\PYG{l+s}{\PYGZdq{}backup\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}order} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}promote\PYGZhy{}db\PYGZhy{}then\PYGZhy{}app\PYGZdq{}} \PYG{n+na}{first=}\PYG{l+s}{\PYGZdq{}database\PYGZdq{}} \PYG{n+na}{first\PYGZhy{}action=}\PYG{l+s}{\PYGZdq{}promote\PYGZdq{}}
     \PYG{n+na}{then=}\PYG{l+s}{\PYGZdq{}myApp\PYGZdq{}} \PYG{n+na}{then\PYGZhy{}action=}\PYG{l+s}{\PYGZdq{}start\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/constraints\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

In the example above, \sphinxstylestrong{myApp} will wait until one of the database
copies has been started and promoted before being started
itself on the same node.  Only if no copies can be promoted will \sphinxstylestrong{myApp} be
prevented from being active.  Additionally, the cluster will wait for
\sphinxstylestrong{myApp} to be stopped before demoting the database.

Colocation of a primitive or group resource with a promotable clone
resource means that it can run on any node with an active instance of
the promotable clone resource that has the specified role (\sphinxcode{\sphinxupquote{Promoted}} or
\sphinxcode{\sphinxupquote{Unpromoted}}).  In the example above, the cluster will choose a location
based on where database is running in the promoted role, and if there are
multiple promoted instances it will also factor in \sphinxstylestrong{myApp}’s own location
preferences when deciding which location to choose.

Colocation with regular clones and other promotable clone resources is also
possible.  In such cases, the set of allowed locations for the \sphinxstylestrong{rsc}
clone is (after role filtering) limited to nodes on which the
\sphinxcode{\sphinxupquote{with-rsc}} promotable clone resource is (or will be) in the specified role.
Placement is then performed as normal.


\paragraph{Using Promotable Clone Resources in Colocation Sets}
\label{\detokenize{collective:using-promotable-clone-resources-in-colocation-sets}}
When a promotable clone is used in a {\hyperref[\detokenize{constraints:s-resource-sets}]{\sphinxcrossref{\DUrole{std,std-ref}{resource set}}}}
inside a colocation constraint, the resource set may take a \sphinxcode{\sphinxupquote{role}} attribute.

In the following example, an instance of \sphinxstylestrong{B} may be promoted only on a node
where \sphinxstylestrong{A} is in the promoted role. Additionally, resources \sphinxstylestrong{C} and \sphinxstylestrong{D}
must be located on a node where both \sphinxstylestrong{A} and \sphinxstylestrong{B} are promoted.

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Colocate C and D with A’s and B’s promoted instances}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}constraints}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}colocation} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}coloc\PYGZhy{}1\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}INFINITY\PYGZdq{}} \PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}set} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}colocated\PYGZhy{}set\PYGZhy{}example\PYGZhy{}1\PYGZdq{}} \PYG{n+na}{sequential=}\PYG{l+s}{\PYGZdq{}true\PYGZdq{}} \PYG{n+na}{role=}\PYG{l+s}{\PYGZdq{}Promoted\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}A\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}B\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}/resource\PYGZus{}set\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}set} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}colocated\PYGZhy{}set\PYGZhy{}example\PYGZhy{}2\PYGZdq{}} \PYG{n+na}{sequential=}\PYG{l+s}{\PYGZdq{}true\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}C\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}D\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}/resource\PYGZus{}set\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/rsc\PYGZus{}colocation\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/constraints\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}


\paragraph{Using Promotable Clone Resources in Ordered Sets}
\label{\detokenize{collective:using-promotable-clone-resources-in-ordered-sets}}
When a promotable clone is used in a {\hyperref[\detokenize{constraints:s-resource-sets}]{\sphinxcrossref{\DUrole{std,std-ref}{resource set}}}}
inside an ordering constraint, the resource set may take an \sphinxcode{\sphinxupquote{action}}
attribute.

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Start C and D after first promoting A and B}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}constraints}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}order} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}order\PYGZhy{}1\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}INFINITY\PYGZdq{}} \PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}set} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}ordered\PYGZhy{}set\PYGZhy{}1\PYGZdq{}} \PYG{n+na}{sequential=}\PYG{l+s}{\PYGZdq{}true\PYGZdq{}} \PYG{n+na}{action=}\PYG{l+s}{\PYGZdq{}promote\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}A\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}B\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}/resource\PYGZus{}set\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}set} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}ordered\PYGZhy{}set\PYGZhy{}2\PYGZdq{}} \PYG{n+na}{sequential=}\PYG{l+s}{\PYGZdq{}true\PYGZdq{}} \PYG{n+na}{action=}\PYG{l+s}{\PYGZdq{}start\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}C\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}D\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}/resource\PYGZus{}set\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/rsc\PYGZus{}order\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/constraints\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

In the above example, \sphinxstylestrong{B} cannot be promoted until \sphinxstylestrong{A} has been promoted.
Additionally, resources \sphinxstylestrong{C} and \sphinxstylestrong{D} must wait until \sphinxstylestrong{A} and \sphinxstylestrong{B} have
been promoted before they can start.

\index{resource-stickiness!clone}\index{clone!resource-stickiness}\ignorespaces 

\subsubsection{Clone Stickiness}
\label{\detokenize{collective:clone-stickiness}}\label{\detokenize{collective:s-clone-stickiness}}\label{\detokenize{collective:index-22}}
To achieve stable assignments, clones are slightly sticky by default. If no
value for \sphinxcode{\sphinxupquote{resource-stickiness}} is provided, the clone will use a value of 1.
Being a small value, it causes minimal disturbance to the score calculations of
other resources but is enough to prevent Pacemaker from needlessly moving
instances around the cluster.

\begin{sphinxadmonition}{note}{Note:}
For globally unique clones, this may result in multiple instances of the
clone staying on a single node, even after another eligible node becomes
active (for example, after being put into standby mode then made active again).
If you do not want this behavior, specify a \sphinxcode{\sphinxupquote{resource-stickiness}} of 0
for the clone temporarily and let the cluster adjust, then set it back
to 1 if you want the default behavior to apply again.
\end{sphinxadmonition}

\begin{sphinxadmonition}{important}{Important:}
If \sphinxcode{\sphinxupquote{resource-stickiness}} is set in the \sphinxcode{\sphinxupquote{rsc\_defaults}} section, it will
apply to clone instances as well. This means an explicit \sphinxcode{\sphinxupquote{resource-stickiness}}
of 0 in \sphinxcode{\sphinxupquote{rsc\_defaults}} works differently from the implicit default used when
\sphinxcode{\sphinxupquote{resource-stickiness}} is not specified.
\end{sphinxadmonition}


\subsubsection{Clone Resource Agent Requirements}
\label{\detokenize{collective:clone-resource-agent-requirements}}
Any resource can be used as an anonymous clone, as it requires no
additional support from the resource agent.  Whether it makes sense to
do so depends on your resource and its resource agent.


\paragraph{Resource Agent Requirements for Globally Unique Clones}
\label{\detokenize{collective:resource-agent-requirements-for-globally-unique-clones}}
Globally unique clones require additional support in the resource agent. In
particular, it must only respond with \sphinxcode{\sphinxupquote{\$\{OCF\_SUCCESS\}}} if the node has that
exact instance active. All other probes for instances of the clone should
result in \sphinxcode{\sphinxupquote{\$\{OCF\_NOT\_RUNNING\}}} (or one of the other OCF error codes if
they are failed).

Individual instances of a clone are identified by appending a colon and a
numerical offset, e.g. \sphinxstylestrong{apache:2}.

Resource agents can find out how many copies there are by examining
the \sphinxcode{\sphinxupquote{OCF\_RESKEY\_CRM\_meta\_clone\_max}} environment variable and which
instance it is by examining \sphinxcode{\sphinxupquote{OCF\_RESKEY\_CRM\_meta\_clone}}.

The resource agent must not make any assumptions (based on
\sphinxcode{\sphinxupquote{OCF\_RESKEY\_CRM\_meta\_clone}}) about which numerical instances are active.  In
particular, the list of active copies will not always be an unbroken
sequence, nor always start at 0.


\paragraph{Resource Agent Requirements for Promotable Clones}
\label{\detokenize{collective:resource-agent-requirements-for-promotable-clones}}
Promotable clone resources require two extra actions, \sphinxcode{\sphinxupquote{demote}} and \sphinxcode{\sphinxupquote{promote}},
which are responsible for changing the state of the resource. Like \sphinxstylestrong{start} and
\sphinxstylestrong{stop}, they should return \sphinxcode{\sphinxupquote{\$\{OCF\_SUCCESS\}}} if they completed successfully or
a relevant error code if they did not.

The states can mean whatever you wish, but when the resource is
started, it must come up in the unpromoted role. From there, the
cluster will decide which instances to promote.

In addition to the clone requirements for monitor actions, agents must
also \sphinxstyleemphasis{accurately} report which state they are in.  The cluster relies
on the agent to report its status (including role) accurately and does
not indicate to the agent what role it currently believes it to be in.


\begin{savenotes}\sphinxattablestart
\centering
\sphinxcapstartof{table}
\sphinxcaption{\sphinxstylestrong{Role implications of OCF return codes}}\label{\detokenize{collective:id9}}
\sphinxaftercaption
\begin{tabular}[t]{|\X{1}{4}|\X{3}{4}|}
\hline
\sphinxstyletheadfamily 
Monitor Return Code
&\sphinxstyletheadfamily 
Description
\\
\hline
OCF\_NOT\_RUNNING
&
\index{OCF\_NOT\_RUNNING}\index{OCF return code!OCF\_NOT\_RUNNING}\ignorespaces 
Stopped
\\
\hline
OCF\_SUCCESS
&
\index{OCF\_SUCCESS}\index{OCF return code!OCF\_SUCCESS}\ignorespaces 
Running (Unpromoted)
\\
\hline
OCF\_RUNNING\_PROMOTED
&
\index{OCF\_RUNNING\_PROMOTED}\index{OCF return code!OCF\_RUNNING\_PROMOTED}\ignorespaces 
Running (Promoted)
\\
\hline
OCF\_FAILED\_PROMOTED
&
\index{OCF\_FAILED\_PROMOTED}\index{OCF return code!OCF\_FAILED\_PROMOTED}\ignorespaces 
Failed (Promoted)
\\
\hline
Other
&
\index{return code}\ignorespaces 
Failed (Unpromoted)
\\
\hline
\end{tabular}
\par
\sphinxattableend\end{savenotes}


\paragraph{Clone Notifications}
\label{\detokenize{collective:clone-notifications}}
If the clone has the \sphinxcode{\sphinxupquote{notify}} meta-attribute set to \sphinxstylestrong{true}, and the resource
agent supports the \sphinxcode{\sphinxupquote{notify}} action, Pacemaker will call the action when
appropriate, passing a number of extra variables which, when combined with
additional context, can be used to calculate the current state of the cluster
and what is about to happen to it.

\index{clone!environment variables}\index{notify!environment variables}\ignorespaces 

\begin{savenotes}\sphinxattablestart
\centering
\sphinxcapstartof{table}
\sphinxcaption{\sphinxstylestrong{Environment variables supplied with Clone notify actions}}\label{\detokenize{collective:id10}}\label{\detokenize{collective:index-28}}
\sphinxaftercaption
\begin{tabular}[t]{|\X{1}{2}|\X{1}{2}|}
\hline
\sphinxstyletheadfamily 
Variable
&\sphinxstyletheadfamily 
Description
\\
\hline
OCF\_RESKEY\_CRM\_meta\_notify\_type
&
\index{environment variable!OCF\_RESKEY\_CRM\_meta\_notify\_type}\index{OCF\_RESKEY\_CRM\_meta\_notify\_type}\ignorespaces 
Allowed values: \sphinxstylestrong{pre}, \sphinxstylestrong{post}
\\
\hline
OCF\_RESKEY\_CRM\_meta\_notify\_operation
&
\index{environment variable!OCF\_RESKEY\_CRM\_meta\_notify\_operation}\index{OCF\_RESKEY\_CRM\_meta\_notify\_operation}\ignorespaces 
Allowed values: \sphinxstylestrong{start}, \sphinxstylestrong{stop}
\\
\hline
OCF\_RESKEY\_CRM\_meta\_notify\_start\_resource
&
\index{environment variable!OCF\_RESKEY\_CRM\_meta\_notify\_start\_resource}\index{OCF\_RESKEY\_CRM\_meta\_notify\_start\_resource}\ignorespaces 
Resources to be started
\\
\hline
OCF\_RESKEY\_CRM\_meta\_notify\_stop\_resource
&
\index{environment variable!OCF\_RESKEY\_CRM\_meta\_notify\_stop\_resource}\index{OCF\_RESKEY\_CRM\_meta\_notify\_stop\_resource}\ignorespaces 
Resources to be stopped
\\
\hline
OCF\_RESKEY\_CRM\_meta\_notify\_active\_resource
&
\index{environment variable!OCF\_RESKEY\_CRM\_meta\_notify\_active\_resource}\index{OCF\_RESKEY\_CRM\_meta\_notify\_active\_resource}\ignorespaces 
Resources that are running
\\
\hline
OCF\_RESKEY\_CRM\_meta\_notify\_inactive\_resource
&
\index{environment variable!OCF\_RESKEY\_CRM\_meta\_notify\_inactive\_resource}\index{OCF\_RESKEY\_CRM\_meta\_notify\_inactive\_resource}\ignorespaces 
Resources that are not running
\\
\hline
OCF\_RESKEY\_CRM\_meta\_notify\_start\_uname
&
\index{environment variable!OCF\_RESKEY\_CRM\_meta\_notify\_start\_uname}\index{OCF\_RESKEY\_CRM\_meta\_notify\_start\_uname}\ignorespaces 
Nodes on which resources will be started
\\
\hline
OCF\_RESKEY\_CRM\_meta\_notify\_stop\_uname
&
\index{environment variable!OCF\_RESKEY\_CRM\_meta\_notify\_stop\_uname}\index{OCF\_RESKEY\_CRM\_meta\_notify\_stop\_uname}\ignorespaces 
Nodes on which resources will be stopped
\\
\hline
OCF\_RESKEY\_CRM\_meta\_notify\_active\_uname
&
\index{environment variable!OCF\_RESKEY\_CRM\_meta\_notify\_active\_uname}\index{OCF\_RESKEY\_CRM\_meta\_notify\_active\_uname}\ignorespaces 
Nodes on which resources are running
\\
\hline
\end{tabular}
\par
\sphinxattableend\end{savenotes}

The variables come in pairs, such as
\sphinxcode{\sphinxupquote{OCF\_RESKEY\_CRM\_meta\_notify\_start\_resource}} and
\sphinxcode{\sphinxupquote{OCF\_RESKEY\_CRM\_meta\_notify\_start\_uname}}, and should be treated as an
array of whitespace-separated elements.

\sphinxcode{\sphinxupquote{OCF\_RESKEY\_CRM\_meta\_notify\_inactive\_resource}} is an exception, as the
matching \sphinxstylestrong{uname} variable does not exist since inactive resources
are not running on any node.

Thus, in order to indicate that \sphinxstylestrong{clone:0} will be started on \sphinxstylestrong{sles-1},
\sphinxstylestrong{clone:2} will be started on \sphinxstylestrong{sles-3}, and \sphinxstylestrong{clone:3} will be started
on \sphinxstylestrong{sles-2}, the cluster would set:

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Notification variables}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
OCF\PYGZus{}RESKEY\PYGZus{}CRM\PYGZus{}meta\PYGZus{}notify\PYGZus{}start\PYGZus{}resource=\PYGZdq{}clone:0 clone:2 clone:3\PYGZdq{}
OCF\PYGZus{}RESKEY\PYGZus{}CRM\PYGZus{}meta\PYGZus{}notify\PYGZus{}start\PYGZus{}uname=\PYGZdq{}sles\PYGZhy{}1 sles\PYGZhy{}3 sles\PYGZhy{}2\PYGZdq{}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

\begin{sphinxadmonition}{note}{Note:}
Pacemaker will log but otherwise ignore failures of notify actions.
\end{sphinxadmonition}


\paragraph{Interpretation of Notification Variables}
\label{\detokenize{collective:interpretation-of-notification-variables}}
\sphinxstylestrong{Pre-notification (stop):}
\begin{itemize}
\item {} 
Active resources: \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_active\_resource}}

\item {} 
Inactive resources: \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_inactive\_resource}}

\item {} 
Resources to be started: \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_start\_resource}}

\item {} 
Resources to be stopped: \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_stop\_resource}}

\end{itemize}

\sphinxstylestrong{Post-notification (stop) / Pre-notification (start):}
\begin{itemize}
\item {} 
Active resources
\begin{itemize}
\item {} 
\sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_active\_resource}}

\item {} 
minus \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_stop\_resource}}

\end{itemize}

\item {} 
Inactive resources
\begin{itemize}
\item {} 
\sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_inactive\_resource}}

\item {} 
plus \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_stop\_resource}}

\end{itemize}

\item {} 
Resources that were started: \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_start\_resource}}

\item {} 
Resources that were stopped: \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_stop\_resource}}

\end{itemize}

\sphinxstylestrong{Post-notification (start):}
\begin{itemize}
\item {} 
Active resources:
\begin{itemize}
\item {} 
\sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_active\_resource}}

\item {} 
minus \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_stop\_resource}}

\item {} 
plus \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_start\_resource}}

\end{itemize}

\item {} 
Inactive resources:
\begin{itemize}
\item {} 
\sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_inactive\_resource}}

\item {} 
plus \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_stop\_resource}}

\item {} 
minus \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_start\_resource}}

\end{itemize}

\item {} 
Resources that were started: \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_start\_resource}}

\item {} 
Resources that were stopped: \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_stop\_resource}}

\end{itemize}


\paragraph{Extra Notifications for Promotable Clones}
\label{\detokenize{collective:extra-notifications-for-promotable-clones}}
\index{clone!environment variables}\index{promotable!environment variables}\ignorespaces 

\begin{savenotes}\sphinxattablestart
\centering
\sphinxcapstartof{table}
\sphinxcaption{\sphinxstylestrong{Extra environment variables supplied for promotable clones}}\label{\detokenize{collective:id11}}\label{\detokenize{collective:index-38}}
\sphinxaftercaption
\begin{tabular}[t]{|\X{1}{2}|\X{1}{2}|}
\hline
\sphinxstyletheadfamily 
Variable
&\sphinxstyletheadfamily 
Description
\\
\hline
OCF\_RESKEY\_CRM\_meta\_notify\_promoted\_resource
&
\index{environment variable!OCF\_RESKEY\_CRM\_meta\_notify\_promoted\_resource}\index{OCF\_RESKEY\_CRM\_meta\_notify\_promoted\_resource}\ignorespaces 
Resources that are running in the promoted role
\\
\hline
OCF\_RESKEY\_CRM\_meta\_notify\_unpromoted\_resource
&
\index{environment variable!OCF\_RESKEY\_CRM\_meta\_notify\_unpromoted\_resource}\index{OCF\_RESKEY\_CRM\_meta\_notify\_unpromoted\_resource}\ignorespaces 
Resources that are running in the unpromoted role
\\
\hline
OCF\_RESKEY\_CRM\_meta\_notify\_promote\_resource
&
\index{environment variable!OCF\_RESKEY\_CRM\_meta\_notify\_promote\_resource}\index{OCF\_RESKEY\_CRM\_meta\_notify\_promote\_resource}\ignorespaces 
Resources to be promoted
\\
\hline
OCF\_RESKEY\_CRM\_meta\_notify\_demote\_resource
&
\index{environment variable!OCF\_RESKEY\_CRM\_meta\_notify\_demote\_resource}\index{OCF\_RESKEY\_CRM\_meta\_notify\_demote\_resource}\ignorespaces 
Resources to be demoted
\\
\hline
OCF\_RESKEY\_CRM\_meta\_notify\_promote\_uname
&
\index{environment variable!OCF\_RESKEY\_CRM\_meta\_notify\_promote\_uname}\index{OCF\_RESKEY\_CRM\_meta\_notify\_promote\_uname}\ignorespaces 
Nodes on which resources will be promoted
\\
\hline
OCF\_RESKEY\_CRM\_meta\_notify\_demote\_uname
&
\index{environment variable!OCF\_RESKEY\_CRM\_meta\_notify\_demote\_uname}\index{OCF\_RESKEY\_CRM\_meta\_notify\_demote\_uname}\ignorespaces 
Nodes on which resources will be demoted
\\
\hline
OCF\_RESKEY\_CRM\_meta\_notify\_promoted\_uname
&
\index{environment variable!OCF\_RESKEY\_CRM\_meta\_notify\_promoted\_uname}\index{OCF\_RESKEY\_CRM\_meta\_notify\_promoted\_uname}\ignorespaces 
Nodes on which resources are running in the promoted role
\\
\hline
OCF\_RESKEY\_CRM\_meta\_notify\_unpromoted\_uname
&
\index{environment variable!OCF\_RESKEY\_CRM\_meta\_notify\_unpromoted\_uname}\index{OCF\_RESKEY\_CRM\_meta\_notify\_unpromoted\_uname}\ignorespaces 
Nodes on which resources are running in the unpromoted role
\\
\hline
\end{tabular}
\par
\sphinxattableend\end{savenotes}


\paragraph{Interpretation of Promotable Notification Variables}
\label{\detokenize{collective:interpretation-of-promotable-notification-variables}}
\sphinxstylestrong{Pre-notification (demote):}
\begin{itemize}
\item {} 
Active resources: \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_active\_resource}}

\item {} 
Promoted resources: \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_promoted\_resource}}

\item {} 
Unpromoted resources: \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_unpromoted\_resource}}

\item {} 
Inactive resources: \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_inactive\_resource}}

\item {} 
Resources to be started: \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_start\_resource}}

\item {} 
Resources to be promoted: \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_promote\_resource}}

\item {} 
Resources to be demoted: \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_demote\_resource}}

\item {} 
Resources to be stopped: \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_stop\_resource}}

\end{itemize}

\sphinxstylestrong{Post-notification (demote) / Pre-notification (stop):}
\begin{itemize}
\item {} 
Active resources: \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_active\_resource}}

\item {} 
Promoted resources:
\begin{itemize}
\item {} 
\sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_promoted\_resource}}

\item {} 
minus \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_demote\_resource}}

\end{itemize}

\item {} 
Unpromoted resources: \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_unpromoted\_resource}}

\item {} 
Inactive resources: \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_inactive\_resource}}

\item {} 
Resources to be started: \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_start\_resource}}

\item {} 
Resources to be promoted: \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_promote\_resource}}

\item {} 
Resources to be demoted: \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_demote\_resource}}

\item {} 
Resources to be stopped: \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_stop\_resource}}

\item {} 
Resources that were demoted: \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_demote\_resource}}

\end{itemize}

\sphinxstylestrong{Post-notification (stop) / Pre-notification (start)}
\begin{itemize}
\item {} 
Active resources:
\begin{itemize}
\item {} 
\sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_active\_resource}}

\item {} 
minus \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_stop\_resource}}

\end{itemize}

\item {} 
Promoted resources:
\begin{itemize}
\item {} 
\sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_promoted\_resource}}

\item {} 
minus \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_demote\_resource}}

\end{itemize}

\item {} 
Unpromoted resources:
\begin{itemize}
\item {} 
\sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_unpromoted\_resource}}

\item {} 
minus \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_stop\_resource}}

\end{itemize}

\item {} 
Inactive resources:
\begin{itemize}
\item {} 
\sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_inactive\_resource}}

\item {} 
plus \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_stop\_resource}}

\end{itemize}

\item {} 
Resources to be started: \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_start\_resource}}

\item {} 
Resources to be promoted: \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_promote\_resource}}

\item {} 
Resources to be demoted: \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_demote\_resource}}

\item {} 
Resources to be stopped: \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_stop\_resource}}

\item {} 
Resources that were demoted: \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_demote\_resource}}

\item {} 
Resources that were stopped: \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_stop\_resource}}

\end{itemize}

\sphinxstylestrong{Post-notification (start) / Pre-notification (promote)}
\begin{itemize}
\item {} 
Active resources:
\begin{itemize}
\item {} 
\sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_active\_resource}}

\item {} 
minus \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_stop\_resource}}

\item {} 
plus \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_start\_resource}}

\end{itemize}

\item {} 
Promoted resources:
\begin{itemize}
\item {} 
\sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_promoted\_resource}}

\item {} 
minus \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_demote\_resource}}

\end{itemize}

\item {} 
Unpromoted resources:
\begin{itemize}
\item {} 
\sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_unpromoted\_resource}}

\item {} 
minus \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_stop\_resource}}

\item {} 
plus \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_start\_resource}}

\end{itemize}

\item {} 
Inactive resources:
\begin{itemize}
\item {} 
\sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_inactive\_resource}}

\item {} 
plus \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_stop\_resource}}

\item {} 
minus \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_start\_resource}}

\end{itemize}

\item {} 
Resources to be started: \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_start\_resource}}

\item {} 
Resources to be promoted: \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_promote\_resource}}

\item {} 
Resources to be demoted: \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_demote\_resource}}

\item {} 
Resources to be stopped: \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_stop\_resource}}

\item {} 
Resources that were started: \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_start\_resource}}

\item {} 
Resources that were demoted: \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_demote\_resource}}

\item {} 
Resources that were stopped: \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_stop\_resource}}

\end{itemize}

\sphinxstylestrong{Post-notification (promote)}
\begin{itemize}
\item {} 
Active resources:
\begin{itemize}
\item {} 
\sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_active\_resource}}

\item {} 
minus \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_stop\_resource}}

\item {} 
plus \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_start\_resource}}

\end{itemize}

\item {} 
Promoted resources:
\begin{itemize}
\item {} 
\sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_promoted\_resource}}

\item {} 
minus \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_demote\_resource}}

\item {} 
plus \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_promote\_resource}}

\end{itemize}

\item {} 
Unpromoted resources:
\begin{itemize}
\item {} 
\sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_unpromoted\_resource}}

\item {} 
minus \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_stop\_resource}}

\item {} 
plus \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_start\_resource}}

\item {} 
minus \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_promote\_resource}}

\end{itemize}

\item {} 
Inactive resources:
\begin{itemize}
\item {} 
\sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_inactive\_resource}}

\item {} 
plus \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_stop\_resource}}

\item {} 
minus \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_start\_resource}}

\end{itemize}

\item {} 
Resources to be started: \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_start\_resource}}

\item {} 
Resources to be promoted: \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_promote\_resource}}

\item {} 
Resources to be demoted: \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_demote\_resource}}

\item {} 
Resources to be stopped: \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_stop\_resource}}

\item {} 
Resources that were started: \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_start\_resource}}

\item {} 
Resources that were promoted: \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_promote\_resource}}

\item {} 
Resources that were demoted: \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_demote\_resource}}

\item {} 
Resources that were stopped: \sphinxcode{\sphinxupquote{\$OCF\_RESKEY\_CRM\_meta\_notify\_stop\_resource}}

\end{itemize}


\subsubsection{Monitoring Promotable Clone Resources}
\label{\detokenize{collective:monitoring-promotable-clone-resources}}
The usual monitor actions are insufficient to monitor a promotable clone
resource, because Pacemaker needs to verify not only that the resource is
active, but also that its actual role matches its intended one.

Define two monitoring actions: the usual one will cover the unpromoted role,
and an additional one with \sphinxcode{\sphinxupquote{role="Promoted"}} will cover the promoted role.

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Monitoring both states of a promotable clone resource}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}clone} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}myPromotableRsc\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}meta\PYGZus{}attributes} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}myPromotableRsc\PYGZhy{}meta\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
       \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}promotable\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}true\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}/meta\PYGZus{}attributes\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}primitive} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}myRsc\PYGZdq{}} \PYG{n+na}{class=}\PYG{l+s}{\PYGZdq{}ocf\PYGZdq{}} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}myApp\PYGZdq{}} \PYG{n+na}{provider=}\PYG{l+s}{\PYGZdq{}myCorp\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}operations}\PYG{n+nt}{\PYGZgt{}}
     \PYG{n+nt}{\PYGZlt{}op} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}public\PYGZhy{}ip\PYGZhy{}unpromoted\PYGZhy{}check\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}monitor\PYGZdq{}} \PYG{n+na}{interval=}\PYG{l+s}{\PYGZdq{}60\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
     \PYG{n+nt}{\PYGZlt{}op} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}public\PYGZhy{}ip\PYGZhy{}promoted\PYGZhy{}check\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}monitor\PYGZdq{}} \PYG{n+na}{interval=}\PYG{l+s}{\PYGZdq{}61\PYGZdq{}} \PYG{n+na}{role=}\PYG{l+s}{\PYGZdq{}Promoted\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/operations\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}/primitive\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/clone\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

\begin{sphinxadmonition}{important}{Important:}
It is crucial that \sphinxstyleemphasis{every} monitor operation has a different interval!
Pacemaker currently differentiates between operations
only by resource and interval; so if (for example) a promotable clone resource
had the same monitor interval for both roles, Pacemaker would ignore the
role when checking the status \textendash{} which would cause unexpected return
codes, and therefore unnecessary complications.
\end{sphinxadmonition}


\subsubsection{Determining Which Instance is Promoted}
\label{\detokenize{collective:determining-which-instance-is-promoted}}\label{\detokenize{collective:s-promotion-scores}}
Pacemaker can choose a promotable clone instance to be promoted in one of two
ways:
\begin{itemize}
\item {} 
Promotion scores: These are node attributes set via the \sphinxcode{\sphinxupquote{crm\_attribute}}
command using the \sphinxcode{\sphinxupquote{-{-}promotion}} option, which generally would be called by
the resource agent’s start action if it supports promotable clones. This tool
automatically detects both the resource and host, and should be used to set a
preference for being promoted. Based on this, \sphinxcode{\sphinxupquote{promoted-max}}, and
\sphinxcode{\sphinxupquote{promoted-node-max}}, the instance(s) with the highest preference will be
promoted.

\item {} 
Constraints: Location constraints can indicate which nodes are most preferred
to be promoted.

\end{itemize}

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Explicitly preferring node1 to be promoted}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}location} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}promoted\PYGZhy{}location\PYGZdq{}} \PYG{n+na}{rsc=}\PYG{l+s}{\PYGZdq{}myPromotableRsc\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}rule} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}promoted\PYGZhy{}rule\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}100\PYGZdq{}} \PYG{n+na}{role=}\PYG{l+s}{\PYGZdq{}Promoted\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}expression} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}promoted\PYGZhy{}exp\PYGZdq{}} \PYG{n+na}{attribute=}\PYG{l+s}{\PYGZdq{}\PYGZsh{}uname\PYGZdq{}} \PYG{n+na}{operation=}\PYG{l+s}{\PYGZdq{}eq\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}node1\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/rule\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/rsc\PYGZus{}location\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}


\subsection{Bundles - Containerized Resources}
\label{\detokenize{collective:bundles-containerized-resources}}\label{\detokenize{collective:s-resource-bundle}}
Pacemaker supports a special syntax for launching a service inside a
\sphinxhref{https://en.wikipedia.org/wiki/Operating-system-level\_virtualization}{container}
with any infrastructure it requires: the \sphinxstyleemphasis{bundle}.

Pacemaker bundles support \sphinxhref{https://www.docker.com/}{Docker},
\sphinxhref{https://podman.io/}{podman} \sphinxstyleemphasis{(since 2.0.1)}, and
\sphinxhref{https://coreos.com/rkt/}{rkt} container technologies. %
\begin{footnote}[2]\sphinxAtStartFootnote
Docker is a trademark of Docker, Inc. No endorsement by or association with
Docker, Inc. is implied.
%
\end{footnote}

\begin{sphinxShadowBox}
\sphinxstyletopictitle{A bundle for a containerized web server}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}bundle} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}httpd\PYGZhy{}bundle\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}podman} \PYG{n+na}{image=}\PYG{l+s}{\PYGZdq{}pcmk:http\PYGZdq{}} \PYG{n+na}{replicas=}\PYG{l+s}{\PYGZdq{}3\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}network} \PYG{n+na}{ip\PYGZhy{}range\PYGZhy{}start=}\PYG{l+s}{\PYGZdq{}192.168.122.131\PYGZdq{}}
            \PYG{n+na}{host\PYGZhy{}netmask=}\PYG{l+s}{\PYGZdq{}24\PYGZdq{}}
            \PYG{n+na}{host\PYGZhy{}interface=}\PYG{l+s}{\PYGZdq{}eth0\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}port\PYGZhy{}mapping} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}httpd\PYGZhy{}port\PYGZdq{}} \PYG{n+na}{port=}\PYG{l+s}{\PYGZdq{}80\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}/network\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}storage}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}storage\PYGZhy{}mapping} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}httpd\PYGZhy{}syslog\PYGZdq{}}
                       \PYG{n+na}{source\PYGZhy{}dir=}\PYG{l+s}{\PYGZdq{}/dev/log\PYGZdq{}}
                       \PYG{n+na}{target\PYGZhy{}dir=}\PYG{l+s}{\PYGZdq{}/dev/log\PYGZdq{}}
                       \PYG{n+na}{options=}\PYG{l+s}{\PYGZdq{}rw\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}storage\PYGZhy{}mapping} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}httpd\PYGZhy{}root\PYGZdq{}}
                       \PYG{n+na}{source\PYGZhy{}dir=}\PYG{l+s}{\PYGZdq{}/srv/html\PYGZdq{}}
                       \PYG{n+na}{target\PYGZhy{}dir=}\PYG{l+s}{\PYGZdq{}/var/www/html\PYGZdq{}}
                       \PYG{n+na}{options=}\PYG{l+s}{\PYGZdq{}rw,Z\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}storage\PYGZhy{}mapping} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}httpd\PYGZhy{}logs\PYGZdq{}}
                       \PYG{n+na}{source\PYGZhy{}dir\PYGZhy{}root=}\PYG{l+s}{\PYGZdq{}/var/log/pacemaker/bundles\PYGZdq{}}
                       \PYG{n+na}{target\PYGZhy{}dir=}\PYG{l+s}{\PYGZdq{}/etc/httpd/logs\PYGZdq{}}
                       \PYG{n+na}{options=}\PYG{l+s}{\PYGZdq{}rw,Z\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}/storage\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}primitive} \PYG{n+na}{class=}\PYG{l+s}{\PYGZdq{}ocf\PYGZdq{}} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}httpd\PYGZdq{}} \PYG{n+na}{provider=}\PYG{l+s}{\PYGZdq{}heartbeat\PYGZdq{}} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}apache\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/bundle\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}


\subsubsection{Bundle Prerequisites}
\label{\detokenize{collective:bundle-prerequisites}}
Before configuring a bundle in Pacemaker, the user must install the appropriate
container launch technology (Docker, podman, or rkt), and supply a fully
configured container image, on every node allowed to run the bundle.

Pacemaker will create an implicit resource of type \sphinxstylestrong{ocf:heartbeat:docker},
\sphinxstylestrong{ocf:heartbeat:podman}, or \sphinxstylestrong{ocf:heartbeat:rkt} to manage a bundle’s
container. The user must ensure that the appropriate resource agent is
installed on every node allowed to run the bundle.

\index{XML element!bundle}\index{bundle!XML element}\ignorespaces 

\subsubsection{Bundle Properties}
\label{\detokenize{collective:bundle-properties}}\label{\detokenize{collective:index-47}}

\begin{savenotes}\sphinxattablestart
\centering
\sphinxcapstartof{table}
\sphinxcaption{\sphinxstylestrong{XML Attributes of a bundle Element}}\label{\detokenize{collective:id12}}
\sphinxaftercaption
\begin{tabular}[t]{|\X{1}{5}|\X{4}{5}|}
\hline
\sphinxstyletheadfamily 
Field
&\sphinxstyletheadfamily 
Description
\\
\hline
id
&
\index{bundle!attribute, id}\index{attribute!id (bundle)}\index{id!bundle attribute}\ignorespaces 
A unique name for the bundle (required)
\\
\hline
description
&
\index{bundle!attribute, description}\index{attribute!description (bundle)}\index{description!bundle attribute}\ignorespaces 
An optional description of the group, for the user’s own
purposes.
E.g. \sphinxcode{\sphinxupquote{manages the container that runs the service}}
\\
\hline
\end{tabular}
\par
\sphinxattableend\end{savenotes}

A bundle must contain exactly one \sphinxcode{\sphinxupquote{docker}}, \sphinxcode{\sphinxupquote{podman}}, or \sphinxcode{\sphinxupquote{rkt}} element.

\index{XML element!docker}\index{docker!XML element}\index{XML element!podman}\index{podman!XML element}\index{XML element!rkt}\index{rkt!XML element}\ignorespaces 

\subsubsection{Bundle Container Properties}
\label{\detokenize{collective:bundle-container-properties}}\label{\detokenize{collective:index-50}}

\begin{savenotes}\sphinxatlongtablestart\begin{longtable}{|\X{2}{9}|\X{3}{9}|\X{4}{9}|}
\caption{\sphinxstylestrong{XML attributes of a docker, podman, or rkt Element}\strut}\label{\detokenize{collective:id13}}\\*[\sphinxlongtablecapskipadjust]
\hline
\sphinxstyletheadfamily 
Attribute
&\sphinxstyletheadfamily 
Default
&\sphinxstyletheadfamily 
Description
\\
\hline
\endfirsthead

\multicolumn{3}{c}%
{\makebox[0pt]{\sphinxtablecontinued{\tablename\ \thetable{} -- continued from previous page}}}\\
\hline
\sphinxstyletheadfamily 
Attribute
&\sphinxstyletheadfamily 
Default
&\sphinxstyletheadfamily 
Description
\\
\hline
\endhead

\hline
\multicolumn{3}{r}{\makebox[0pt][r]{\sphinxtablecontinued{Continued on next page}}}\\
\endfoot

\endlastfoot

image
&&
\index{docker!attribute, image}\index{attribute!image (docker)}\index{image!docker attribute}\index{podman!attribute, image}\index{attribute!image (podman)}\index{image!podman attribute}\index{rkt!attribute, image}\index{attribute!image (rkt)}\index{image!rkt attribute}\ignorespaces 
Container image tag (required)
\\
\hline
replicas
&
Value of \sphinxcode{\sphinxupquote{promoted-max}}
if that is positive, else 1
&
\index{docker!attribute, replicas}\index{attribute!replicas (docker)}\index{replicas!docker attribute}\index{podman!attribute, replicas}\index{attribute!replicas (podman)}\index{replicas!podman attribute}\index{rkt!attribute, replicas}\index{attribute!replicas (rkt)}\index{replicas!rkt attribute}\ignorespaces 
A positive integer specifying the number of
container instances to launch
\\
\hline
replicas-per-host
&
1
&
\index{docker!attribute, replicas-per-host}\index{attribute!replicas-per-host (docker)}\index{replicas-per-host!docker attribute}\index{podman!attribute, replicas-per-host}\index{attribute!replicas-per-host (podman)}\index{replicas-per-host!podman attribute}\index{rkt!attribute, replicas-per-host}\index{attribute!replicas-per-host (rkt)}\index{replicas-per-host!rkt attribute}\ignorespaces 
A positive integer specifying the number of
container instances allowed to run on a
single node
\\
\hline
promoted-max
&
0
&
\index{docker!attribute, promoted-max}\index{attribute!promoted-max (docker)}\index{promoted-max!docker attribute}\index{podman!attribute, promoted-max}\index{attribute!promoted-max (podman)}\index{promoted-max!podman attribute}\index{rkt!attribute, promoted-max}\index{attribute!promoted-max (rkt)}\index{promoted-max!rkt attribute}\ignorespaces 
A non-negative integer that, if positive,
indicates that the containerized service
should be treated as a promotable service,
with this many replicas allowed to run the
service in the promoted role
\\
\hline
network
&&
\index{docker!attribute, network}\index{attribute!network (docker)}\index{network!docker attribute}\index{podman!attribute, network}\index{attribute!network (podman)}\index{network!podman attribute}\index{rkt!attribute, network}\index{attribute!network (rkt)}\index{network!rkt attribute}\ignorespaces 
If specified, this will be passed to the
\sphinxcode{\sphinxupquote{docker run}}, \sphinxcode{\sphinxupquote{podman run}}, or
\sphinxcode{\sphinxupquote{rkt run}} command as the network setting
for the container.
\\
\hline
run-command
&
\sphinxcode{\sphinxupquote{/usr/sbin/pacemaker-remoted}} if
bundle contains a \sphinxstylestrong{primitive},
otherwise none
&
\index{docker!attribute, run-command}\index{attribute!run-command (docker)}\index{run-command!docker attribute}\index{podman!attribute, run-command}\index{attribute!run-command (podman)}\index{run-command!podman attribute}\index{rkt!attribute, run-command}\index{attribute!run-command (rkt)}\index{run-command!rkt attribute}\ignorespaces 
This command will be run inside the container
when launching it (“PID 1”). If the bundle
contains a \sphinxstylestrong{primitive}, this command \sphinxstyleemphasis{must}
start \sphinxcode{\sphinxupquote{pacemaker-remoted}} (but could, for
example, be a script that does other stuff, too).
\\
\hline
options
&&
\index{docker!attribute, options}\index{attribute!options (docker)}\index{options!docker attribute}\index{podman!attribute, options}\index{attribute!options (podman)}\index{options!podman attribute}\index{rkt!attribute, options}\index{attribute!options (rkt)}\index{options!rkt attribute}\ignorespaces 
Extra command-line options to pass to the
\sphinxcode{\sphinxupquote{docker run}}, \sphinxcode{\sphinxupquote{podman run}}, or \sphinxcode{\sphinxupquote{rkt run}}
command
\\
\hline
\end{longtable}\sphinxatlongtableend\end{savenotes}

\begin{sphinxadmonition}{note}{Note:}
Considerations when using cluster configurations or container images from
Pacemaker 1.1:
\begin{itemize}
\item {} 
If the container image has a pre-2.0.0 version of Pacemaker, set \sphinxcode{\sphinxupquote{run-command}}
to \sphinxcode{\sphinxupquote{/usr/sbin/pacemaker\_remoted}} (note the underbar instead of dash).

\item {} 
\sphinxcode{\sphinxupquote{masters}} is accepted as an alias for \sphinxcode{\sphinxupquote{promoted-max}}, but is deprecated since
2.0.0, and support for it will be removed in a future version.

\end{itemize}
\end{sphinxadmonition}


\subsubsection{Bundle Network Properties}
\label{\detokenize{collective:bundle-network-properties}}
A bundle may optionally contain one \sphinxcode{\sphinxupquote{\textless{}network\textgreater{}}} element.

\index{XML element!network}\index{network!XML element}\index{bundle!network}\ignorespaces 

\begin{savenotes}\sphinxattablestart
\centering
\sphinxcapstartof{table}
\sphinxcaption{\sphinxstylestrong{XML attributes of a network Element}}\label{\detokenize{collective:id14}}\label{\detokenize{collective:index-58}}
\sphinxaftercaption
\begin{tabular}[t]{|\X{2}{8}|\X{1}{8}|\X{5}{8}|}
\hline
\sphinxstyletheadfamily 
Attribute
&\sphinxstyletheadfamily 
Default
&\sphinxstyletheadfamily 
Description
\\
\hline
add-host
&
TRUE
&
\index{network!attribute, add-host}\index{attribute!add-host (network)}\index{add-host!network attribute}\ignorespaces 
If TRUE, and \sphinxcode{\sphinxupquote{ip-range-start}} is used, Pacemaker will
automatically ensure that \sphinxcode{\sphinxupquote{/etc/hosts}} inside the
containers has entries for each
{\hyperref[\detokenize{collective:s-resource-bundle-note-replica-names}]{\sphinxcrossref{\DUrole{std,std-ref}{replica name}}}}
and its assigned IP.
\\
\hline
ip-range-start
&&
\index{network!attribute, ip-range-start}\index{attribute!ip-range-start (network)}\index{ip-range-start!network attribute}\ignorespaces 
If specified, Pacemaker will create an implicit
\sphinxcode{\sphinxupquote{ocf:heartbeat:IPaddr2}} resource for each container
instance, starting with this IP address, using up to
\sphinxcode{\sphinxupquote{replicas}} sequential addresses. These addresses can be
used from the host’s network to reach the service inside
the container, though it is not visible within the
container itself. Only IPv4 addresses are currently
supported.
\\
\hline
host-netmask
&
32
&
\index{network!attribute!host-netmask}\index{attribute!host-netmask (network)}\index{host-netmask!network attribute}\ignorespaces 
If \sphinxcode{\sphinxupquote{ip-range-start}} is specified, the IP addresses
are created with this CIDR netmask (as a number of bits).
\\
\hline
host-interface
&&
\index{network!attribute!host-interface}\index{attribute!host-interface (network)}\index{host-interface!network attribute}\ignorespaces 
If \sphinxcode{\sphinxupquote{ip-range-start}} is specified, the IP addresses are
created on this host interface (by default, it will be
determined from the IP address).
\\
\hline
control-port
&
3121
&
\index{network!attribute!control-port}\index{attribute!control-port (network)}\index{control-port!network attribute}\ignorespaces 
If the bundle contains a \sphinxcode{\sphinxupquote{primitive}}, the cluster will
use this integer TCP port for communication with
Pacemaker Remote inside the container. Changing this is
useful when the container is unable to listen on the
default port, for example, when the container uses the
host’s network rather than \sphinxcode{\sphinxupquote{ip-range-start}} (in which
case \sphinxcode{\sphinxupquote{replicas-per-host}} must be 1), or when the bundle
may run on a Pacemaker Remote node that is already
listening on the default port. Any \sphinxcode{\sphinxupquote{PCMK\_remote\_port}}
environment variable set on the host or in the container
is ignored for bundle connections.
\\
\hline
\end{tabular}
\par
\sphinxattableend\end{savenotes}
\phantomsection\label{\detokenize{collective:s-resource-bundle-note-replica-names}}
\begin{sphinxadmonition}{note}{Note:}
Replicas are named by the bundle id plus a dash and an integer counter starting
with zero. For example, if a bundle named \sphinxstylestrong{httpd-bundle} has \sphinxstylestrong{replicas=2}, its
containers will be named \sphinxstylestrong{httpd-bundle-0} and \sphinxstylestrong{httpd-bundle-1}.
\end{sphinxadmonition}

\index{XML element!port-mapping}\index{port-mapping!XML element}\ignorespaces 
Additionally, a \sphinxcode{\sphinxupquote{network}} element may optionally contain one or more
\sphinxcode{\sphinxupquote{port-mapping}} elements.


\begin{savenotes}\sphinxattablestart
\centering
\sphinxcapstartof{table}
\sphinxcaption{\sphinxstylestrong{Attributes of a port-mapping Element}}\label{\detokenize{collective:id15}}
\sphinxaftercaption
\begin{tabular}[t]{|\X{2}{8}|\X{1}{8}|\X{5}{8}|}
\hline
\sphinxstyletheadfamily 
Attribute
&\sphinxstyletheadfamily 
Default
&\sphinxstyletheadfamily 
Description
\\
\hline
id
&&
\index{port-mapping!attribute, id}\index{attribute!id (port-mapping)}\index{id!port-mapping attribute}\ignorespaces 
A unique name for the port mapping (required)
\\
\hline
port
&&
\index{port-mapping!attribute, port}\index{attribute!port (port-mapping)}\index{port!port-mapping attribute}\ignorespaces 
If this is specified, connections to this TCP port
number on the host network (on the container’s
assigned IP address, if \sphinxcode{\sphinxupquote{ip-range-start}} is
specified) will be forwarded to the container
network. Exactly one of \sphinxcode{\sphinxupquote{port}} or \sphinxcode{\sphinxupquote{range}}
must be specified in a \sphinxcode{\sphinxupquote{port-mapping}}.
\\
\hline
internal-port
&
value of \sphinxcode{\sphinxupquote{port}}
&
\index{port-mapping!attribute, internal-port}\index{attribute!internal-port (port-mapping)}\index{internal-port!port-mapping attribute}\ignorespaces 
If \sphinxcode{\sphinxupquote{port}} and this are specified, connections
to \sphinxcode{\sphinxupquote{port}} on the host’s network will be
forwarded to this port on the container network.
\\
\hline
range
&&
\index{port-mapping!attribute, range}\index{attribute!range (port-mapping)}\index{range!port-mapping attribute}\ignorespaces 
If this is specified, connections to these TCP
port numbers (expressed as \sphinxstyleemphasis{first\_port}-\sphinxstyleemphasis{last\_port})
on the host network (on the container’s assigned IP
address, if \sphinxcode{\sphinxupquote{ip-range-start}} is specified) will
be forwarded to the same ports in the container
network. Exactly one of \sphinxcode{\sphinxupquote{port}} or \sphinxcode{\sphinxupquote{range}}
must be specified in a \sphinxcode{\sphinxupquote{port-mapping}}.
\\
\hline
\end{tabular}
\par
\sphinxattableend\end{savenotes}

\begin{sphinxadmonition}{note}{Note:}
If the bundle contains a \sphinxcode{\sphinxupquote{primitive}}, Pacemaker will automatically map the
\sphinxcode{\sphinxupquote{control-port}}, so it is not necessary to specify that port in a
\sphinxcode{\sphinxupquote{port-mapping}}.
\end{sphinxadmonition}


\subsubsection{Bundle Storage Properties}
\label{\detokenize{collective:bundle-storage-properties}}\label{\detokenize{collective:s-bundle-storage}}
A bundle may optionally contain one \sphinxcode{\sphinxupquote{storage}} element. A \sphinxcode{\sphinxupquote{storage}} element
has no properties of its own, but may contain one or more \sphinxcode{\sphinxupquote{storage-mapping}}
elements.


\begin{savenotes}\sphinxattablestart
\centering
\sphinxcapstartof{table}
\sphinxcaption{\sphinxstylestrong{Attributes of a storage-mapping Element}}\label{\detokenize{collective:id16}}
\sphinxaftercaption
\begin{tabular}[t]{|\X{2}{8}|\X{1}{8}|\X{5}{8}|}
\hline
\sphinxstyletheadfamily 
Attribute
&\sphinxstyletheadfamily 
Default
&\sphinxstyletheadfamily 
Description
\\
\hline
id
&&
\index{storage-mapping!attribute, id}\index{attribute!id (storage-mapping)}\index{id!storage-mapping attribute}\ignorespaces 
A unique name for the storage mapping (required)
\\
\hline
source-dir
&&
\index{storage-mapping!attribute, source-dir}\index{attribute!source-dir (storage-mapping)}\index{source-dir!storage-mapping attribute}\ignorespaces 
The absolute path on the host’s filesystem that will be
mapped into the container. Exactly one of \sphinxcode{\sphinxupquote{source-dir}}
and \sphinxcode{\sphinxupquote{source-dir-root}} must be specified in a
\sphinxcode{\sphinxupquote{storage-mapping}}.
\\
\hline
source-dir-root
&&
\index{storage-mapping!attribute, source-dir-root}\index{attribute!source-dir-root (storage-mapping)}\index{source-dir-root!storage-mapping attribute}\ignorespaces 
The start of a path on the host’s filesystem that will
be mapped into the container, using a different
subdirectory on the host for each container instance.
The subdirectory will be named the same as the
{\hyperref[\detokenize{collective:s-resource-bundle-note-replica-names}]{\sphinxcrossref{\DUrole{std,std-ref}{replica name}}}}.
Exactly one of \sphinxcode{\sphinxupquote{source-dir}} and \sphinxcode{\sphinxupquote{source-dir-root}}
must be specified in a \sphinxcode{\sphinxupquote{storage-mapping}}.
\\
\hline
target-dir
&&
\index{storage-mapping!attribute, target-dir}\index{attribute!target-dir (storage-mapping)}\index{target-dir!storage-mapping attribute}\ignorespaces 
The path name within the container where the host
storage will be mapped (required)
\\
\hline
options
&&
\index{storage-mapping!attribute, options}\index{attribute!options (storage-mapping)}\index{options!storage-mapping attribute}\ignorespaces 
A comma-separated list of file system mount
options to use when mapping the storage
\\
\hline
\end{tabular}
\par
\sphinxattableend\end{savenotes}

\begin{sphinxadmonition}{note}{Note:}
Pacemaker does not define the behavior if the source directory does not already
exist on the host. However, it is expected that the container technology and/or
its resource agent will create the source directory in that case.
\end{sphinxadmonition}

\begin{sphinxadmonition}{note}{Note:}
If the bundle contains a \sphinxcode{\sphinxupquote{primitive}},
Pacemaker will automatically map the equivalent of
\sphinxcode{\sphinxupquote{source-dir=/etc/pacemaker/authkey target-dir=/etc/pacemaker/authkey}}
and \sphinxcode{\sphinxupquote{source-dir-root=/var/log/pacemaker/bundles target-dir=/var/log}} into the
container, so it is not necessary to specify those paths in a
\sphinxcode{\sphinxupquote{storage-mapping}}.
\end{sphinxadmonition}

\begin{sphinxadmonition}{important}{Important:}
The \sphinxcode{\sphinxupquote{PCMK\_authkey\_location}} environment variable must not be set to anything
other than the default of \sphinxcode{\sphinxupquote{/etc/pacemaker/authkey}} on any node in the cluster.
\end{sphinxadmonition}

\begin{sphinxadmonition}{important}{Important:}
If SELinux is used in enforcing mode on the host, you must ensure the container
is allowed to use any storage you mount into it. For Docker and podman bundles,
adding “Z” to the mount options will create a container-specific label for the
mount that allows the container access.
\end{sphinxadmonition}

\index{bundle!primitive}\ignorespaces 

\subsubsection{Bundle Primitive}
\label{\detokenize{collective:bundle-primitive}}\label{\detokenize{collective:index-74}}
A bundle may optionally contain one {\hyperref[\detokenize{resources:primitive-resource}]{\sphinxcrossref{\DUrole{std,std-ref}{primitive}}}}
resource. The primitive may have operations, instance attributes, and
meta-attributes defined, as usual.

If a bundle contains a primitive resource, the container image must include
the Pacemaker Remote daemon, and at least one of \sphinxcode{\sphinxupquote{ip-range-start}} or
\sphinxcode{\sphinxupquote{control-port}} must be configured in the bundle. Pacemaker will create an
implicit \sphinxstylestrong{ocf:pacemaker:remote} resource for the connection, launch
Pacemaker Remote within the container, and monitor and manage the primitive
resource via Pacemaker Remote.

If the bundle has more than one container instance (replica), the primitive
resource will function as an implicit {\hyperref[\detokenize{collective:s-resource-clone}]{\sphinxcrossref{\DUrole{std,std-ref}{clone}}}} \textendash{} a
{\hyperref[\detokenize{collective:s-resource-promotable}]{\sphinxcrossref{\DUrole{std,std-ref}{promotable clone}}}} if the bundle has \sphinxcode{\sphinxupquote{promoted-max}}
greater than zero.

\begin{sphinxadmonition}{note}{Note:}
If you want to pass environment variables to a bundle’s Pacemaker Remote
connection or primitive, you have two options:
\begin{itemize}
\item {} 
Environment variables whose value is the same regardless of the underlying host
may be set using the container element’s \sphinxcode{\sphinxupquote{options}} attribute.

\item {} 
If you want variables to have host-specific values, you can use the
{\hyperref[\detokenize{collective:s-bundle-storage}]{\sphinxcrossref{\DUrole{std,std-ref}{storage-mapping}}}} element to map a file on the host as
\sphinxcode{\sphinxupquote{/etc/pacemaker/pcmk-init.env}} in the container \sphinxstyleemphasis{(since 2.0.3)}.
Pacemaker Remote will parse this file as a shell-like format, with
variables set as NAME=VALUE, ignoring blank lines and comments starting
with “\#”.

\end{itemize}
\end{sphinxadmonition}

\begin{sphinxadmonition}{important}{Important:}
When a bundle has a \sphinxcode{\sphinxupquote{primitive}}, Pacemaker on all cluster nodes must be able to
contact Pacemaker Remote inside the bundle’s containers.
\begin{itemize}
\item {} 
The containers must have an accessible network (for example, \sphinxcode{\sphinxupquote{network}} should
not be set to “none” with a \sphinxcode{\sphinxupquote{primitive}}).

\item {} 
The default, using a distinct network space inside the container, works in
combination with \sphinxcode{\sphinxupquote{ip-range-start}}. Any firewall must allow access from all
cluster nodes to the \sphinxcode{\sphinxupquote{control-port}} on the container IPs.

\item {} 
If the container shares the host’s network space (for example, by setting
\sphinxcode{\sphinxupquote{network}} to “host”), a unique \sphinxcode{\sphinxupquote{control-port}} should be specified for each
bundle. Any firewall must allow access from all cluster nodes to the
\sphinxcode{\sphinxupquote{control-port}} on all cluster and remote node IPs.

\end{itemize}
\end{sphinxadmonition}

\index{bundle!node attributes}\ignorespaces 

\subsubsection{Bundle Node Attributes}
\label{\detokenize{collective:bundle-node-attributes}}\label{\detokenize{collective:s-bundle-attributes}}\label{\detokenize{collective:index-75}}
If the bundle has a \sphinxcode{\sphinxupquote{primitive}}, the primitive’s resource agent may want to set
node attributes such as {\hyperref[\detokenize{collective:s-promotion-scores}]{\sphinxcrossref{\DUrole{std,std-ref}{promotion scores}}}}. However, with
containers, it is not apparent which node should get the attribute.

If the container uses shared storage that is the same no matter which node the
container is hosted on, then it is appropriate to use the promotion score on the
bundle node itself.

On the other hand, if the container uses storage exported from the underlying host,
then it may be more appropriate to use the promotion score on the underlying host.

Since this depends on the particular situation, the
\sphinxcode{\sphinxupquote{container-attribute-target}} resource meta-attribute allows the user to specify
which approach to use. If it is set to \sphinxcode{\sphinxupquote{host}}, then user-defined node attributes
will be checked on the underlying host. If it is anything else, the local node
(in this case the bundle node) is used as usual.

This only applies to user-defined attributes; the cluster will always check the
local node for cluster-defined attributes such as \sphinxcode{\sphinxupquote{\#uname}}.

If \sphinxcode{\sphinxupquote{container-attribute-target}} is \sphinxcode{\sphinxupquote{host}}, the cluster will pass additional
environment variables to the primitive’s resource agent that allow it to set
node attributes appropriately: \sphinxcode{\sphinxupquote{CRM\_meta\_container\_attribute\_target}} (identical
to the meta-attribute value) and \sphinxcode{\sphinxupquote{CRM\_meta\_physical\_host}} (the name of the
underlying host).

\begin{sphinxadmonition}{note}{Note:}
When called by a resource agent, the \sphinxcode{\sphinxupquote{attrd\_updater}} and \sphinxcode{\sphinxupquote{crm\_attribute}}
commands will automatically check those environment variables and set
attributes appropriately.
\end{sphinxadmonition}

\index{bundle!meta-attributes}\ignorespaces 

\subsubsection{Bundle Meta-Attributes}
\label{\detokenize{collective:bundle-meta-attributes}}\label{\detokenize{collective:index-76}}
Any meta-attribute set on a bundle will be inherited by the bundle’s
primitive and any resources implicitly created by Pacemaker for the bundle.

This includes options such as \sphinxcode{\sphinxupquote{priority}}, \sphinxcode{\sphinxupquote{target-role}}, and \sphinxcode{\sphinxupquote{is-managed}}. See
{\hyperref[\detokenize{resources:resource-options}]{\sphinxcrossref{\DUrole{std,std-ref}{Resource Options}}}} for more information.

Bundles support clone meta-attributes including \sphinxcode{\sphinxupquote{notify}}, \sphinxcode{\sphinxupquote{ordered}}, and
\sphinxcode{\sphinxupquote{interleave}}.


\subsubsection{Limitations of Bundles}
\label{\detokenize{collective:limitations-of-bundles}}
Restarting pacemaker while a bundle is unmanaged or the cluster is in
maintenance mode may cause the bundle to fail.

Bundles may not be explicitly cloned or included in groups. This includes the
bundle’s primitive and any resources implicitly created by Pacemaker for the
bundle. (If \sphinxcode{\sphinxupquote{replicas}} is greater than 1, the bundle will behave like a clone
implicitly.)

Bundles do not have instance attributes, utilization attributes, or operations,
though a bundle’s primitive may have them.

A bundle with a primitive can run on a Pacemaker Remote node only if the bundle
uses a distinct \sphinxcode{\sphinxupquote{control-port}}.


\section{Reusing Parts of the Configuration}
\label{\detokenize{reusing-configuration:reusing-parts-of-the-configuration}}\label{\detokenize{reusing-configuration::doc}}
Pacemaker provides multiple ways to simplify the configuration XML by reusing
parts of it in multiple places.

Besides simplifying the XML, this also allows you to manipulate multiple
configuration elements with a single reference.


\subsection{Reusing Resource Definitions}
\label{\detokenize{reusing-configuration:reusing-resource-definitions}}
If you want to create lots of resources with similar configurations, defining a
\sphinxstyleemphasis{resource template} simplifies the task. Once defined, it can be referenced in
primitives or in certain types of constraints.


\subsubsection{Configuring Resources with Templates}
\label{\detokenize{reusing-configuration:configuring-resources-with-templates}}
The primitives referencing the template will inherit all meta-attributes,
instance attributes, utilization attributes and operations defined
in the template. And you can define specific attributes and operations for any
of the primitives. If any of these are defined in both the template and the
primitive, the values defined in the primitive will take precedence over the
ones defined in the template.

Hence, resource templates help to reduce the amount of configuration work.
If any changes are needed, they can be done to the template definition and
will take effect globally in all resource definitions referencing that
template.

Resource templates have a syntax similar to that of primitives.

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Resource template for a migratable Xen virtual machine}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}template} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}vm\PYGZhy{}template\PYGZdq{}} \PYG{n+na}{class=}\PYG{l+s}{\PYGZdq{}ocf\PYGZdq{}} \PYG{n+na}{provider=}\PYG{l+s}{\PYGZdq{}heartbeat\PYGZdq{}} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}Xen\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}meta\PYGZus{}attributes} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}vm\PYGZhy{}template\PYGZhy{}meta\PYGZus{}attributes\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}vm\PYGZhy{}template\PYGZhy{}meta\PYGZus{}attributes\PYGZhy{}allow\PYGZhy{}migrate\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}allow\PYGZhy{}migrate\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}true\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}/meta\PYGZus{}attributes\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}utilization} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}vm\PYGZhy{}template\PYGZhy{}utilization\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}vm\PYGZhy{}template\PYGZhy{}utilization\PYGZhy{}memory\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}memory\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}512\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}/utilization\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}operations}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}op} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}vm\PYGZhy{}template\PYGZhy{}monitor\PYGZhy{}15s\PYGZdq{}} \PYG{n+na}{interval=}\PYG{l+s}{\PYGZdq{}15s\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}monitor\PYGZdq{}} \PYG{n+na}{timeout=}\PYG{l+s}{\PYGZdq{}60s\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}op} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}vm\PYGZhy{}template\PYGZhy{}start\PYGZhy{}0\PYGZdq{}} \PYG{n+na}{interval=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}start\PYGZdq{}} \PYG{n+na}{timeout=}\PYG{l+s}{\PYGZdq{}60s\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}/operations\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/template\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

Once you define a resource template, you can use it in primitives by specifying the
\sphinxcode{\sphinxupquote{template}} property.

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Xen primitive resource using a resource template}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}primitive} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}vm1\PYGZdq{}} \PYG{n+na}{template=}\PYG{l+s}{\PYGZdq{}vm\PYGZhy{}template\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}instance\PYGZus{}attributes} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}vm1\PYGZhy{}instance\PYGZus{}attributes\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}vm1\PYGZhy{}instance\PYGZus{}attributes\PYGZhy{}name\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}name\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}vm1\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}vm1\PYGZhy{}instance\PYGZus{}attributes\PYGZhy{}xmfile\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}xmfile\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}/etc/xen/shared\PYGZhy{}vm/vm1\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}/instance\PYGZus{}attributes\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/primitive\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

In the example above, the new primitive \sphinxcode{\sphinxupquote{vm1}} will inherit everything from \sphinxcode{\sphinxupquote{vm-template}}. For
example, the equivalent of the above two examples would be:

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Equivalent Xen primitive resource not using a resource template}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}primitive} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}vm1\PYGZdq{}} \PYG{n+na}{class=}\PYG{l+s}{\PYGZdq{}ocf\PYGZdq{}} \PYG{n+na}{provider=}\PYG{l+s}{\PYGZdq{}heartbeat\PYGZdq{}} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}Xen\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}meta\PYGZus{}attributes} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}vm\PYGZhy{}template\PYGZhy{}meta\PYGZus{}attributes\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}vm\PYGZhy{}template\PYGZhy{}meta\PYGZus{}attributes\PYGZhy{}allow\PYGZhy{}migrate\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}allow\PYGZhy{}migrate\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}true\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}/meta\PYGZus{}attributes\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}utilization} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}vm\PYGZhy{}template\PYGZhy{}utilization\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}vm\PYGZhy{}template\PYGZhy{}utilization\PYGZhy{}memory\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}memory\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}512\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}/utilization\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}operations}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}op} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}vm\PYGZhy{}template\PYGZhy{}monitor\PYGZhy{}15s\PYGZdq{}} \PYG{n+na}{interval=}\PYG{l+s}{\PYGZdq{}15s\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}monitor\PYGZdq{}} \PYG{n+na}{timeout=}\PYG{l+s}{\PYGZdq{}60s\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}op} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}vm\PYGZhy{}template\PYGZhy{}start\PYGZhy{}0\PYGZdq{}} \PYG{n+na}{interval=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}start\PYGZdq{}} \PYG{n+na}{timeout=}\PYG{l+s}{\PYGZdq{}60s\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}/operations\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}instance\PYGZus{}attributes} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}vm1\PYGZhy{}instance\PYGZus{}attributes\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}vm1\PYGZhy{}instance\PYGZus{}attributes\PYGZhy{}name\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}name\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}vm1\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}vm1\PYGZhy{}instance\PYGZus{}attributes\PYGZhy{}xmfile\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}xmfile\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}/etc/xen/shared\PYGZhy{}vm/vm1\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}/instance\PYGZus{}attributes\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/primitive\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

If you want to overwrite some attributes or operations, add them to the
particular primitive’s definition.

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Xen resource overriding template values}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}primitive} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}vm2\PYGZdq{}} \PYG{n+na}{template=}\PYG{l+s}{\PYGZdq{}vm\PYGZhy{}template\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}meta\PYGZus{}attributes} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}vm2\PYGZhy{}meta\PYGZus{}attributes\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}vm2\PYGZhy{}meta\PYGZus{}attributes\PYGZhy{}allow\PYGZhy{}migrate\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}allow\PYGZhy{}migrate\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}false\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}/meta\PYGZus{}attributes\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}utilization} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}vm2\PYGZhy{}utilization\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}vm2\PYGZhy{}utilization\PYGZhy{}memory\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}memory\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}1024\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}/utilization\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}instance\PYGZus{}attributes} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}vm2\PYGZhy{}instance\PYGZus{}attributes\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}vm2\PYGZhy{}instance\PYGZus{}attributes\PYGZhy{}name\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}name\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}vm2\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}vm2\PYGZhy{}instance\PYGZus{}attributes\PYGZhy{}xmfile\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}xmfile\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}/etc/xen/shared\PYGZhy{}vm/vm2\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}/instance\PYGZus{}attributes\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}operations}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}op} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}vm2\PYGZhy{}monitor\PYGZhy{}30s\PYGZdq{}} \PYG{n+na}{interval=}\PYG{l+s}{\PYGZdq{}30s\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}monitor\PYGZdq{}} \PYG{n+na}{timeout=}\PYG{l+s}{\PYGZdq{}120s\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}op} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}vm2\PYGZhy{}stop\PYGZhy{}0\PYGZdq{}} \PYG{n+na}{interval=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}stop\PYGZdq{}} \PYG{n+na}{timeout=}\PYG{l+s}{\PYGZdq{}60s\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}/operations\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/primitive\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

In the example above, the new primitive \sphinxcode{\sphinxupquote{vm2}} has special attribute values.
Its \sphinxcode{\sphinxupquote{monitor}} operation has a longer \sphinxcode{\sphinxupquote{timeout}} and \sphinxcode{\sphinxupquote{interval}}, and
the primitive has an additional \sphinxcode{\sphinxupquote{stop}} operation.

To see the resulting definition of a resource, run:

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYGZsh{} crm\PYGZus{}resource \PYGZhy{}\PYGZhy{}query\PYGZhy{}xml \PYGZhy{}\PYGZhy{}resource vm2
\end{sphinxVerbatim}

To see the raw definition of a resource in the CIB, run:

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYGZsh{} crm\PYGZus{}resource \PYGZhy{}\PYGZhy{}query\PYGZhy{}xml\PYGZhy{}raw \PYGZhy{}\PYGZhy{}resource vm2
\end{sphinxVerbatim}


\subsubsection{Using Templates in Constraints}
\label{\detokenize{reusing-configuration:using-templates-in-constraints}}
A resource template can be referenced in the following types of constraints:
\begin{itemize}
\item {} 
\sphinxcode{\sphinxupquote{order}} constraints (see {\hyperref[\detokenize{constraints:s-resource-ordering}]{\sphinxcrossref{\DUrole{std,std-ref}{Specifying the Order in which Resources Should Start/Stop}}}})

\item {} 
\sphinxcode{\sphinxupquote{colocation}} constraints (see {\hyperref[\detokenize{constraints:s-resource-colocation}]{\sphinxcrossref{\DUrole{std,std-ref}{Placing Resources Relative to other Resources}}}})

\item {} 
\sphinxcode{\sphinxupquote{rsc\_ticket}} constraints (for multi-site clusters as described in {\hyperref[\detokenize{multi-site-clusters:ticket-constraints}]{\sphinxcrossref{\DUrole{std,std-ref}{Configuring Ticket Dependencies}}}})

\end{itemize}

Resource templates referenced in constraints stand for all primitives which are
derived from that template. This means, the constraint applies to all primitive
resources referencing the resource template. Referencing resource templates in
constraints is an alternative to resource sets and can simplify the cluster
configuration considerably.

For example, given the example templates earlier in this chapter:

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}colocation} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}vm\PYGZhy{}template\PYGZhy{}colo\PYGZhy{}base\PYGZhy{}rsc\PYGZdq{}} \PYG{n+na}{rsc=}\PYG{l+s}{\PYGZdq{}vm\PYGZhy{}template\PYGZdq{}} \PYG{n+na}{rsc\PYGZhy{}role=}\PYG{l+s}{\PYGZdq{}Started\PYGZdq{}} \PYG{n+na}{with\PYGZhy{}rsc=}\PYG{l+s}{\PYGZdq{}base\PYGZhy{}rsc\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}INFINITY\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
\end{sphinxVerbatim}

would colocate all VMs with \sphinxcode{\sphinxupquote{base-rsc}} and is the equivalent of the following constraint configuration:

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}colocation} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}vm\PYGZhy{}colo\PYGZhy{}base\PYGZhy{}rsc\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}INFINITY\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}set} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}vm\PYGZhy{}colo\PYGZhy{}base\PYGZhy{}rsc\PYGZhy{}0\PYGZdq{}} \PYG{n+na}{sequential=}\PYG{l+s}{\PYGZdq{}false\PYGZdq{}} \PYG{n+na}{role=}\PYG{l+s}{\PYGZdq{}Started\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}vm1\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}vm2\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}/resource\PYGZus{}set\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}set} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}vm\PYGZhy{}colo\PYGZhy{}base\PYGZhy{}rsc\PYGZhy{}1\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}base\PYGZhy{}rsc\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}/resource\PYGZus{}set\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/rsc\PYGZus{}colocation\PYGZgt{}}
\end{sphinxVerbatim}

\begin{sphinxadmonition}{note}{Note:}
In a colocation constraint, only one template may be referenced from either
\sphinxcode{\sphinxupquote{rsc}} or \sphinxcode{\sphinxupquote{with-rsc}}; the other reference must be a regular resource.
\end{sphinxadmonition}


\subsubsection{Using Templates in Resource Sets}
\label{\detokenize{reusing-configuration:using-templates-in-resource-sets}}
Resource templates can also be referenced in resource sets.

For example, given the example templates earlier in this section, then:

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}order} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}order1\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}INFINITY\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}set} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}order1\PYGZhy{}0\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}base\PYGZhy{}rsc\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}vm\PYGZhy{}template\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}top\PYGZhy{}rsc\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}/resource\PYGZus{}set\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/rsc\PYGZus{}order\PYGZgt{}}
\end{sphinxVerbatim}

is the equivalent of the following constraint using a sequential resource set:

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}order} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}order1\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}INFINITY\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}set} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}order1\PYGZhy{}0\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}base\PYGZhy{}rsc\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}vm1\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}vm2\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}top\PYGZhy{}rsc\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}/resource\PYGZus{}set\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/rsc\PYGZus{}order\PYGZgt{}}
\end{sphinxVerbatim}

Or, if the resources referencing the template can run in parallel, then:

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}order} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}order2\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}INFINITY\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}set} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}order2\PYGZhy{}0\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}base\PYGZhy{}rsc\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}/resource\PYGZus{}set\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}set} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}order2\PYGZhy{}1\PYGZdq{}} \PYG{n+na}{sequential=}\PYG{l+s}{\PYGZdq{}false\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}vm\PYGZhy{}template\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}/resource\PYGZus{}set\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}set} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}order2\PYGZhy{}2\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}top\PYGZhy{}rsc\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}/resource\PYGZus{}set\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/rsc\PYGZus{}order\PYGZgt{}}
\end{sphinxVerbatim}

is the equivalent of the following constraint configuration:

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}order} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}order2\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}INFINITY\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}set} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}order2\PYGZhy{}0\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}base\PYGZhy{}rsc\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}/resource\PYGZus{}set\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}set} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}order2\PYGZhy{}1\PYGZdq{}} \PYG{n+na}{sequential=}\PYG{l+s}{\PYGZdq{}false\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}vm1\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}vm2\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}/resource\PYGZus{}set\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}set} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}order2\PYGZhy{}2\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}top\PYGZhy{}rsc\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}/resource\PYGZus{}set\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/rsc\PYGZus{}order\PYGZgt{}}
\end{sphinxVerbatim}


\subsection{Reusing Rules, Options and Sets of Operations}
\label{\detokenize{reusing-configuration:reusing-rules-options-and-sets-of-operations}}\label{\detokenize{reusing-configuration:s-reusing-config-elements}}
Sometimes a number of constraints need to use the same set of rules,
and resources need to set the same options and parameters.  To
simplify this situation, you can refer to an existing object using an
\sphinxcode{\sphinxupquote{id-ref}} instead of an \sphinxcode{\sphinxupquote{id}}.

So if for one resource you have

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}location} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}WebServer\PYGZhy{}connectivity\PYGZdq{}} \PYG{n+na}{rsc=}\PYG{l+s}{\PYGZdq{}Webserver\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}rule} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}ping\PYGZhy{}prefer\PYGZhy{}rule\PYGZdq{}} \PYG{n+na}{score\PYGZhy{}attribute=}\PYG{l+s}{\PYGZdq{}pingd\PYGZdq{}} \PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}expression} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}ping\PYGZhy{}prefer\PYGZdq{}} \PYG{n+na}{attribute=}\PYG{l+s}{\PYGZdq{}pingd\PYGZdq{}} \PYG{n+na}{operation=}\PYG{l+s}{\PYGZdq{}defined\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}/rule\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/rsc\PYGZus{}location\PYGZgt{}}
\end{sphinxVerbatim}

Then instead of duplicating the rule for all your other resources, you can instead specify:

\begin{sphinxShadowBox}
\sphinxstyletopictitle{\sphinxstylestrong{Referencing rules from other constraints}}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}location} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}WebDB\PYGZhy{}connectivity\PYGZdq{}} \PYG{n+na}{rsc=}\PYG{l+s}{\PYGZdq{}WebDB\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}rule} \PYG{n+na}{id\PYGZhy{}ref=}\PYG{l+s}{\PYGZdq{}ping\PYGZhy{}prefer\PYGZhy{}rule\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/rsc\PYGZus{}location\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

\begin{sphinxadmonition}{important}{Important:}
The cluster will insist that the \sphinxcode{\sphinxupquote{rule}} exists somewhere.  Attempting
to add a reference to a non-existing rule will cause a validation
failure, as will attempting to remove a \sphinxcode{\sphinxupquote{rule}} that is referenced
elsewhere.
\end{sphinxadmonition}

The same principle applies for \sphinxcode{\sphinxupquote{meta\_attributes}} and
\sphinxcode{\sphinxupquote{instance\_attributes}} as illustrated in the example below:

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Referencing attributes, options, and operations from other resources}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}primitive} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}mySpecialRsc\PYGZdq{}} \PYG{n+na}{class=}\PYG{l+s}{\PYGZdq{}ocf\PYGZdq{}} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}Special\PYGZdq{}} \PYG{n+na}{provider=}\PYG{l+s}{\PYGZdq{}me\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}instance\PYGZus{}attributes} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}mySpecialRsc\PYGZhy{}attrs\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}1\PYGZdq{}} \PYG{n+nt}{\PYGZgt{}}
     \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}default\PYGZhy{}interface\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}interface\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}eth0\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
     \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}default\PYGZhy{}port\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}port\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}9999\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}/instance\PYGZus{}attributes\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}meta\PYGZus{}attributes} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}mySpecialRsc\PYGZhy{}options\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
     \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}failure\PYGZhy{}timeout\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}failure\PYGZhy{}timeout\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}5m\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
     \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}migration\PYGZhy{}threshold\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}migration\PYGZhy{}threshold\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}1\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
     \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}stickiness\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}resource\PYGZhy{}stickiness\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}/meta\PYGZus{}attributes\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}operations} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}health\PYGZhy{}checks\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
     \PYG{n+nt}{\PYGZlt{}op} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}health\PYGZhy{}check\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}monitor\PYGZdq{}} \PYG{n+na}{interval=}\PYG{l+s}{\PYGZdq{}60s\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
     \PYG{n+nt}{\PYGZlt{}op} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}health\PYGZhy{}check\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}monitor\PYGZdq{}} \PYG{n+na}{interval=}\PYG{l+s}{\PYGZdq{}30min\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}/operations\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/primitive\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}primitive} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}myOtherlRsc\PYGZdq{}} \PYG{n+na}{class=}\PYG{l+s}{\PYGZdq{}ocf\PYGZdq{}} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}Other\PYGZdq{}} \PYG{n+na}{provider=}\PYG{l+s}{\PYGZdq{}me\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}instance\PYGZus{}attributes} \PYG{n+na}{id\PYGZhy{}ref=}\PYG{l+s}{\PYGZdq{}mySpecialRsc\PYGZhy{}attrs\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}meta\PYGZus{}attributes} \PYG{n+na}{id\PYGZhy{}ref=}\PYG{l+s}{\PYGZdq{}mySpecialRsc\PYGZhy{}options\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}operations} \PYG{n+na}{id\PYGZhy{}ref=}\PYG{l+s}{\PYGZdq{}health\PYGZhy{}checks\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/primitive\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

\sphinxcode{\sphinxupquote{id-ref}} can similarly be used with \sphinxcode{\sphinxupquote{resource\_set}} (in any constraint type),
\sphinxcode{\sphinxupquote{nvpair}}, and \sphinxcode{\sphinxupquote{operations}}.


\subsection{Tagging Configuration Elements}
\label{\detokenize{reusing-configuration:tagging-configuration-elements}}
Pacemaker allows you to \sphinxstyleemphasis{tag} any configuration element that has an XML ID.

The main purpose of tagging is to support higher-level user interface tools;
Pacemaker itself only uses tags within constraints. Therefore, what you can
do with tags mostly depends on the tools you use.


\subsubsection{Configuring Tags}
\label{\detokenize{reusing-configuration:configuring-tags}}
A tag is simply a named list of XML IDs.

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Tag referencing three resources}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}tags}\PYG{n+nt}{\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}tag} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}all\PYGZhy{}vms\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}obj\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}vm1\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}obj\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}vm2\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}obj\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}vm3\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}/tag\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/tags\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

What you can do with this new tag depends on what your higher-level tools
support. For example, a tool might allow you to enable or disable all of
the tagged resources at once, or show the status of just the tagged
resources.

A single configuration element can be listed in any number of tags.

\begin{sphinxadmonition}{important}{Important:}
If listing nodes in a tag, you must list the node’s \sphinxcode{\sphinxupquote{id}}, not name.
\end{sphinxadmonition}


\subsubsection{Using Tags in Constraints and Resource Sets}
\label{\detokenize{reusing-configuration:using-tags-in-constraints-and-resource-sets}}
Pacemaker itself only uses tags in constraints. If you supply a tag name
instead of a resource name in any constraint, the constraint will apply to
all resources listed in that tag.

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Constraint using a tag}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}order} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}order1\PYGZdq{}} \PYG{n+na}{first=}\PYG{l+s}{\PYGZdq{}storage\PYGZdq{}} \PYG{n+na}{then=}\PYG{l+s}{\PYGZdq{}all\PYGZhy{}vms\PYGZdq{}} \PYG{n+na}{kind=}\PYG{l+s}{\PYGZdq{}Mandatory\PYGZdq{}} \PYG{n+nt}{/\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

In the example above, assuming the \sphinxcode{\sphinxupquote{all-vms}} tag is defined as in the previous
example, the constraint will behave the same as:

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Equivalent constraints without tags}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}order} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}order1\PYGZhy{}1\PYGZdq{}} \PYG{n+na}{first=}\PYG{l+s}{\PYGZdq{}storage\PYGZdq{}} \PYG{n+na}{then=}\PYG{l+s}{\PYGZdq{}vm1\PYGZdq{}} \PYG{n+na}{kind=}\PYG{l+s}{\PYGZdq{}Mandatory\PYGZdq{}} \PYG{n+nt}{/\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}order} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}order1\PYGZhy{}2\PYGZdq{}} \PYG{n+na}{first=}\PYG{l+s}{\PYGZdq{}storage\PYGZdq{}} \PYG{n+na}{then=}\PYG{l+s}{\PYGZdq{}vm2\PYGZdq{}} \PYG{n+na}{kind=}\PYG{l+s}{\PYGZdq{}Mandatory\PYGZdq{}} \PYG{n+nt}{/\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}order} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}order1\PYGZhy{}3\PYGZdq{}} \PYG{n+na}{first=}\PYG{l+s}{\PYGZdq{}storage\PYGZdq{}} \PYG{n+na}{then=}\PYG{l+s}{\PYGZdq{}vm3\PYGZdq{}} \PYG{n+na}{kind=}\PYG{l+s}{\PYGZdq{}Mandatory\PYGZdq{}} \PYG{n+nt}{/\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

A tag may be used directly in the constraint, or indirectly by being
listed in a {\hyperref[\detokenize{constraints:s-resource-sets}]{\sphinxcrossref{\DUrole{std,std-ref}{resource set}}}} used in the constraint.
When used in a resource set, an expanded tag will honor the set’s
\sphinxcode{\sphinxupquote{sequential}} property.


\subsubsection{Filtering With Tags}
\label{\detokenize{reusing-configuration:filtering-with-tags}}
The \sphinxcode{\sphinxupquote{crm\_mon}} tool can be used to display lots of information about the
state of the cluster.  On large or complicated clusters, this can include
a lot of information, which makes it difficult to find the one thing you
are interested in.  The \sphinxcode{\sphinxupquote{-{-}resource=}} and \sphinxcode{\sphinxupquote{-{-}node=}} command line
options can be used to filter results.  In their most basic usage, these
options take a single resource or node name.  However, they can also
be supplied with a tag name to display several objects at once.

For instance, given the following CIB section:

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}resources}\PYG{n+nt}{\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}primitive} \PYG{n+na}{class=}\PYG{l+s}{\PYGZdq{}stonith\PYGZdq{}} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}Fencing\PYGZdq{}} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}fence\PYGZus{}xvm\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}primitive} \PYG{n+na}{class=}\PYG{l+s}{\PYGZdq{}ocf\PYGZdq{}} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}dummy\PYGZdq{}} \PYG{n+na}{provider=}\PYG{l+s}{\PYGZdq{}pacemaker\PYGZdq{}} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}Dummy\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}group} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}inactive\PYGZhy{}group\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}primitive} \PYG{n+na}{class=}\PYG{l+s}{\PYGZdq{}ocf\PYGZdq{}} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}inactive\PYGZhy{}dummy\PYGZhy{}1\PYGZdq{}} \PYG{n+na}{provider=}\PYG{l+s}{\PYGZdq{}pacemaker\PYGZdq{}} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}Dummy\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}primitive} \PYG{n+na}{class=}\PYG{l+s}{\PYGZdq{}ocf\PYGZdq{}} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}inactive\PYGZhy{}dummy\PYGZhy{}2\PYGZdq{}} \PYG{n+na}{provider=}\PYG{l+s}{\PYGZdq{}pacemaker\PYGZdq{}} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}Dummy\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}/group\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}clone} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}inactive\PYGZhy{}clone\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}primitive} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}inactive\PYGZhy{}dhcpd\PYGZdq{}} \PYG{n+na}{class=}\PYG{l+s}{\PYGZdq{}lsb\PYGZdq{}} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}dhcpd\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}/clone\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/resources\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}tags}\PYG{n+nt}{\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}tag} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}inactive\PYGZhy{}rscs\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}obj\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}inactive\PYGZhy{}group\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}obj\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}inactive\PYGZhy{}clone\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}/tag\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/tags\PYGZgt{}}
\end{sphinxVerbatim}

The following would be output for \sphinxcode{\sphinxupquote{crm\_mon -{-}resource=inactive-rscs -r}}:

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
Cluster Summary:
  * Stack: corosync
  * Current DC: cluster02 (version 2.0.4\PYGZhy{}1.e97f9675f.git.el7\PYGZhy{}e97f9675f) \PYGZhy{} partition with quorum
  * Last updated: Tue Oct 20 16:09:01 2020
  * Last change:  Tue May  5 12:04:36 2020 by hacluster via crmd on cluster01
  * 5 nodes configured
  * 27 resource instances configured (4 DISABLED)

Node List:
  * Online: [ cluster01 cluster02 ]

Full List of Resources:
  * Clone Set: inactive\PYGZhy{}clone [inactive\PYGZhy{}dhcpd] (disabled):
    * Stopped (disabled): [ cluster01 cluster02 ]
  * Resource Group: inactive\PYGZhy{}group (disabled):
    * inactive\PYGZhy{}dummy\PYGZhy{}1  (ocf::pacemaker:Dummy):  Stopped (disabled)
    * inactive\PYGZhy{}dummy\PYGZhy{}2  (ocf::pacemaker:Dummy):  Stopped (disabled)
\end{sphinxVerbatim}


\section{Utilization and Placement Strategy}
\label{\detokenize{utilization:utilization-and-placement-strategy}}\label{\detokenize{utilization:utilization}}\label{\detokenize{utilization::doc}}
Pacemaker decides where to place a resource according to the resource
assignment scores on every node. The resource will be assigned to the
node where the resource has the highest score.

If the resource assignment scores on all the nodes are equal, by the default
placement strategy, Pacemaker will choose a node with the least number of
assigned resources for balancing the load. If the number of resources on each
node is equal, the first eligible node listed in the CIB will be chosen to run
the resource.

Often, in real-world situations, different resources use significantly
different proportions of a node’s capacities (memory, I/O, etc.).
We cannot balance the load ideally just according to the number of resources
assigned to a node. Besides, if resources are placed such that their combined
requirements exceed the provided capacity, they may fail to start completely or
run with degraded performance.

To take these factors into account, Pacemaker allows you to configure:
\begin{enumerate}
\item {} 
The capacity a certain node provides.

\item {} 
The capacity a certain resource requires.

\item {} 
An overall strategy for placement of resources.

\end{enumerate}


\subsection{Utilization attributes}
\label{\detokenize{utilization:utilization-attributes}}
To configure the capacity that a node provides or a resource requires,
you can use \sphinxstyleemphasis{utilization attributes} in \sphinxcode{\sphinxupquote{node}} and \sphinxcode{\sphinxupquote{resource}} objects.
You can name utilization attributes according to your preferences and define as
many name/value pairs as your configuration needs. However, the attributes’
values must be integers.

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Specifying CPU and RAM capacities of two nodes}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}node} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}node1\PYGZdq{}} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}normal\PYGZdq{}} \PYG{n+na}{uname=}\PYG{l+s}{\PYGZdq{}node1\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}utilization} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}node1\PYGZhy{}utilization\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}node1\PYGZhy{}utilization\PYGZhy{}cpu\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}cpu\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}2\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}node1\PYGZhy{}utilization\PYGZhy{}memory\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}memory\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}2048\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}/utilization\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/node\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}node} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}node2\PYGZdq{}} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}normal\PYGZdq{}} \PYG{n+na}{uname=}\PYG{l+s}{\PYGZdq{}node2\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}utilization} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}node2\PYGZhy{}utilization\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}node2\PYGZhy{}utilization\PYGZhy{}cpu\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}cpu\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}4\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}node2\PYGZhy{}utilization\PYGZhy{}memory\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}memory\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}4096\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}/utilization\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/node\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Specifying CPU and RAM consumed by several resources}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}primitive} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}rsc\PYGZhy{}small\PYGZdq{}} \PYG{n+na}{class=}\PYG{l+s}{\PYGZdq{}ocf\PYGZdq{}} \PYG{n+na}{provider=}\PYG{l+s}{\PYGZdq{}pacemaker\PYGZdq{}} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}Dummy\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}utilization} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}rsc\PYGZhy{}small\PYGZhy{}utilization\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}rsc\PYGZhy{}small\PYGZhy{}utilization\PYGZhy{}cpu\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}cpu\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}1\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}rsc\PYGZhy{}small\PYGZhy{}utilization\PYGZhy{}memory\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}memory\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}1024\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}/utilization\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/primitive\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}primitive} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}rsc\PYGZhy{}medium\PYGZdq{}} \PYG{n+na}{class=}\PYG{l+s}{\PYGZdq{}ocf\PYGZdq{}} \PYG{n+na}{provider=}\PYG{l+s}{\PYGZdq{}pacemaker\PYGZdq{}} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}Dummy\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}utilization} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}rsc\PYGZhy{}medium\PYGZhy{}utilization\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}rsc\PYGZhy{}medium\PYGZhy{}utilization\PYGZhy{}cpu\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}cpu\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}2\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}rsc\PYGZhy{}medium\PYGZhy{}utilization\PYGZhy{}memory\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}memory\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}2048\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}/utilization\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/primitive\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}primitive} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}rsc\PYGZhy{}large\PYGZdq{}} \PYG{n+na}{class=}\PYG{l+s}{\PYGZdq{}ocf\PYGZdq{}} \PYG{n+na}{provider=}\PYG{l+s}{\PYGZdq{}pacemaker\PYGZdq{}} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}Dummy\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}utilization} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}rsc\PYGZhy{}large\PYGZhy{}utilization\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}rsc\PYGZhy{}large\PYGZhy{}utilization\PYGZhy{}cpu\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}cpu\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}3\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}rsc\PYGZhy{}large\PYGZhy{}utilization\PYGZhy{}memory\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}memory\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}3072\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}/utilization\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/primitive\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

A node is considered eligible for a resource if it has sufficient free
capacity to satisfy the resource’s requirements. The nature of the required
or provided capacities is completely irrelevant to Pacemaker \textendash{} it just makes
sure that all capacity requirements of a resource are satisfied before placing
a resource to a node.

Utilization attributes used on a node object can also be \sphinxstyleemphasis{transient} \sphinxstyleemphasis{(since 2.1.6)}.
These attributes are added to a \sphinxcode{\sphinxupquote{transient\_attributes}} section for the node
and are forgotten by the cluster when the node goes offline.  The \sphinxcode{\sphinxupquote{attrd\_updater}}
tool can be used to set these attributes.

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Transient utilization attribute for node cluster-1}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}transient\PYGZus{}attributes} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}cluster\PYGZhy{}1\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}utilization} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}status\PYGZhy{}cluster\PYGZhy{}1\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}status\PYGZhy{}cluster\PYGZhy{}1\PYGZhy{}cpu\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}cpu\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}1\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}/utilization\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/transient\PYGZus{}attributes\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

\begin{sphinxadmonition}{note}{Note:}
Utilization is supported for bundles \sphinxstyleemphasis{(since 2.1.3)}, but only for bundles
with an inner primitive. Any resource utilization values should be specified
for the inner primitive, but any priority meta-attribute should be specified
for the outer bundle.
\end{sphinxadmonition}


\subsection{Placement Strategy}
\label{\detokenize{utilization:placement-strategy}}
After you have configured the capacities your nodes provide and the
capacities your resources require, you need to set the \sphinxcode{\sphinxupquote{placement-strategy}}
in the global cluster options, otherwise the capacity configurations have
\sphinxstyleemphasis{no effect}.

Four values are available for the \sphinxcode{\sphinxupquote{placement-strategy}}:
\begin{itemize}
\item {} 
\sphinxstylestrong{default}
\begin{quote}

Utilization values are not taken into account at all.
Resources are assigned according to assignment scores. If scores are equal,
resources are evenly distributed across nodes.
\end{quote}

\item {} 
\sphinxstylestrong{utilization}
\begin{quote}

Utilization values are taken into account \sphinxstyleemphasis{only} when deciding whether a node
is considered eligible (i.e. whether it has sufficient free capacity to satisfy
the resource’s requirements). Load-balancing is still done based on the
number of resources assigned to a node.
\end{quote}

\item {} 
\sphinxstylestrong{balanced}
\begin{quote}

Utilization values are taken into account when deciding whether a node
is eligible to serve a resource \sphinxstyleemphasis{and} when load-balancing, so an attempt is
made to spread the resources in a way that optimizes resource performance.
\end{quote}

\item {} 
\sphinxstylestrong{minimal}
\begin{quote}

Utilization values are taken into account \sphinxstyleemphasis{only} when deciding whether a node
is eligible to serve a resource. For load-balancing, an attempt is made to
concentrate the resources on as few nodes as possible, thereby enabling
possible power savings on the remaining nodes.
\end{quote}

\end{itemize}

Set \sphinxcode{\sphinxupquote{placement-strategy}} with \sphinxcode{\sphinxupquote{crm\_attribute}}:
\begin{quote}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYGZsh{} crm\PYGZus{}attribute \PYGZhy{}\PYGZhy{}name placement\PYGZhy{}strategy \PYGZhy{}\PYGZhy{}update balanced
\end{sphinxVerbatim}
\end{quote}

Now Pacemaker will ensure the load from your resources will be distributed
evenly throughout the cluster, without the need for convoluted sets of
colocation constraints.


\subsection{Assignment Details}
\label{\detokenize{utilization:assignment-details}}

\subsubsection{Which node is preferred to get consumed first when assigning resources?}
\label{\detokenize{utilization:which-node-is-preferred-to-get-consumed-first-when-assigning-resources}}\begin{itemize}
\item {} 
The node with the highest node weight gets consumed first. Node weight
is a score maintained by the cluster to represent node health.

\item {} 
If multiple nodes have the same node weight:

\end{itemize}
\begin{itemize}
\item {} 
If \sphinxcode{\sphinxupquote{placement-strategy}} is \sphinxcode{\sphinxupquote{default}} or \sphinxcode{\sphinxupquote{utilization}},
the node that has the least number of assigned resources gets consumed first.
\begin{itemize}
\item {} 
If their numbers of assigned resources are equal,
the first eligible node listed in the CIB gets consumed first.

\end{itemize}

\item {} 
If \sphinxcode{\sphinxupquote{placement-strategy}} is \sphinxcode{\sphinxupquote{balanced}},
the node that has the most free capacity gets consumed first.
\begin{itemize}
\item {} 
If the free capacities of the nodes are equal,
the node that has the least number of assigned resources gets consumed first.
\begin{itemize}
\item {} 
If their numbers of assigned resources are equal,
the first eligible node listed in the CIB gets consumed first.

\end{itemize}

\end{itemize}

\item {} 
If \sphinxcode{\sphinxupquote{placement-strategy}} is \sphinxcode{\sphinxupquote{minimal}},
the first eligible node listed in the CIB gets consumed first.

\end{itemize}


\subsubsection{Which node has more free capacity?}
\label{\detokenize{utilization:which-node-has-more-free-capacity}}
If only one type of utilization attribute has been defined, free capacity
is a simple numeric comparison.

If multiple types of utilization attributes have been defined, then
the node that is numerically highest in the the most attribute types
has the most free capacity. For example:
\begin{itemize}
\item {} 
If \sphinxcode{\sphinxupquote{nodeA}} has more free \sphinxcode{\sphinxupquote{cpus}}, and \sphinxcode{\sphinxupquote{nodeB}} has more free \sphinxcode{\sphinxupquote{memory}},
then their free capacities are equal.

\item {} 
If \sphinxcode{\sphinxupquote{nodeA}} has more free \sphinxcode{\sphinxupquote{cpus}}, while \sphinxcode{\sphinxupquote{nodeB}} has more free \sphinxcode{\sphinxupquote{memory}}
and \sphinxcode{\sphinxupquote{storage}}, then \sphinxcode{\sphinxupquote{nodeB}} has more free capacity.

\end{itemize}


\subsubsection{Which resource is preferred to be assigned first?}
\label{\detokenize{utilization:which-resource-is-preferred-to-be-assigned-first}}\begin{itemize}
\item {} 
The resource that has the highest \sphinxcode{\sphinxupquote{priority}} (see {\hyperref[\detokenize{resources:resource-options}]{\sphinxcrossref{\DUrole{std,std-ref}{Resource Options}}}}) gets
assigned first.

\item {} 
If their priorities are equal, check whether they are already running. The
resource that has the highest score on the node where it’s running gets assigned
first, to prevent resource shuffling.

\item {} 
If the scores above are equal or the resources are not running, the resource has
the highest score on the preferred node gets assigned first.

\item {} 
If the scores above are equal, the first runnable resource listed in the CIB
gets assigned first.

\end{itemize}


\subsection{Limitations and Workarounds}
\label{\detokenize{utilization:limitations-and-workarounds}}
The type of problem Pacemaker is dealing with here is known as the
\sphinxhref{http://en.wikipedia.org/wiki/Knapsack\_problem}{knapsack problem} and falls into
the \sphinxhref{http://en.wikipedia.org/wiki/NP-complete}{NP-complete} category of computer
science problems \textendash{} a fancy way of saying “it takes a really long time
to solve”.

Clearly in a HA cluster, it’s not acceptable to spend minutes, let alone hours
or days, finding an optimal solution while services remain unavailable.

So instead of trying to solve the problem completely, Pacemaker uses a
\sphinxstyleemphasis{best effort} algorithm for determining which node should host a particular
service. This means it arrives at a solution much faster than traditional
linear programming algorithms, but by doing so at the price of leaving some
services stopped.

In the contrived example at the start of this chapter:
\begin{itemize}
\item {} 
\sphinxcode{\sphinxupquote{rsc-small}} would be assigned to \sphinxcode{\sphinxupquote{node1}}

\item {} 
\sphinxcode{\sphinxupquote{rsc-medium}} would be assigned to \sphinxcode{\sphinxupquote{node2}}

\item {} 
\sphinxcode{\sphinxupquote{rsc-large}} would remain inactive

\end{itemize}

Which is not ideal.

There are various approaches to dealing with the limitations of
pacemaker’s placement strategy:
\begin{itemize}
\item {} 
\sphinxstylestrong{Ensure you have sufficient physical capacity.}
\begin{quote}

It might sound obvious, but if the physical capacity of your nodes is (close to)
maxed out by the cluster under normal conditions, then failover isn’t going to
go well. Even without the utilization feature, you’ll start hitting timeouts and
getting secondary failures.
\end{quote}

\item {} 
\sphinxstylestrong{Build some buffer into the capabilities advertised by the nodes.}
\begin{quote}

Advertise slightly more resources than we physically have, on the (usually valid)
assumption that a resource will not use 100\% of the configured amount of
CPU, memory and so forth \sphinxstyleemphasis{all} the time. This practice is sometimes called \sphinxstyleemphasis{overcommit}.
\end{quote}

\item {} 
\sphinxstylestrong{Specify resource priorities.}
\begin{quote}

If the cluster is going to sacrifice services, it should be the ones you care
about (comparatively) the least. Ensure that resource priorities are properly set
so that your most important resources are scheduled first.
\end{quote}

\end{itemize}

\index{Access Control List (ACL)}\ignorespaces 

\section{Access Control Lists (ACLs)}
\label{\detokenize{acls:access-control-lists-acls}}\label{\detokenize{acls:acl}}\label{\detokenize{acls:index-0}}\label{\detokenize{acls::doc}}
By default, the \sphinxcode{\sphinxupquote{root}} user or any user in the \sphinxcode{\sphinxupquote{haclient}} group can
modify Pacemaker’s CIB without restriction. Pacemaker offers \sphinxstyleemphasis{access control
lists (ACLs)} to provide more fine-grained authorization.

\begin{sphinxadmonition}{important}{Important:}
Being able to modify the CIB’s resource section allows a user to run any
executable file as root, by configuring it as an LSB resource with a full
path.
\end{sphinxadmonition}


\subsection{ACL Prerequisites}
\label{\detokenize{acls:acl-prerequisites}}
In order to use ACLs:
\begin{itemize}
\item {} 
The \sphinxcode{\sphinxupquote{enable-acl}} {\hyperref[\detokenize{cluster-options:cluster-options}]{\sphinxcrossref{\DUrole{std,std-ref}{cluster option}}}} must be set to
true.

\item {} 
Desired users must have user accounts in the \sphinxcode{\sphinxupquote{haclient}} group on all
cluster nodes in the cluster.

\item {} 
If your CIB was created before Pacemaker 1.1.12, it might need to be updated
to the current schema (using \sphinxcode{\sphinxupquote{cibadmin -{-}upgrade}} or a higher-level tool
equivalent) in order to use the syntax documented here.

\item {} 
Prior to the 2.1.0 release, the Pacemaker software had to have been built
with ACL support. If you are using an older release, your installation
supports ACLs only if the output of the command \sphinxcode{\sphinxupquote{pacemakerd -{-}features}}
contains \sphinxcode{\sphinxupquote{acls}}. In newer versions, ACLs are always enabled.

\end{itemize}

\index{Access Control List (ACL)!acls}\index{acls!XML element}\index{XML element!acls}\ignorespaces 

\subsection{ACL Configuration}
\label{\detokenize{acls:acl-configuration}}\label{\detokenize{acls:index-1}}
ACLs are specified within an \sphinxcode{\sphinxupquote{acls}} element of the CIB. The \sphinxcode{\sphinxupquote{acls}} element
may contain any number of \sphinxcode{\sphinxupquote{acl\_role}}, \sphinxcode{\sphinxupquote{acl\_target}}, and \sphinxcode{\sphinxupquote{acl\_group}}
elements.

\index{Access Control List (ACL)!acl\_role}\index{acl\_role!XML element}\index{XML element!acl\_role}\ignorespaces 

\subsection{ACL Roles}
\label{\detokenize{acls:acl-roles}}\label{\detokenize{acls:index-2}}
An ACL \sphinxstyleemphasis{role} is a collection of permissions allowing or denying access to
particular portions of the CIB. A role is configured with an \sphinxcode{\sphinxupquote{acl\_role}}
element in the CIB \sphinxcode{\sphinxupquote{acls}} section.


\begin{savenotes}\sphinxattablestart
\centering
\sphinxcapstartof{table}
\sphinxcaption{\sphinxstylestrong{Properties of an acl\_role element}}\label{\detokenize{acls:id1}}
\sphinxaftercaption
\begin{tabular}[t]{|\X{1}{4}|\X{3}{4}|}
\hline
\sphinxstyletheadfamily 
Attribute
&\sphinxstyletheadfamily 
Description
\\
\hline
id
&
\index{acl\_role!id (attribute)}\index{id!acl\_role attribute}\index{attribute!id (acl\_role)}\ignorespaces 
A unique name for the role \sphinxstyleemphasis{(required)}
\\
\hline
description
&
\index{acl\_role!description (attribute)}\index{description!acl\_role attribute}\index{attribute!description (acl\_role)}\ignorespaces 
Arbitrary text (not used by Pacemaker)
\\
\hline
\end{tabular}
\par
\sphinxattableend\end{savenotes}

An \sphinxcode{\sphinxupquote{acl\_role}} element may contain any number of \sphinxcode{\sphinxupquote{acl\_permission}} elements.

\index{Access Control List (ACL)!acl\_permission}\index{acl\_permission!XML element}\index{XML element!acl\_permission}\ignorespaces 

\begin{savenotes}\sphinxattablestart
\centering
\sphinxcapstartof{table}
\sphinxcaption{\sphinxstylestrong{Properties of an acl\_permission element}}\label{\detokenize{acls:id2}}\label{\detokenize{acls:index-5}}
\sphinxaftercaption
\begin{tabular}[t]{|\X{1}{4}|\X{3}{4}|}
\hline
\sphinxstyletheadfamily 
Attribute
&\sphinxstyletheadfamily 
Description
\\
\hline
id
&
\index{acl\_permission!id (attribute)}\index{id!acl\_permission attribute}\index{attribute!id (acl\_permission)}\ignorespaces 
A unique name for the permission \sphinxstyleemphasis{(required)}
\\
\hline
description
&
\index{acl\_permission!description (attribute)}\index{description!acl\_permission attribute}\index{attribute!description (acl\_permission)}\ignorespaces 
Arbitrary text (not used by Pacemaker)
\\
\hline
kind
&
\index{acl\_permission!kind (attribute)}\index{kind!acl\_permission attribute}\index{attribute!kind (acl\_permission)}\ignorespaces 
The access being granted. Allowed values are \sphinxcode{\sphinxupquote{read}},
\sphinxcode{\sphinxupquote{write}}, and \sphinxcode{\sphinxupquote{deny}}. A value of \sphinxcode{\sphinxupquote{write}} grants both
read and write access.
\\
\hline
object-type
&
\index{acl\_permission!object-type (attribute)}\index{object-type!acl\_permission attribute}\index{attribute!object-type (acl\_permission)}\ignorespaces 
The name of an XML element in the CIB to which the
permission applies. (Exactly one of \sphinxcode{\sphinxupquote{object-type}},
\sphinxcode{\sphinxupquote{xpath}}, and \sphinxcode{\sphinxupquote{reference}} must be specified for a
permission.)
\\
\hline
attribute
&
\index{acl\_permission!attribute (attribute)}\index{attribute!acl\_permission attribute}\index{attribute!attribute (acl\_permission)}\ignorespaces 
If specified, the permission applies only to
\sphinxcode{\sphinxupquote{object-type}} elements that have this attribute set (to
any value). If not specified, the permission applies to
all \sphinxcode{\sphinxupquote{object-type}} elements. May only be used with
\sphinxcode{\sphinxupquote{object-type}}.
\\
\hline
reference
&
\index{acl\_permission!reference (attribute)}\index{reference!acl\_permission attribute}\index{attribute!reference (acl\_permission)}\ignorespaces 
The ID of an XML element in the CIB to which the
permission applies. (Exactly one of \sphinxcode{\sphinxupquote{object-type}},
\sphinxcode{\sphinxupquote{xpath}}, and \sphinxcode{\sphinxupquote{reference}} must be specified for a
permission.)
\\
\hline
xpath
&
\index{acl\_permission!xpath (attribute)}\index{xpath!acl\_permission attribute}\index{attribute!xpath (acl\_permission)}\ignorespaces 
An \sphinxhref{https://www.w3.org/TR/xpath-10/}{XPath}
specification selecting an XML element in the CIB to
which the permission applies. Attributes may be specified
in the XPath to select particular elements, but the
permissions apply to the entire element. (Exactly one of
\sphinxcode{\sphinxupquote{object-type}}, \sphinxcode{\sphinxupquote{xpath}}, and \sphinxcode{\sphinxupquote{reference}} must be
specified for a permission.)
\\
\hline
\end{tabular}
\par
\sphinxattableend\end{savenotes}

\begin{sphinxadmonition}{important}{Important:}\begin{itemize}
\item {} 
Permissions are applied to the selected XML element’s entire XML subtree
(all elements enclosed within it).

\item {} 
Write permission grants the ability to create, modify, or remove the
element and its subtree, and also the ability to create any “scaffolding”
elements (enclosing elements that do not have attributes other than an
ID).

\item {} 
Permissions for more specific matches (more deeply nested elements) take
precedence over more general ones.

\item {} 
If multiple permissions are configured for the same match (for example, in
different roles applied to the same user), any \sphinxcode{\sphinxupquote{deny}} permission takes
precedence, then \sphinxcode{\sphinxupquote{write}}, then lastly \sphinxcode{\sphinxupquote{read}}.

\end{itemize}
\end{sphinxadmonition}


\subsection{ACL Targets and Groups}
\label{\detokenize{acls:acl-targets-and-groups}}
ACL targets correspond to user accounts on the system.

\index{Access Control List (ACL)!acl\_target}\index{acl\_target!XML element}\index{XML element!acl\_target}\ignorespaces 

\begin{savenotes}\sphinxattablestart
\centering
\sphinxcapstartof{table}
\sphinxcaption{\sphinxstylestrong{Properties of an acl\_target element}}\label{\detokenize{acls:id3}}\label{\detokenize{acls:index-13}}
\sphinxaftercaption
\begin{tabular}[t]{|\X{1}{4}|\X{3}{4}|}
\hline
\sphinxstyletheadfamily 
Attribute
&\sphinxstyletheadfamily 
Description
\\
\hline
id
&
\index{acl\_target!id (attribute)}\index{id!acl\_target attribute}\index{attribute!id (acl\_target)}\ignorespaces 
A unique identifier for the target (if \sphinxcode{\sphinxupquote{name}} is not
specified, this must be the name of the user account)
\sphinxstyleemphasis{(required)}
\\
\hline
name
&
\index{acl\_target!name (attribute)}\index{name!acl\_target attribute}\index{attribute!name (acl\_target)}\ignorespaces 
If specified, the user account name (this allows you to
specify a user name that is already used as the \sphinxcode{\sphinxupquote{id}}
for some other configuration element) \sphinxstyleemphasis{(since 2.1.5)}
\\
\hline
\end{tabular}
\par
\sphinxattableend\end{savenotes}

ACL groups correspond to groups on the system. Any role configured for these
groups apply to all users in that group \sphinxstyleemphasis{(since 2.1.5)}.

\index{Access Control List (ACL)!acl\_group}\index{acl\_group!XML element}\index{XML element!acl\_group}\ignorespaces 

\begin{savenotes}\sphinxattablestart
\centering
\sphinxcapstartof{table}
\sphinxcaption{\sphinxstylestrong{Properties of an acl\_group element}}\label{\detokenize{acls:id4}}\label{\detokenize{acls:index-16}}
\sphinxaftercaption
\begin{tabular}[t]{|\X{1}{4}|\X{3}{4}|}
\hline
\sphinxstyletheadfamily 
Attribute
&\sphinxstyletheadfamily 
Description
\\
\hline
id
&
\index{acl\_group!id (attribute)}\index{id!acl\_group attribute}\index{attribute!id (acl\_group)}\ignorespaces 
A unique identifier for the group (if \sphinxcode{\sphinxupquote{name}} is not
specified, this must be the group name) \sphinxstyleemphasis{(required)}
\\
\hline
name
&
\index{acl\_group!name (attribute)}\index{name!acl\_group attribute}\index{attribute!name (acl\_group)}\ignorespaces 
If specified, the group name (this allows you to specify
a group name that is already used as the \sphinxcode{\sphinxupquote{id}} for some
other configuration element)
\\
\hline
\end{tabular}
\par
\sphinxattableend\end{savenotes}

Each \sphinxcode{\sphinxupquote{acl\_target}} and \sphinxcode{\sphinxupquote{acl\_group}} element may contain any number of \sphinxcode{\sphinxupquote{role}}
elements.

\begin{sphinxadmonition}{note}{Note:}
If the system users and groups are defined by some network service (such as
LDAP), the cluster itself will be unaffected by outages in the service, but
affected users and groups will not be able to make changes to the CIB.
\end{sphinxadmonition}

\index{Access Control List (ACL)!role}\index{role!XML element}\index{XML element!role}\ignorespaces 

\begin{savenotes}\sphinxattablestart
\centering
\sphinxcapstartof{table}
\sphinxcaption{\sphinxstylestrong{Properties of a role element}}\label{\detokenize{acls:id5}}\label{\detokenize{acls:index-19}}
\sphinxaftercaption
\begin{tabular}[t]{|\X{1}{4}|\X{3}{4}|}
\hline
\sphinxstyletheadfamily 
Attribute
&\sphinxstyletheadfamily 
Description
\\
\hline
id
&
\index{role!id (attribute)}\index{id!role attribute}\index{attribute!id (role)}\ignorespaces 
The \sphinxcode{\sphinxupquote{id}} of an \sphinxcode{\sphinxupquote{acl\_role}} element that specifies
permissions granted to the enclosing target or group.
\\
\hline
\end{tabular}
\par
\sphinxattableend\end{savenotes}

\begin{sphinxadmonition}{important}{Important:}
The \sphinxcode{\sphinxupquote{root}} and \sphinxcode{\sphinxupquote{hacluster}} user accounts always have full access to
the CIB, regardless of ACLs. For all other user accounts, when \sphinxcode{\sphinxupquote{enable-acl}}
is true, permission to all parts of the CIB is denied by default (permissions
must be explicitly granted).
\end{sphinxadmonition}


\subsection{ACL Examples}
\label{\detokenize{acls:acl-examples}}
\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}acls}\PYG{n+nt}{\PYGZgt{}}

   \PYG{n+nt}{\PYGZlt{}acl\PYGZus{}role} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}read\PYGZus{}all\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
       \PYG{n+nt}{\PYGZlt{}acl\PYGZus{}permission} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}read\PYGZus{}all\PYGZhy{}cib\PYGZdq{}} \PYG{n+na}{kind=}\PYG{l+s}{\PYGZdq{}read\PYGZdq{}} \PYG{n+na}{xpath=}\PYG{l+s}{\PYGZdq{}/cib\PYGZdq{}} \PYG{n+nt}{/\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}/acl\PYGZus{}role\PYGZgt{}}

   \PYG{n+nt}{\PYGZlt{}acl\PYGZus{}role} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}operator\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}

       \PYG{n+nt}{\PYGZlt{}acl\PYGZus{}permission} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}operator\PYGZhy{}maintenance\PYGZhy{}mode\PYGZdq{}} \PYG{n+na}{kind=}\PYG{l+s}{\PYGZdq{}write\PYGZdq{}}
           \PYG{n+na}{xpath=}\PYG{l+s}{\PYGZdq{}//crm\PYGZus{}config//nvpair[@name=\PYGZsq{}maintenance\PYGZhy{}mode\PYGZsq{}]\PYGZdq{}} \PYG{n+nt}{/\PYGZgt{}}

       \PYG{n+nt}{\PYGZlt{}acl\PYGZus{}permission} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}operator\PYGZhy{}maintenance\PYGZhy{}attr\PYGZdq{}} \PYG{n+na}{kind=}\PYG{l+s}{\PYGZdq{}write\PYGZdq{}}
           \PYG{n+na}{xpath=}\PYG{l+s}{\PYGZdq{}//nvpair[@name=\PYGZsq{}maintenance\PYGZsq{}]\PYGZdq{}} \PYG{n+nt}{/\PYGZgt{}}

       \PYG{n+nt}{\PYGZlt{}acl\PYGZus{}permission} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}operator\PYGZhy{}target\PYGZhy{}role\PYGZdq{}} \PYG{n+na}{kind=}\PYG{l+s}{\PYGZdq{}write\PYGZdq{}}
           \PYG{n+na}{xpath=}\PYG{l+s}{\PYGZdq{}//resources//meta\PYGZus{}attributes/nvpair[@name=\PYGZsq{}target\PYGZhy{}role\PYGZsq{}]\PYGZdq{}} \PYG{n+nt}{/\PYGZgt{}}

       \PYG{n+nt}{\PYGZlt{}acl\PYGZus{}permission} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}operator\PYGZhy{}is\PYGZhy{}managed\PYGZdq{}} \PYG{n+na}{kind=}\PYG{l+s}{\PYGZdq{}write\PYGZdq{}}
           \PYG{n+na}{xpath=}\PYG{l+s}{\PYGZdq{}//resources//nvpair[@name=\PYGZsq{}is\PYGZhy{}managed\PYGZsq{}]\PYGZdq{}} \PYG{n+nt}{/\PYGZgt{}}

       \PYG{n+nt}{\PYGZlt{}acl\PYGZus{}permission} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}operator\PYGZhy{}rsc\PYGZus{}location\PYGZdq{}} \PYG{n+na}{kind=}\PYG{l+s}{\PYGZdq{}write\PYGZdq{}}
           \PYG{n+na}{object\PYGZhy{}type=}\PYG{l+s}{\PYGZdq{}rsc\PYGZus{}location\PYGZdq{}} \PYG{n+nt}{/\PYGZgt{}}

   \PYG{n+nt}{\PYGZlt{}/acl\PYGZus{}role\PYGZgt{}}

   \PYG{n+nt}{\PYGZlt{}acl\PYGZus{}role} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}administrator\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
       \PYG{n+nt}{\PYGZlt{}acl\PYGZus{}permission} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}administrator\PYGZhy{}cib\PYGZdq{}} \PYG{n+na}{kind=}\PYG{l+s}{\PYGZdq{}write\PYGZdq{}} \PYG{n+na}{xpath=}\PYG{l+s}{\PYGZdq{}/cib\PYGZdq{}} \PYG{n+nt}{/\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}/acl\PYGZus{}role\PYGZgt{}}

   \PYG{n+nt}{\PYGZlt{}acl\PYGZus{}role} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}minimal\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}

       \PYG{n+nt}{\PYGZlt{}acl\PYGZus{}permission} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}minimal\PYGZhy{}standby\PYGZdq{}} \PYG{n+na}{kind=}\PYG{l+s}{\PYGZdq{}read\PYGZdq{}}
           \PYG{n+na}{description=}\PYG{l+s}{\PYGZdq{}allow reading standby node attribute (permanent or transient)\PYGZdq{}}
           \PYG{n+na}{xpath=}\PYG{l+s}{\PYGZdq{}//instance\PYGZus{}attributes/nvpair[@name=\PYGZsq{}standby\PYGZsq{}]\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}

       \PYG{n+nt}{\PYGZlt{}acl\PYGZus{}permission} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}minimal\PYGZhy{}maintenance\PYGZdq{}} \PYG{n+na}{kind=}\PYG{l+s}{\PYGZdq{}read\PYGZdq{}}
           \PYG{n+na}{description=}\PYG{l+s}{\PYGZdq{}allow reading maintenance node attribute (permanent or transient)\PYGZdq{}}
           \PYG{n+na}{xpath=}\PYG{l+s}{\PYGZdq{}//nvpair[@name=\PYGZsq{}maintenance\PYGZsq{}]\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}

       \PYG{n+nt}{\PYGZlt{}acl\PYGZus{}permission} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}minimal\PYGZhy{}target\PYGZhy{}role\PYGZdq{}} \PYG{n+na}{kind=}\PYG{l+s}{\PYGZdq{}read\PYGZdq{}}
           \PYG{n+na}{description=}\PYG{l+s}{\PYGZdq{}allow reading resource target roles\PYGZdq{}}
           \PYG{n+na}{xpath=}\PYG{l+s}{\PYGZdq{}//resources//meta\PYGZus{}attributes/nvpair[@name=\PYGZsq{}target\PYGZhy{}role\PYGZsq{}]\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}

       \PYG{n+nt}{\PYGZlt{}acl\PYGZus{}permission} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}minimal\PYGZhy{}is\PYGZhy{}managed\PYGZdq{}} \PYG{n+na}{kind=}\PYG{l+s}{\PYGZdq{}read\PYGZdq{}}
           \PYG{n+na}{description=}\PYG{l+s}{\PYGZdq{}allow reading resource managed status\PYGZdq{}}
           \PYG{n+na}{xpath=}\PYG{l+s}{\PYGZdq{}//resources//meta\PYGZus{}attributes/nvpair[@name=\PYGZsq{}is\PYGZhy{}managed\PYGZsq{}]\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}

       \PYG{n+nt}{\PYGZlt{}acl\PYGZus{}permission} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}minimal\PYGZhy{}deny\PYGZhy{}instance\PYGZhy{}attributes\PYGZdq{}} \PYG{n+na}{kind=}\PYG{l+s}{\PYGZdq{}deny\PYGZdq{}}
           \PYG{n+na}{xpath=}\PYG{l+s}{\PYGZdq{}//instance\PYGZus{}attributes\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}

       \PYG{n+nt}{\PYGZlt{}acl\PYGZus{}permission} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}minimal\PYGZhy{}deny\PYGZhy{}meta\PYGZhy{}attributes\PYGZdq{}} \PYG{n+na}{kind=}\PYG{l+s}{\PYGZdq{}deny\PYGZdq{}}
           \PYG{n+na}{xpath=}\PYG{l+s}{\PYGZdq{}//meta\PYGZus{}attributes\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}

       \PYG{n+nt}{\PYGZlt{}acl\PYGZus{}permission} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}minimal\PYGZhy{}deny\PYGZhy{}operations\PYGZdq{}} \PYG{n+na}{kind=}\PYG{l+s}{\PYGZdq{}deny\PYGZdq{}}
           \PYG{n+na}{xpath=}\PYG{l+s}{\PYGZdq{}//operations\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}

       \PYG{n+nt}{\PYGZlt{}acl\PYGZus{}permission} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}minimal\PYGZhy{}deny\PYGZhy{}utilization\PYGZdq{}} \PYG{n+na}{kind=}\PYG{l+s}{\PYGZdq{}deny\PYGZdq{}}
           \PYG{n+na}{xpath=}\PYG{l+s}{\PYGZdq{}//utilization\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}

       \PYG{n+nt}{\PYGZlt{}acl\PYGZus{}permission} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}minimal\PYGZhy{}nodes\PYGZdq{}} \PYG{n+na}{kind=}\PYG{l+s}{\PYGZdq{}read\PYGZdq{}}
           \PYG{n+na}{description=}\PYG{l+s}{\PYGZdq{}allow reading node names/IDs (attributes are denied separately)\PYGZdq{}}
           \PYG{n+na}{xpath=}\PYG{l+s}{\PYGZdq{}/cib/configuration/nodes\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}

       \PYG{n+nt}{\PYGZlt{}acl\PYGZus{}permission} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}minimal\PYGZhy{}resources\PYGZdq{}} \PYG{n+na}{kind=}\PYG{l+s}{\PYGZdq{}read\PYGZdq{}}
           \PYG{n+na}{description=}\PYG{l+s}{\PYGZdq{}allow reading resource names/agents (parameters are denied separately)\PYGZdq{}}
           \PYG{n+na}{xpath=}\PYG{l+s}{\PYGZdq{}/cib/configuration/resources\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}

       \PYG{n+nt}{\PYGZlt{}acl\PYGZus{}permission} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}minimal\PYGZhy{}deny\PYGZhy{}constraints\PYGZdq{}} \PYG{n+na}{kind=}\PYG{l+s}{\PYGZdq{}deny\PYGZdq{}}
           \PYG{n+na}{xpath=}\PYG{l+s}{\PYGZdq{}/cib/configuration/constraints\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}

       \PYG{n+nt}{\PYGZlt{}acl\PYGZus{}permission} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}minimal\PYGZhy{}deny\PYGZhy{}topology\PYGZdq{}} \PYG{n+na}{kind=}\PYG{l+s}{\PYGZdq{}deny\PYGZdq{}}
           \PYG{n+na}{xpath=}\PYG{l+s}{\PYGZdq{}/cib/configuration/fencing\PYGZhy{}topology\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}

       \PYG{n+nt}{\PYGZlt{}acl\PYGZus{}permission} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}minimal\PYGZhy{}deny\PYGZhy{}op\PYGZus{}defaults\PYGZdq{}} \PYG{n+na}{kind=}\PYG{l+s}{\PYGZdq{}deny\PYGZdq{}}
           \PYG{n+na}{xpath=}\PYG{l+s}{\PYGZdq{}/cib/configuration/op\PYGZus{}defaults\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}

       \PYG{n+nt}{\PYGZlt{}acl\PYGZus{}permission} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}minimal\PYGZhy{}deny\PYGZhy{}rsc\PYGZus{}defaults\PYGZdq{}} \PYG{n+na}{kind=}\PYG{l+s}{\PYGZdq{}deny\PYGZdq{}}
           \PYG{n+na}{xpath=}\PYG{l+s}{\PYGZdq{}/cib/configuration/rsc\PYGZus{}defaults\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}

       \PYG{n+nt}{\PYGZlt{}acl\PYGZus{}permission} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}minimal\PYGZhy{}deny\PYGZhy{}alerts\PYGZdq{}} \PYG{n+na}{kind=}\PYG{l+s}{\PYGZdq{}deny\PYGZdq{}}
           \PYG{n+na}{xpath=}\PYG{l+s}{\PYGZdq{}/cib/configuration/alerts\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}

       \PYG{n+nt}{\PYGZlt{}acl\PYGZus{}permission} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}minimal\PYGZhy{}deny\PYGZhy{}acls\PYGZdq{}} \PYG{n+na}{kind=}\PYG{l+s}{\PYGZdq{}deny\PYGZdq{}}
           \PYG{n+na}{xpath=}\PYG{l+s}{\PYGZdq{}/cib/configuration/acls\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}

       \PYG{n+nt}{\PYGZlt{}acl\PYGZus{}permission} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}minimal\PYGZhy{}cib\PYGZdq{}} \PYG{n+na}{kind=}\PYG{l+s}{\PYGZdq{}read\PYGZdq{}}
           \PYG{n+na}{description=}\PYG{l+s}{\PYGZdq{}allow reading cib element and crm\PYGZus{}config/status sections\PYGZdq{}}
           \PYG{n+na}{xpath=}\PYG{l+s}{\PYGZdq{}/cib\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}

   \PYG{n+nt}{\PYGZlt{}/acl\PYGZus{}role\PYGZgt{}}

   \PYG{n+nt}{\PYGZlt{}acl\PYGZus{}target} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}alice\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}role} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}minimal\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}/acl\PYGZus{}target\PYGZgt{}}

   \PYG{n+nt}{\PYGZlt{}acl\PYGZus{}target} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}bob\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}role} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}read\PYGZus{}all\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}/acl\PYGZus{}target\PYGZgt{}}

   \PYG{n+nt}{\PYGZlt{}acl\PYGZus{}target} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}carol\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}role} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}read\PYGZus{}all\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}role} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}operator\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}/acl\PYGZus{}target\PYGZgt{}}

   \PYG{n+nt}{\PYGZlt{}acl\PYGZus{}target} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}dave\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}role} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}administrator\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
   \PYG{n+nt}{\PYGZlt{}/acl\PYGZus{}target\PYGZgt{}}

\PYG{n+nt}{\PYGZlt{}/acls\PYGZgt{}}
\end{sphinxVerbatim}

In the above example, the user \sphinxcode{\sphinxupquote{alice}} has the minimal permissions necessary
to run basic Pacemaker CLI tools, including using \sphinxcode{\sphinxupquote{crm\_mon}} to view the
cluster status, without being able to modify anything. The user \sphinxcode{\sphinxupquote{bob}} can
view the entire configuration and status of the cluster, but not make any
changes. The user \sphinxcode{\sphinxupquote{carol}} can read everything, and change selected cluster
properties as well as resource roles and location constraints. Finally,
\sphinxcode{\sphinxupquote{dave}} has full read and write access to the entire CIB.

Looking at the \sphinxcode{\sphinxupquote{minimal}} role in more depth, it is designed to allow read
access to the \sphinxcode{\sphinxupquote{cib}} tag itself, while denying access to particular portions
of its subtree (which is the entire CIB).

This is because the DC node is indicated in the \sphinxcode{\sphinxupquote{cib}} tag, so \sphinxcode{\sphinxupquote{crm\_mon}}
will not be able to report the DC otherwise. However, this does change the
security model to allow by default, since any portions of the CIB not
explicitly denied will be readable. The \sphinxcode{\sphinxupquote{cib}} read access could be removed
and replaced with read access to just the \sphinxcode{\sphinxupquote{crm\_config}} and \sphinxcode{\sphinxupquote{status}}
sections, for a safer approach at the cost of not seeing the DC in status
output.

For a simpler configuration, the \sphinxcode{\sphinxupquote{minimal}} role allows read access to the
entire \sphinxcode{\sphinxupquote{crm\_config}} section, which contains cluster properties. It would be
possible to allow read access to specific properties instead (such as
\sphinxcode{\sphinxupquote{stonith-enabled}}, \sphinxcode{\sphinxupquote{dc-uuid}}, \sphinxcode{\sphinxupquote{have-quorum}}, and \sphinxcode{\sphinxupquote{cluster-name}}) to
restrict access further while still allowing status output, but cluster
properties are unlikely to be considered sensitive.


\subsection{ACL Limitations}
\label{\detokenize{acls:acl-limitations}}

\subsubsection{Actions performed via IPC rather than the CIB}
\label{\detokenize{acls:actions-performed-via-ipc-rather-than-the-cib}}
ACLs apply \sphinxstyleemphasis{only} to the CIB.

That means ACLs apply to command-line tools that operate by reading or writing
the CIB, such as \sphinxcode{\sphinxupquote{crm\_attribute}} when managing permanent node attributes,
\sphinxcode{\sphinxupquote{crm\_mon}}, and \sphinxcode{\sphinxupquote{cibadmin}}.

However, command-line tools that communicate directly with Pacemaker daemons
via IPC are not affected by ACLs. For example, users in the \sphinxcode{\sphinxupquote{haclient}}
group may still do the following, regardless of ACLs:
\begin{itemize}
\item {} 
Query transient node attribute values using \sphinxcode{\sphinxupquote{crm\_attribute}} and
\sphinxcode{\sphinxupquote{attrd\_updater}}.

\item {} 
Query basic node information using \sphinxcode{\sphinxupquote{crm\_node}}.

\item {} 
Erase resource operation history using \sphinxcode{\sphinxupquote{crm\_resource}}.

\item {} 
Query fencing configuration information, and execute fencing against nodes,
using \sphinxcode{\sphinxupquote{stonith\_admin}}.

\end{itemize}


\subsubsection{ACLs and Pacemaker Remote}
\label{\detokenize{acls:acls-and-pacemaker-remote}}
ACLs apply to commands run on Pacemaker Remote nodes using the Pacemaker Remote
node’s name as the ACL user name.

The idea is that Pacemaker Remote nodes (especially virtual machines and
containers) are likely to be purpose-built and have different user accounts
from full cluster nodes.

\index{status}\index{XML element, status}\ignorespaces 

\section{Status \textendash{} Here be dragons}
\label{\detokenize{status:status-here-be-dragons}}\label{\detokenize{status:index-0}}\label{\detokenize{status::doc}}
Most users never need to understand the contents of the status section
and can be happy with the output from \sphinxcode{\sphinxupquote{crm\_mon}}.

However for those with a curious inclination, this section attempts to
provide an overview of its contents.

\index{node!status}\ignorespaces 

\subsection{Node Status}
\label{\detokenize{status:node-status}}\label{\detokenize{status:index-1}}
In addition to the cluster’s configuration, the CIB holds an
up-to-date representation of each cluster node in the \sphinxcode{\sphinxupquote{status}} section.

\begin{sphinxShadowBox}
\sphinxstyletopictitle{A bare-bones status entry for a healthy node \sphinxstylestrong{cl-virt-1}}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}node\PYGZus{}state} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}1\PYGZdq{}} \PYG{n+na}{uname=}\PYG{l+s}{\PYGZdq{}cl\PYGZhy{}virt\PYGZhy{}1\PYGZdq{}} \PYG{n+na}{in\PYGZus{}ccm=}\PYG{l+s}{\PYGZdq{}true\PYGZdq{}} \PYG{n+na}{crmd=}\PYG{l+s}{\PYGZdq{}online\PYGZdq{}} \PYG{n+na}{crm\PYGZhy{}debug\PYGZhy{}origin=}\PYG{l+s}{\PYGZdq{}do\PYGZus{}update\PYGZus{}resource\PYGZdq{}} \PYG{n+na}{join=}\PYG{l+s}{\PYGZdq{}member\PYGZdq{}} \PYG{n+na}{expected=}\PYG{l+s}{\PYGZdq{}member\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
 \PYG{n+nt}{\PYGZlt{}transient\PYGZus{}attributes} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}1\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
 \PYG{n+nt}{\PYGZlt{}lrm} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}1\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/node\PYGZus{}state\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

Users are highly recommended \sphinxstyleemphasis{not} to modify any part of a node’s
state \sphinxstyleemphasis{directly}.  The cluster will periodically regenerate the entire
section from authoritative sources, so any changes should be done
with the tools appropriate to those sources.


\begin{savenotes}\sphinxattablestart
\centering
\sphinxcapstartof{table}
\sphinxcaption{\sphinxstylestrong{Authoritative Sources for State Information}}\label{\detokenize{status:id3}}
\sphinxaftercaption
\begin{tabular}[t]{|\X{1}{2}|\X{1}{2}|}
\hline
\sphinxstyletheadfamily 
CIB Object
&\sphinxstyletheadfamily 
Authoritative Source
\\
\hline
node\_state
&
pacemaker-controld
\\
\hline
transient\_attributes
&
pacemaker-attrd
\\
\hline
lrm
&
pacemaker-execd
\\
\hline
\end{tabular}
\par
\sphinxattableend\end{savenotes}

The fields used in the \sphinxcode{\sphinxupquote{node\_state}} objects are named as they are
largely for historical reasons, to maintain compatibility with older versions.


\begin{savenotes}\sphinxattablestart
\centering
\sphinxcapstartof{table}
\sphinxcaption{\sphinxstylestrong{Node Status Fields}}\label{\detokenize{status:id4}}
\sphinxaftercaption
\begin{tabular}[t]{|\X{1}{4}|\X{3}{4}|}
\hline
\sphinxstyletheadfamily 
Field
&\sphinxstyletheadfamily 
Description
\\
\hline
id
&
Unique identifier for the node.  Corosync-based clusters
use a numeric counter.
\\
\hline
uname
&
\index{uname!node status}\index{node!status, uname}\ignorespaces 
The node’s name as known by the cluster
\\
\hline
in\_ccm
&
\index{in\_ccm!node status}\index{node!status, in\_ccm}\ignorespaces 
Is the node a member at the cluster communication later?
Allowed values: \sphinxcode{\sphinxupquote{true}}, \sphinxcode{\sphinxupquote{false}}.
\\
\hline
crmd
&
\index{crmd!node status}\index{node!status, crmd}\ignorespaces 
Is the node a member at the pacemaker layer?  Allowed
values: \sphinxcode{\sphinxupquote{online}}, \sphinxcode{\sphinxupquote{offline}}.
\\
\hline
crm-debug-origin
&
\index{crm-debug-origin!node status}\index{node!status, crm-debug-origin}\ignorespaces 
The name of the source function that made the most
recent change (for debugging purposes).
\\
\hline
join
&
\index{join!node status}\index{node!status, join}\ignorespaces 
Does the node participate in hosting resources?
Allowed values: \sphinxcode{\sphinxupquote{down}}, \sphinxcode{\sphinxupquote{pending}}, \sphinxcode{\sphinxupquote{member}}.
\sphinxcode{\sphinxupquote{banned}}.
\\
\hline
expected
&
\index{expected!node status}\index{node!status, expected}\ignorespaces 
Expected value for \sphinxcode{\sphinxupquote{join}}.
\\
\hline
\end{tabular}
\par
\sphinxattableend\end{savenotes}

The cluster uses these fields to determine whether, at the node level, the
node is healthy or is in a failed state and needs to be fenced.


\subsection{Transient Node Attributes}
\label{\detokenize{status:transient-node-attributes}}
Like regular {\hyperref[\detokenize{nodes:node-attributes}]{\sphinxcrossref{\DUrole{std,std-ref}{Node Attributes}}}}, the name/value
pairs listed in the \sphinxcode{\sphinxupquote{transient\_attributes}} section help to describe the
node.  However they are forgotten by the cluster when the node goes offline.
This can be useful, for instance, when you want a node to be in standby mode
(not able to run resources) just until the next reboot.

In addition to any values the administrator sets, the cluster will
also store information about failed resources here.

\begin{sphinxShadowBox}
\sphinxstyletopictitle{A set of transient node attributes for node \sphinxstylestrong{cl-virt-1}}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}transient\PYGZus{}attributes} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}cl\PYGZhy{}virt\PYGZhy{}1\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}instance\PYGZus{}attributes} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}status\PYGZhy{}cl\PYGZhy{}virt\PYGZhy{}1\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
     \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}status\PYGZhy{}cl\PYGZhy{}virt\PYGZhy{}1\PYGZhy{}pingd\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}pingd\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}3\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
     \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}status\PYGZhy{}cl\PYGZhy{}virt\PYGZhy{}1\PYGZhy{}probe\PYGZus{}complete\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}probe\PYGZus{}complete\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}true\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
     \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}status\PYGZhy{}cl\PYGZhy{}virt\PYGZhy{}1\PYGZhy{}fail\PYGZhy{}count\PYGZhy{}pingd:0.monitor\PYGZus{}30000\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}fail\PYGZhy{}count\PYGZhy{}pingd:0\PYGZsh{}monitor\PYGZus{}30000\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}1\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
     \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}status\PYGZhy{}cl\PYGZhy{}virt\PYGZhy{}1\PYGZhy{}last\PYGZhy{}failure\PYGZhy{}pingd:0\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}last\PYGZhy{}failure\PYGZhy{}pingd:0\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}1239009742\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}/instance\PYGZus{}attributes\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/transient\PYGZus{}attributes\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

In the above example, we can see that a monitor on the \sphinxcode{\sphinxupquote{pingd:0}} resource has
failed once, at 09:22:22 UTC 6 April 2009. %
\begin{footnote}[1]\sphinxAtStartFootnote
You can use the standard \sphinxcode{\sphinxupquote{date}} command to print a human-readable version
of any seconds-since-epoch value, for example \sphinxcode{\sphinxupquote{date -d @1239009742}}.
%
\end{footnote}.

We also see that the node is connected to three \sphinxstylestrong{pingd} peers and that
all known resources have been checked for on this machine (\sphinxcode{\sphinxupquote{probe\_complete}}).

\index{Operation History}\ignorespaces 

\subsection{Operation History}
\label{\detokenize{status:operation-history}}\label{\detokenize{status:index-8}}
A node’s resource history is held in the \sphinxcode{\sphinxupquote{lrm\_resources}} element (a child
of the \sphinxcode{\sphinxupquote{lrm}} element). The information stored here includes enough
information for the cluster to stop the resource safely if it is
removed from the \sphinxcode{\sphinxupquote{configuration}} section. Specifically, the resource’s
\sphinxcode{\sphinxupquote{id}}, \sphinxcode{\sphinxupquote{class}}, \sphinxcode{\sphinxupquote{type}} and \sphinxcode{\sphinxupquote{provider}} are stored.

\begin{sphinxShadowBox}
\sphinxstyletopictitle{A record of the \sphinxstyleliteralintitle{\sphinxupquote{apcstonith}} resource}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}lrm\PYGZus{}resource} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}apcstonith\PYGZdq{}} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}fence\PYGZus{}apc\PYGZus{}snmp\PYGZdq{}} \PYG{n+na}{class=}\PYG{l+s}{\PYGZdq{}stonith\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

Additionally, we store history entries for certain actions.


\begin{savenotes}\sphinxatlongtablestart\begin{longtable}{|\X{1}{4}|\X{3}{4}|}
\caption{\sphinxstylestrong{Attributes of an lrm\_rsc\_op element}\strut}\label{\detokenize{status:id5}}\\*[\sphinxlongtablecapskipadjust]
\hline
\sphinxstyletheadfamily 
Field
&\sphinxstyletheadfamily 
Description
\\
\hline
\endfirsthead

\multicolumn{2}{c}%
{\makebox[0pt]{\sphinxtablecontinued{\tablename\ \thetable{} -- continued from previous page}}}\\
\hline
\sphinxstyletheadfamily 
Field
&\sphinxstyletheadfamily 
Description
\\
\hline
\endhead

\hline
\multicolumn{2}{r}{\makebox[0pt][r]{\sphinxtablecontinued{Continued on next page}}}\\
\endfoot

\endlastfoot

id
&
\index{id!action status}\index{action!status, id}\ignorespaces 
Identifier for the history entry constructed from the
resource ID, action name, and operation interval.
\\
\hline
call-id
&
\index{call-id!action status}\index{action!status, call-id}\ignorespaces 
A node-specific counter used to determine the order in
which actions were executed.
\\
\hline
operation
&
\index{operation!action status}\index{action!status, operation}\ignorespaces 
The action name the resource agent was invoked with.
\\
\hline
interval
&
\index{interval!action status}\index{action!status, interval}\ignorespaces 
The frequency, in milliseconds, at which the operation
will be repeated. One-time execution is indicated by 0.
\\
\hline
op-status
&
\index{op-status!action status}\index{action!status, op-status}\ignorespaces 
The execution status of this action. The meanings of
these codes are internal to Pacemaker.
\\
\hline
rc-code
&
\index{rc-code!action status}\index{action!status, rc-code}\ignorespaces 
The resource agent’s exit status for this action. Refer
to the \sphinxstyleemphasis{Resource Agents} chapter of
\sphinxstyleemphasis{Pacemaker Administration} for how these values are
interpreted.
\\
\hline
last-rc-change
&
\index{last-rc-change!action status}\index{action!status, last-rc-change}\ignorespaces 
Machine-local date/time, in seconds since epoch, at
which the action first returned the current value of
\sphinxcode{\sphinxupquote{rc-code}}.  For diagnostic purposes.
\\
\hline
exec-time
&
\index{exec-time!action status}\index{action!status, exec-time}\ignorespaces 
Time, in milliseconds, that the action was running for.
For diagnostic purposes.
\\
\hline
queue-time
&
\index{queue-time!action status}\index{action!status, queue-time}\ignorespaces 
Time, in seconds, that the action was queued for in the
local executor. For diagnostic purposes.
\\
\hline
crm\_feature\_set
&
\index{crm\_feature\_set!action status}\index{action!status, crm\_feature\_set}\ignorespaces 
The Pacemaker feature set used to record this entry.
\\
\hline
transition-key
&
\index{transition-key!action status}\index{action!status, transition-key}\ignorespaces 
A concatenation of the action’s graph action number, the
graph number, the expected result and the UUID of the
controller instance that scheduled it. This is used to
construct \sphinxcode{\sphinxupquote{transition-magic}} (below).
\\
\hline
transition-magic
&
\index{transition-magic!action status}\index{action!status, transition-magic}\ignorespaces 
A concatenation of \sphinxcode{\sphinxupquote{op-status}}, \sphinxcode{\sphinxupquote{rc-code}}
and \sphinxcode{\sphinxupquote{transition-key}}. Guaranteed to be unique for the
life of the cluster (which ensures it is part of CIB
update notifications) and contains all the information
needed for the controller to correctly analyze and
process the completed action. Most importantly, the
decomposed elements tell the controller if the history
entry was expected and whether it failed.
\\
\hline
op-digest
&
\index{op-digest!action status}\index{action!status, op-digest}\ignorespaces 
An MD5 sum representing the parameters passed to the
action. Used to detect changes to the configuration, to
restart resources if necessary.
\\
\hline
crm-debug-origin
&
\index{crm-debug-origin!action status}\index{action!status, crm-debug-origin}\ignorespaces 
The origin of the current values.  For diagnostic
purposes.
\\
\hline
\end{longtable}\sphinxatlongtableend\end{savenotes}


\subsubsection{Simple Operation History Example}
\label{\detokenize{status:simple-operation-history-example}}
\begin{sphinxShadowBox}
\sphinxstyletopictitle{A monitor operation (determines current state of the \sphinxstyleliteralintitle{\sphinxupquote{apcstonith}} resource)}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}lrm\PYGZus{}resource} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}apcstonith\PYGZdq{}} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}fence\PYGZus{}apc\PYGZus{}snmp\PYGZdq{}} \PYG{n+na}{class=}\PYG{l+s}{\PYGZdq{}stonith\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}lrm\PYGZus{}rsc\PYGZus{}op} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}apcstonith\PYGZus{}monitor\PYGZus{}0\PYGZdq{}} \PYG{n+na}{operation=}\PYG{l+s}{\PYGZdq{}monitor\PYGZdq{}} \PYG{n+na}{call\PYGZhy{}id=}\PYG{l+s}{\PYGZdq{}2\PYGZdq{}}
    \PYG{n+na}{rc\PYGZhy{}code=}\PYG{l+s}{\PYGZdq{}7\PYGZdq{}} \PYG{n+na}{op\PYGZhy{}status=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{interval=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}}
    \PYG{n+na}{crm\PYGZhy{}debug\PYGZhy{}origin=}\PYG{l+s}{\PYGZdq{}do\PYGZus{}update\PYGZus{}resource\PYGZdq{}} \PYG{n+na}{crm\PYGZus{}feature\PYGZus{}set=}\PYG{l+s}{\PYGZdq{}3.0.1\PYGZdq{}}
    \PYG{n+na}{op\PYGZhy{}digest=}\PYG{l+s}{\PYGZdq{}2e3da9274d3550dc6526fb24bfcbcba0\PYGZdq{}}
    \PYG{n+na}{transition\PYGZhy{}key=}\PYG{l+s}{\PYGZdq{}22:2:7:2668bbeb\PYGZhy{}06d5\PYGZhy{}40f9\PYGZhy{}936d\PYGZhy{}24cb7f87006a\PYGZdq{}}
    \PYG{n+na}{transition\PYGZhy{}magic=}\PYG{l+s}{\PYGZdq{}0:7;22:2:7:2668bbeb\PYGZhy{}06d5\PYGZhy{}40f9\PYGZhy{}936d\PYGZhy{}24cb7f87006a\PYGZdq{}}
    \PYG{n+na}{last\PYGZhy{}rc\PYGZhy{}change=}\PYG{l+s}{\PYGZdq{}1239008085\PYGZdq{}} \PYG{n+na}{exec\PYGZhy{}time=}\PYG{l+s}{\PYGZdq{}10\PYGZdq{}} \PYG{n+na}{queue\PYGZhy{}time=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/lrm\PYGZus{}resource\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

In the above example, the action is a non-recurring monitor operation
often referred to as a “probe” for the \sphinxcode{\sphinxupquote{apcstonith}} resource.

The cluster schedules probes for every configured resource on a node when
the node first starts, in order to determine the resource’s current state
before it takes any further action.

From the \sphinxcode{\sphinxupquote{transition-key}}, we can see that this was the 22nd action of
the 2nd graph produced by this instance of the controller
(2668bbeb-06d5-40f9-936d-24cb7f87006a).

The third field of the \sphinxcode{\sphinxupquote{transition-key}} contains a 7, which indicates
that the cluster expects to find the resource inactive. By looking at the
\sphinxcode{\sphinxupquote{rc-code}} property, we see that this was the case.

As that is the only action recorded for this node, we can conclude that
the cluster started the resource elsewhere.


\subsubsection{Complex Operation History Example}
\label{\detokenize{status:complex-operation-history-example}}
\begin{sphinxShadowBox}
\sphinxstyletopictitle{Resource history of a \sphinxstyleliteralintitle{\sphinxupquote{pingd}} clone with multiple entries}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}lrm\PYGZus{}resource} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}pingd:0\PYGZdq{}} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}pingd\PYGZdq{}} \PYG{n+na}{class=}\PYG{l+s}{\PYGZdq{}ocf\PYGZdq{}} \PYG{n+na}{provider=}\PYG{l+s}{\PYGZdq{}pacemaker\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}lrm\PYGZus{}rsc\PYGZus{}op} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}pingd:0\PYGZus{}monitor\PYGZus{}30000\PYGZdq{}} \PYG{n+na}{operation=}\PYG{l+s}{\PYGZdq{}monitor\PYGZdq{}} \PYG{n+na}{call\PYGZhy{}id=}\PYG{l+s}{\PYGZdq{}34\PYGZdq{}}
    \PYG{n+na}{rc\PYGZhy{}code=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{op\PYGZhy{}status=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{interval=}\PYG{l+s}{\PYGZdq{}30000\PYGZdq{}}
    \PYG{n+na}{crm\PYGZhy{}debug\PYGZhy{}origin=}\PYG{l+s}{\PYGZdq{}do\PYGZus{}update\PYGZus{}resource\PYGZdq{}} \PYG{n+na}{crm\PYGZus{}feature\PYGZus{}set=}\PYG{l+s}{\PYGZdq{}3.0.1\PYGZdq{}}
    \PYG{n+na}{transition\PYGZhy{}key=}\PYG{l+s}{\PYGZdq{}10:11:0:2668bbeb\PYGZhy{}06d5\PYGZhy{}40f9\PYGZhy{}936d\PYGZhy{}24cb7f87006a\PYGZdq{}}
    \PYG{n+na}{last\PYGZhy{}rc\PYGZhy{}change=}\PYG{l+s}{\PYGZdq{}1239009741\PYGZdq{}} \PYG{n+na}{exec\PYGZhy{}time=}\PYG{l+s}{\PYGZdq{}10\PYGZdq{}} \PYG{n+na}{queue\PYGZhy{}time=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}lrm\PYGZus{}rsc\PYGZus{}op} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}pingd:0\PYGZus{}stop\PYGZus{}0\PYGZdq{}} \PYG{n+na}{operation=}\PYG{l+s}{\PYGZdq{}stop\PYGZdq{}}
    \PYG{n+na}{crm\PYGZhy{}debug\PYGZhy{}origin=}\PYG{l+s}{\PYGZdq{}do\PYGZus{}update\PYGZus{}resource\PYGZdq{}} \PYG{n+na}{crm\PYGZus{}feature\PYGZus{}set=}\PYG{l+s}{\PYGZdq{}3.0.1\PYGZdq{}} \PYG{n+na}{call\PYGZhy{}id=}\PYG{l+s}{\PYGZdq{}32\PYGZdq{}}
    \PYG{n+na}{rc\PYGZhy{}code=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{op\PYGZhy{}status=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{interval=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}}
    \PYG{n+na}{transition\PYGZhy{}key=}\PYG{l+s}{\PYGZdq{}11:11:0:2668bbeb\PYGZhy{}06d5\PYGZhy{}40f9\PYGZhy{}936d\PYGZhy{}24cb7f87006a\PYGZdq{}}
    \PYG{n+na}{last\PYGZhy{}rc\PYGZhy{}change=}\PYG{l+s}{\PYGZdq{}1239009741\PYGZdq{}} \PYG{n+na}{exec\PYGZhy{}time=}\PYG{l+s}{\PYGZdq{}10\PYGZdq{}} \PYG{n+na}{queue\PYGZhy{}time=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}lrm\PYGZus{}rsc\PYGZus{}op} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}pingd:0\PYGZus{}start\PYGZus{}0\PYGZdq{}} \PYG{n+na}{operation=}\PYG{l+s}{\PYGZdq{}start\PYGZdq{}} \PYG{n+na}{call\PYGZhy{}id=}\PYG{l+s}{\PYGZdq{}33\PYGZdq{}}
    \PYG{n+na}{rc\PYGZhy{}code=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{op\PYGZhy{}status=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{interval=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}}
    \PYG{n+na}{crm\PYGZhy{}debug\PYGZhy{}origin=}\PYG{l+s}{\PYGZdq{}do\PYGZus{}update\PYGZus{}resource\PYGZdq{}} \PYG{n+na}{crm\PYGZus{}feature\PYGZus{}set=}\PYG{l+s}{\PYGZdq{}3.0.1\PYGZdq{}}
    \PYG{n+na}{transition\PYGZhy{}key=}\PYG{l+s}{\PYGZdq{}31:11:0:2668bbeb\PYGZhy{}06d5\PYGZhy{}40f9\PYGZhy{}936d\PYGZhy{}24cb7f87006a\PYGZdq{}}
    \PYG{n+na}{last\PYGZhy{}rc\PYGZhy{}change=}\PYG{l+s}{\PYGZdq{}1239009741\PYGZdq{}} \PYG{n+na}{exec\PYGZhy{}time=}\PYG{l+s}{\PYGZdq{}10\PYGZdq{}} \PYG{n+na}{queue\PYGZhy{}time=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}lrm\PYGZus{}rsc\PYGZus{}op} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}pingd:0\PYGZus{}monitor\PYGZus{}0\PYGZdq{}} \PYG{n+na}{operation=}\PYG{l+s}{\PYGZdq{}monitor\PYGZdq{}} \PYG{n+na}{call\PYGZhy{}id=}\PYG{l+s}{\PYGZdq{}3\PYGZdq{}}
    \PYG{n+na}{rc\PYGZhy{}code=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{op\PYGZhy{}status=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{interval=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}}
    \PYG{n+na}{crm\PYGZhy{}debug\PYGZhy{}origin=}\PYG{l+s}{\PYGZdq{}do\PYGZus{}update\PYGZus{}resource\PYGZdq{}} \PYG{n+na}{crm\PYGZus{}feature\PYGZus{}set=}\PYG{l+s}{\PYGZdq{}3.0.1\PYGZdq{}}
    \PYG{n+na}{transition\PYGZhy{}key=}\PYG{l+s}{\PYGZdq{}23:2:7:2668bbeb\PYGZhy{}06d5\PYGZhy{}40f9\PYGZhy{}936d\PYGZhy{}24cb7f87006a\PYGZdq{}}
    \PYG{n+na}{last\PYGZhy{}rc\PYGZhy{}change=}\PYG{l+s}{\PYGZdq{}1239008085\PYGZdq{}} \PYG{n+na}{exec\PYGZhy{}time=}\PYG{l+s}{\PYGZdq{}20\PYGZdq{}} \PYG{n+na}{queue\PYGZhy{}time=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}/lrm\PYGZus{}resource\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

When more than one history entry exists, it is important to first sort
them by \sphinxcode{\sphinxupquote{call-id}} before interpreting them.

Once sorted, the above example can be summarized as:
\begin{enumerate}
\item {} 
A non-recurring monitor operation returning 7 (not running), with a \sphinxcode{\sphinxupquote{call-id}} of 3

\item {} 
A stop operation returning 0 (success), with a \sphinxcode{\sphinxupquote{call-id}} of 32

\item {} 
A start operation returning 0 (success), with a \sphinxcode{\sphinxupquote{call-id}} of 33

\item {} 
A recurring monitor returning 0 (success), with a \sphinxcode{\sphinxupquote{call-id}} of 34

\end{enumerate}

The cluster processes each history entry to build up a picture of the
resource’s state.  After the first and second entries, it is
considered stopped, and after the third it considered active.

Based on the last operation, we can tell that the resource is
currently active.

Additionally, from the presence of a \sphinxcode{\sphinxupquote{stop}} operation with a lower
\sphinxcode{\sphinxupquote{call-id}} than that of the \sphinxcode{\sphinxupquote{start}} operation, we can conclude that the
resource has been restarted.  Specifically this occurred as part of
actions 11 and 31 of transition 11 from the controller instance with the key
\sphinxcode{\sphinxupquote{2668bbeb...}}.  This information can be helpful for locating the
relevant section of the logs when looking for the source of a failure.


\section{Multi-Site Clusters and Tickets}
\label{\detokenize{multi-site-clusters:multi-site-clusters-and-tickets}}\label{\detokenize{multi-site-clusters::doc}}
Apart from local clusters, Pacemaker also supports multi-site clusters.
That means you can have multiple, geographically dispersed sites, each with a
local cluster. Failover between these clusters can be coordinated
manually by the administrator, or automatically by a higher-level entity called
a \sphinxstyleemphasis{Cluster Ticket Registry (CTR)}.


\subsection{Challenges for Multi-Site Clusters}
\label{\detokenize{multi-site-clusters:challenges-for-multi-site-clusters}}
Typically, multi-site environments are too far apart to support
synchronous communication and data replication between the sites.
That leads to significant challenges:
\begin{itemize}
\item {} 
How do we make sure that a cluster site is up and running?

\item {} 
How do we make sure that resources are only started once?

\item {} 
How do we make sure that quorum can be reached between the different
sites and a split-brain scenario avoided?

\item {} 
How do we manage failover between sites?

\item {} 
How do we deal with high latency in case of resources that need to be
stopped?

\end{itemize}

In the following sections, learn how to meet these challenges.


\subsection{Conceptual Overview}
\label{\detokenize{multi-site-clusters:conceptual-overview}}
Multi-site clusters can be considered as “overlay” clusters where
each cluster site corresponds to a cluster node in a traditional cluster.
The overlay cluster can be managed by a CTR in order to
guarantee that any cluster resource will be active
on no more than one cluster site. This is achieved by using
\sphinxstyleemphasis{tickets} that are treated as failover domain between cluster
sites, in case a site should be down.

The following sections explain the individual components and mechanisms
that were introduced for multi-site clusters in more detail.


\subsubsection{Ticket}
\label{\detokenize{multi-site-clusters:ticket}}
Tickets are, essentially, cluster-wide attributes. A ticket grants the
right to run certain resources on a specific cluster site. Resources can
be bound to a certain ticket by \sphinxcode{\sphinxupquote{rsc\_ticket}} constraints. Only if the
ticket is available at a site can the respective resources be started there.
Vice versa, if the ticket is revoked, the resources depending on that
ticket must be stopped.

The ticket thus is similar to a \sphinxstyleemphasis{site quorum}, i.e. the permission to
manage/own resources associated with that site. (One can also think of the
current \sphinxcode{\sphinxupquote{have-quorum}} flag as a special, cluster-wide ticket that is
granted in case of node majority.)

Tickets can be granted and revoked either manually by administrators
(which could be the default for classic enterprise clusters), or via
the automated CTR mechanism described below.

A ticket can only be owned by one site at a time. Initially, none
of the sites has a ticket. Each ticket must be granted once by the cluster
administrator.

The presence or absence of tickets for a site is stored in the CIB as a
cluster status. With regards to a certain ticket, there are only two states
for a site: \sphinxcode{\sphinxupquote{true}} (the site has the ticket) or \sphinxcode{\sphinxupquote{false}} (the site does
not have the ticket). The absence of a certain ticket (during the initial
state of the multi-site cluster) is the same as the value \sphinxcode{\sphinxupquote{false}}.


\subsubsection{Dead Man Dependency}
\label{\detokenize{multi-site-clusters:dead-man-dependency}}
A site can only activate resources safely if it can be sure that the
other site has deactivated them. However after a ticket is revoked, it can
take a long time until all resources depending on that ticket are stopped
“cleanly”, especially in case of cascaded resources. To cut that process
short, the concept of a \sphinxstyleemphasis{Dead Man Dependency} was introduced.

If a dead man dependency is in force, if a ticket is revoked from a site, the
nodes that are hosting dependent resources are fenced. This considerably speeds
up the recovery process of the cluster and makes sure that resources can be
migrated more quickly.

This can be configured by specifying a \sphinxcode{\sphinxupquote{loss-policy="fence"}} in
\sphinxcode{\sphinxupquote{rsc\_ticket}} constraints.


\subsubsection{Cluster Ticket Registry}
\label{\detokenize{multi-site-clusters:cluster-ticket-registry}}
A CTR is a coordinated group of network daemons that automatically handles
granting, revoking, and timing out tickets (instead of the administrator
revoking the ticket somewhere, waiting for everything to stop, and then
granting it on the desired site).

Pacemaker does not implement its own CTR, but interoperates with external
software designed for that purpose (similar to how resource and fencing agents
are not directly part of pacemaker).

Participating clusters run the CTR daemons, which connect to each other, exchange
information about their connectivity, and vote on which sites gets which
tickets.

A ticket is granted to a site only once the CTR is sure that the ticket
has been relinquished by the previous owner, implemented via a timer in most
scenarios. If a site loses connection to its peers, its tickets time out and
recovery occurs. After the connection timeout plus the recovery timeout has
passed, the other sites are allowed to re-acquire the ticket and start the
resources again.

This can also be thought of as a “quorum server”, except that it is not
a single quorum ticket, but several.


\subsubsection{Configuration Replication}
\label{\detokenize{multi-site-clusters:configuration-replication}}
As usual, the CIB is synchronized within each cluster, but it is \sphinxstyleemphasis{not} synchronized
across cluster sites of a multi-site cluster. You have to configure the resources
that will be highly available across the multi-site cluster for every site
accordingly.


\subsection{Configuring Ticket Dependencies}
\label{\detokenize{multi-site-clusters:configuring-ticket-dependencies}}\label{\detokenize{multi-site-clusters:ticket-constraints}}
The \sphinxstylestrong{rsc\_ticket} constraint lets you specify the resources depending on a certain
ticket. Together with the constraint, you can set a \sphinxstylestrong{loss-policy} that defines
what should happen to the respective resources if the ticket is revoked.

The attribute \sphinxstylestrong{loss-policy} can have the following values:
\begin{itemize}
\item {} 
\sphinxcode{\sphinxupquote{fence:}} Fence the nodes that are running the relevant resources.

\item {} 
\sphinxcode{\sphinxupquote{stop:}} Stop the relevant resources.

\item {} 
\sphinxcode{\sphinxupquote{freeze:}} Do nothing to the relevant resources.

\item {} 
\sphinxcode{\sphinxupquote{demote:}} Demote relevant resources that are running in the promoted role.

\end{itemize}

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Constraint that fences node if \sphinxstyleliteralintitle{\sphinxupquote{ticketA}} is revoked}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}ticket} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}rsc1\PYGZhy{}req\PYGZhy{}ticketA\PYGZdq{}} \PYG{n+na}{rsc=}\PYG{l+s}{\PYGZdq{}rsc1\PYGZdq{}} \PYG{n+na}{ticket=}\PYG{l+s}{\PYGZdq{}ticketA\PYGZdq{}} \PYG{n+na}{loss\PYGZhy{}policy=}\PYG{l+s}{\PYGZdq{}fence\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

The example above creates a constraint with the ID \sphinxcode{\sphinxupquote{rsc1-req-ticketA}}. It
defines that the resource \sphinxcode{\sphinxupquote{rsc1}} depends on \sphinxcode{\sphinxupquote{ticketA}} and that the node running
the resource should be fenced if \sphinxcode{\sphinxupquote{ticketA}} is revoked.

If resource \sphinxcode{\sphinxupquote{rsc1}} were a promotable resource, you might want to configure
that only being in the promoted role depends on \sphinxcode{\sphinxupquote{ticketA}}. With the following
configuration, \sphinxcode{\sphinxupquote{rsc1}} will be demoted if \sphinxcode{\sphinxupquote{ticketA}} is revoked:

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Constraint that demotes \sphinxstyleliteralintitle{\sphinxupquote{rsc1}} if \sphinxstyleliteralintitle{\sphinxupquote{ticketA}} is revoked}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}ticket} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}rsc1\PYGZhy{}req\PYGZhy{}ticketA\PYGZdq{}} \PYG{n+na}{rsc=}\PYG{l+s}{\PYGZdq{}rsc1\PYGZdq{}} \PYG{n+na}{rsc\PYGZhy{}role=}\PYG{l+s}{\PYGZdq{}Promoted\PYGZdq{}} \PYG{n+na}{ticket=}\PYG{l+s}{\PYGZdq{}ticketA\PYGZdq{}} \PYG{n+na}{loss\PYGZhy{}policy=}\PYG{l+s}{\PYGZdq{}demote\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

You can create multiple \sphinxstylestrong{rsc\_ticket} constraints to let multiple resources
depend on the same ticket. However, \sphinxstylestrong{rsc\_ticket} also supports resource sets
(see {\hyperref[\detokenize{constraints:s-resource-sets}]{\sphinxcrossref{\DUrole{std,std-ref}{Resource Sets}}}}), so one can easily list all the resources in one
\sphinxstylestrong{rsc\_ticket} constraint instead.

\begin{sphinxShadowBox}
\sphinxstyletopictitle{Ticket constraint for multiple resources}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}ticket} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}resources\PYGZhy{}dep\PYGZhy{}ticketA\PYGZdq{}} \PYG{n+na}{ticket=}\PYG{l+s}{\PYGZdq{}ticketA\PYGZdq{}} \PYG{n+na}{loss\PYGZhy{}policy=}\PYG{l+s}{\PYGZdq{}fence\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}set} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}resources\PYGZhy{}dep\PYGZhy{}ticketA\PYGZhy{}0\PYGZdq{}} \PYG{n+na}{role=}\PYG{l+s}{\PYGZdq{}Started\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}rsc1\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}group1\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}clone1\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}/resource\PYGZus{}set\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}set} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}resources\PYGZhy{}dep\PYGZhy{}ticketA\PYGZhy{}1\PYGZdq{}} \PYG{n+na}{role=}\PYG{l+s}{\PYGZdq{}Promoted\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}resource\PYGZus{}ref} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}ms1\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}/resource\PYGZus{}set\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/rsc\PYGZus{}ticket\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

In the example above, there are two resource sets, so we can list resources
with different roles in a single \sphinxcode{\sphinxupquote{rsc\_ticket}} constraint. There’s no dependency
between the two resource sets, and there’s no dependency among the
resources within a resource set. Each of the resources just depends on
\sphinxcode{\sphinxupquote{ticketA}}.

Referencing resource templates in \sphinxcode{\sphinxupquote{rsc\_ticket}} constraints, and even
referencing them within resource sets, is also supported.

If you want other resources to depend on further tickets, create as many
constraints as necessary with \sphinxcode{\sphinxupquote{rsc\_ticket}}.


\subsection{Managing Multi-Site Clusters}
\label{\detokenize{multi-site-clusters:managing-multi-site-clusters}}

\subsubsection{Granting and Revoking Tickets Manually}
\label{\detokenize{multi-site-clusters:granting-and-revoking-tickets-manually}}
You can grant tickets to sites or revoke them from sites manually.
If you want to re-distribute a ticket, you should wait for
the dependent resources to stop cleanly at the previous site before you
grant the ticket to the new site.

Use the \sphinxstylestrong{crm\_ticket} command line tool to grant and revoke tickets.

To grant a ticket to this site:
\begin{quote}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYGZsh{} crm\PYGZus{}ticket \PYGZhy{}\PYGZhy{}ticket ticketA \PYGZhy{}\PYGZhy{}grant
\end{sphinxVerbatim}
\end{quote}

To revoke a ticket from this site:
\begin{quote}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYGZsh{} crm\PYGZus{}ticket \PYGZhy{}\PYGZhy{}ticket ticketA \PYGZhy{}\PYGZhy{}revoke
\end{sphinxVerbatim}
\end{quote}

\begin{sphinxadmonition}{important}{Important:}
If you are managing tickets manually, use the \sphinxstylestrong{crm\_ticket} command with
great care, because it cannot check whether the same ticket is already
granted elsewhere.
\end{sphinxadmonition}


\subsubsection{Granting and Revoking Tickets via a Cluster Ticket Registry}
\label{\detokenize{multi-site-clusters:granting-and-revoking-tickets-via-a-cluster-ticket-registry}}
We will use \sphinxhref{https://github.com/ClusterLabs/booth}{Booth} here as an example of
software that can be used with pacemaker as a Cluster Ticket Registry.  Booth
implements the \sphinxhref{http://en.wikipedia.org/wiki/Raft\_\%28computer\_science\%29}{Raft}
algorithm to guarantee the distributed consensus among different
cluster sites, and manages the ticket distribution (and thus the failover
process between sites).

Each of the participating clusters and \sphinxstyleemphasis{arbitrators} runs the Booth daemon
\sphinxstylestrong{boothd}.

An \sphinxstyleemphasis{arbitrator} is the multi-site equivalent of a quorum-only node in a local
cluster. If you have a setup with an even number of sites,
you need an additional instance to reach consensus about decisions such
as failover of resources across sites. In this case, add one or more
arbitrators running at additional sites. Arbitrators are single machines
that run a booth instance in a special mode. An arbitrator is especially
important for a two-site scenario, otherwise there is no way for one site
to distinguish between a network failure between it and the other site, and
a failure of the other site.

The most common multi-site scenario is probably a multi-site cluster with two
sites and a single arbitrator on a third site. However, technically, there are
no limitations with regards to the number of sites and the number of
arbitrators involved.

\sphinxstylestrong{Boothd} at each site connects to its peers running at the other sites and
exchanges connectivity details. Once a ticket is granted to a site, the
booth mechanism will manage the ticket automatically: If the site which
holds the ticket is out of service, the booth daemons will vote which
of the other sites will get the ticket. To protect against brief
connection failures, sites that lose the vote (either explicitly or
implicitly by being disconnected from the voting body) need to
relinquish the ticket after a time-out. Thus, it is made sure that a
ticket will only be re-distributed after it has been relinquished by the
previous site.  The resources that depend on that ticket will fail over
to the new site holding the ticket. The nodes that have run the
resources before will be treated according to the \sphinxstylestrong{loss-policy} you set
within the \sphinxstylestrong{rsc\_ticket} constraint.

Before the booth can manage a certain ticket within the multi-site cluster,
you initially need to grant it to a site manually via the \sphinxstylestrong{booth} command-line
tool. After you have initially granted a ticket to a site, \sphinxstylestrong{boothd}
will take over and manage the ticket automatically.

\begin{sphinxadmonition}{important}{Important:}
The \sphinxstylestrong{booth} command-line tool can be used to grant, list, or
revoke tickets and can be run on any machine where \sphinxstylestrong{boothd} is running.
If you are managing tickets via Booth, use only \sphinxstylestrong{booth} for manual
intervention, not \sphinxstylestrong{crm\_ticket}. That ensures the same ticket
will only be owned by one cluster site at a time.
\end{sphinxadmonition}


\paragraph{Booth Requirements}
\label{\detokenize{multi-site-clusters:booth-requirements}}\begin{itemize}
\item {} 
All clusters that will be part of the multi-site cluster must be based on
Pacemaker.

\item {} 
Booth must be installed on all cluster nodes and on all arbitrators that will
be part of the multi-site cluster.

\item {} 
Nodes belonging to the same cluster site should be synchronized via NTP. However,
time synchronization is not required between the individual cluster sites.

\end{itemize}


\subsubsection{General Management of Tickets}
\label{\detokenize{multi-site-clusters:general-management-of-tickets}}
Display the information of tickets:
\begin{quote}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYGZsh{} crm\PYGZus{}ticket \PYGZhy{}\PYGZhy{}info
\end{sphinxVerbatim}
\end{quote}

Or you can monitor them with:
\begin{quote}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYGZsh{} crm\PYGZus{}mon \PYGZhy{}\PYGZhy{}tickets
\end{sphinxVerbatim}
\end{quote}

Display the \sphinxcode{\sphinxupquote{rsc\_ticket}} constraints that apply to a ticket:
\begin{quote}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYGZsh{} crm\PYGZus{}ticket \PYGZhy{}\PYGZhy{}ticket ticketA \PYGZhy{}\PYGZhy{}constraints
\end{sphinxVerbatim}
\end{quote}

When you want to do maintenance or manual switch-over of a ticket,
revoking the ticket would trigger the loss policies. If
\sphinxcode{\sphinxupquote{loss-policy="fence"}}, the dependent resources could not be gracefully
stopped/demoted, and other unrelated resources could even be affected.

The proper way is making the ticket \sphinxstyleemphasis{standby} first with:
\begin{quote}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYGZsh{} crm\PYGZus{}ticket \PYGZhy{}\PYGZhy{}ticket ticketA \PYGZhy{}\PYGZhy{}standby
\end{sphinxVerbatim}
\end{quote}

Then the dependent resources will be stopped or demoted gracefully without
triggering the loss policies.

If you have finished the maintenance and want to activate the ticket again,
you can run:
\begin{quote}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYGZsh{} crm\PYGZus{}ticket \PYGZhy{}\PYGZhy{}ticket ticketA \PYGZhy{}\PYGZhy{}activate
\end{sphinxVerbatim}
\end{quote}


\subsection{For more information}
\label{\detokenize{multi-site-clusters:for-more-information}}\begin{itemize}
\item {} 
\sphinxhref{https://www.suse.com/documentation/sle-ha-geo-12/art\_ha\_geo\_quick/data/art\_ha\_geo\_quick.html}{SUSE’s Geo Clustering quick start}

\item {} 
\sphinxhref{https://github.com/ClusterLabs/booth}{Booth}

\end{itemize}


\section{Sample Configurations}
\label{\detokenize{ap-samples:sample-configurations}}\label{\detokenize{ap-samples::doc}}

\subsection{Empty}
\label{\detokenize{ap-samples:empty}}
\begin{sphinxShadowBox}
\sphinxstyletopictitle{An Empty Configuration}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}cib} \PYG{n+na}{crm\PYGZus{}feature\PYGZus{}set=}\PYG{l+s}{\PYGZdq{}3.0.7\PYGZdq{}} \PYG{n+na}{validate\PYGZhy{}with=}\PYG{l+s}{\PYGZdq{}pacemaker\PYGZhy{}1.2\PYGZdq{}} \PYG{n+na}{admin\PYGZus{}epoch=}\PYG{l+s}{\PYGZdq{}1\PYGZdq{}} \PYG{n+na}{epoch=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{num\PYGZus{}updates=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}configuration}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}crm\PYGZus{}config}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nodes}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}resources}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}constraints}\PYG{n+nt}{/\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}/configuration\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}status}\PYG{n+nt}{/\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/cib\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}


\subsection{Simple}
\label{\detokenize{ap-samples:simple}}
\begin{sphinxShadowBox}
\sphinxstyletopictitle{A simple configuration with two nodes, some cluster options and a resource}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}cib} \PYG{n+na}{crm\PYGZus{}feature\PYGZus{}set=}\PYG{l+s}{\PYGZdq{}3.0.7\PYGZdq{}} \PYG{n+na}{validate\PYGZhy{}with=}\PYG{l+s}{\PYGZdq{}pacemaker\PYGZhy{}1.2\PYGZdq{}} \PYG{n+na}{admin\PYGZus{}epoch=}\PYG{l+s}{\PYGZdq{}1\PYGZdq{}} \PYG{n+na}{epoch=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{num\PYGZus{}updates=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}configuration}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}crm\PYGZus{}config}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}cluster\PYGZus{}property\PYGZus{}set} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}cib\PYGZhy{}bootstrap\PYGZhy{}options\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}option\PYGZhy{}1\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}symmetric\PYGZhy{}cluster\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}true\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}option\PYGZhy{}2\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}no\PYGZhy{}quorum\PYGZhy{}policy\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}stop\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}option\PYGZhy{}3\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}stonith\PYGZhy{}enabled\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}/cluster\PYGZus{}property\PYGZus{}set\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/crm\PYGZus{}config\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nodes}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}node} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}xxx\PYGZdq{}} \PYG{n+na}{uname=}\PYG{l+s}{\PYGZdq{}c001n01\PYGZdq{}} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}normal\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}node} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}yyy\PYGZdq{}} \PYG{n+na}{uname=}\PYG{l+s}{\PYGZdq{}c001n02\PYGZdq{}} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}normal\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/nodes\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}resources}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}primitive} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}myAddr\PYGZdq{}} \PYG{n+na}{class=}\PYG{l+s}{\PYGZdq{}ocf\PYGZdq{}} \PYG{n+na}{provider=}\PYG{l+s}{\PYGZdq{}heartbeat\PYGZdq{}} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}IPaddr\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}operations}\PYG{n+nt}{\PYGZgt{}}
          \PYG{n+nt}{\PYGZlt{}op} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}myAddr\PYGZhy{}monitor\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}monitor\PYGZdq{}} \PYG{n+na}{interval=}\PYG{l+s}{\PYGZdq{}300s\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}/operations\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}instance\PYGZus{}attributes} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}myAddr\PYGZhy{}params\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
          \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}myAddr\PYGZhy{}ip\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}ip\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}192.0.2.10\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}/instance\PYGZus{}attributes\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}/primitive\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/resources\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}constraints}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}location} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}myAddr\PYGZhy{}prefer\PYGZdq{}} \PYG{n+na}{rsc=}\PYG{l+s}{\PYGZdq{}myAddr\PYGZdq{}} \PYG{n+na}{node=}\PYG{l+s}{\PYGZdq{}c001n01\PYGZdq{}} \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}INFINITY\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/constraints\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}defaults}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}meta\PYGZus{}attributes} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}rsc\PYGZus{}defaults\PYGZhy{}options\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}rsc\PYGZhy{}default\PYGZhy{}1\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}resource\PYGZhy{}stickiness\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}100\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}rsc\PYGZhy{}default\PYGZhy{}2\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}migration\PYGZhy{}threshold\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}10\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}/meta\PYGZus{}attributes\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/rsc\PYGZus{}defaults\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}op\PYGZus{}defaults}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}meta\PYGZus{}attributes} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}op\PYGZus{}defaults\PYGZhy{}options\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}op\PYGZhy{}default\PYGZhy{}1\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}timeout\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}30s\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}/meta\PYGZus{}attributes\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/op\PYGZus{}defaults\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}/configuration\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}status}\PYG{n+nt}{/\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/cib\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}

In the above example, we have one resource (an IP address) that we check
every five minutes and will run on host \sphinxcode{\sphinxupquote{c001n01}} until either the
resource fails 10 times or the host shuts down.


\subsection{Advanced Configuration}
\label{\detokenize{ap-samples:advanced-configuration}}
\begin{sphinxShadowBox}
\sphinxstyletopictitle{An advanced configuration with groups, clones and STONITH}

\fvset{hllines={, ,}}%
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nt}{\PYGZlt{}cib} \PYG{n+na}{crm\PYGZus{}feature\PYGZus{}set=}\PYG{l+s}{\PYGZdq{}3.0.7\PYGZdq{}} \PYG{n+na}{validate\PYGZhy{}with=}\PYG{l+s}{\PYGZdq{}pacemaker\PYGZhy{}1.2\PYGZdq{}} \PYG{n+na}{admin\PYGZus{}epoch=}\PYG{l+s}{\PYGZdq{}1\PYGZdq{}} \PYG{n+na}{epoch=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}} \PYG{n+na}{num\PYGZus{}updates=}\PYG{l+s}{\PYGZdq{}0\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}configuration}\PYG{n+nt}{\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}crm\PYGZus{}config}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}cluster\PYGZus{}property\PYGZus{}set} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}cib\PYGZhy{}bootstrap\PYGZhy{}options\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}option\PYGZhy{}1\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}symmetric\PYGZhy{}cluster\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}true\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}option\PYGZhy{}2\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}no\PYGZhy{}quorum\PYGZhy{}policy\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}stop\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}option\PYGZhy{}3\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}stonith\PYGZhy{}enabled\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}true\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}/cluster\PYGZus{}property\PYGZus{}set\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/crm\PYGZus{}config\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}nodes}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}node} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}xxx\PYGZdq{}} \PYG{n+na}{uname=}\PYG{l+s}{\PYGZdq{}c001n01\PYGZdq{}} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}normal\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}node} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}yyy\PYGZdq{}} \PYG{n+na}{uname=}\PYG{l+s}{\PYGZdq{}c001n02\PYGZdq{}} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}normal\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}node} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}zzz\PYGZdq{}} \PYG{n+na}{uname=}\PYG{l+s}{\PYGZdq{}c001n03\PYGZdq{}} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}normal\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/nodes\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}resources}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}primitive} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}myAddr\PYGZdq{}} \PYG{n+na}{class=}\PYG{l+s}{\PYGZdq{}ocf\PYGZdq{}} \PYG{n+na}{provider=}\PYG{l+s}{\PYGZdq{}heartbeat\PYGZdq{}} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}IPaddr\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}operations}\PYG{n+nt}{\PYGZgt{}}
          \PYG{n+nt}{\PYGZlt{}op} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}myAddr\PYGZhy{}monitor\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}monitor\PYGZdq{}} \PYG{n+na}{interval=}\PYG{l+s}{\PYGZdq{}300s\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}/operations\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}instance\PYGZus{}attributes} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}myAddr\PYGZhy{}attrs\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
          \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}myAddr\PYGZhy{}attr\PYGZhy{}1\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}ip\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}192.0.2.10\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}/instance\PYGZus{}attributes\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}/primitive\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}group} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}myGroup\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}primitive} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}database\PYGZdq{}} \PYG{n+na}{class=}\PYG{l+s}{\PYGZdq{}lsb\PYGZdq{}} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}oracle\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
          \PYG{n+nt}{\PYGZlt{}operations}\PYG{n+nt}{\PYGZgt{}}
            \PYG{n+nt}{\PYGZlt{}op} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}database\PYGZhy{}monitor\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}monitor\PYGZdq{}} \PYG{n+na}{interval=}\PYG{l+s}{\PYGZdq{}300s\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
          \PYG{n+nt}{\PYGZlt{}/operations\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}/primitive\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}primitive} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}webserver\PYGZdq{}} \PYG{n+na}{class=}\PYG{l+s}{\PYGZdq{}lsb\PYGZdq{}} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}apache\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
          \PYG{n+nt}{\PYGZlt{}operations}\PYG{n+nt}{\PYGZgt{}}
            \PYG{n+nt}{\PYGZlt{}op} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}webserver\PYGZhy{}monitor\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}monitor\PYGZdq{}} \PYG{n+na}{interval=}\PYG{l+s}{\PYGZdq{}300s\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
          \PYG{n+nt}{\PYGZlt{}/operations\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}/primitive\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}/group\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}clone} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}STONITH\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}meta\PYGZus{}attributes} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}stonith\PYGZhy{}options\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
          \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}stonith\PYGZhy{}option\PYGZhy{}1\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}globally\PYGZhy{}unique\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}false\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}/meta\PYGZus{}attributes\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}primitive} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}stonithclone\PYGZdq{}} \PYG{n+na}{class=}\PYG{l+s}{\PYGZdq{}stonith\PYGZdq{}} \PYG{n+na}{type=}\PYG{l+s}{\PYGZdq{}external/ssh\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
          \PYG{n+nt}{\PYGZlt{}operations}\PYG{n+nt}{\PYGZgt{}}
            \PYG{n+nt}{\PYGZlt{}op} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}stonith\PYGZhy{}op\PYGZhy{}mon\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}monitor\PYGZdq{}} \PYG{n+na}{interval=}\PYG{l+s}{\PYGZdq{}5s\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
          \PYG{n+nt}{\PYGZlt{}/operations\PYGZgt{}}
          \PYG{n+nt}{\PYGZlt{}instance\PYGZus{}attributes} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}stonith\PYGZhy{}attrs\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
            \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}stonith\PYGZhy{}attr\PYGZhy{}1\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}hostlist\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}c001n01,c001n02\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
          \PYG{n+nt}{\PYGZlt{}/instance\PYGZus{}attributes\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}/primitive\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}/clone\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/resources\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}constraints}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}location} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}myAddr\PYGZhy{}prefer\PYGZdq{}} \PYG{n+na}{rsc=}\PYG{l+s}{\PYGZdq{}myAddr\PYGZdq{}} \PYG{n+na}{node=}\PYG{l+s}{\PYGZdq{}c001n01\PYGZdq{}}
        \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}INFINITY\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}colocation} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}group\PYGZhy{}with\PYGZhy{}ip\PYGZdq{}} \PYG{n+na}{rsc=}\PYG{l+s}{\PYGZdq{}myGroup\PYGZdq{}} \PYG{n+na}{with\PYGZhy{}rsc=}\PYG{l+s}{\PYGZdq{}myAddr\PYGZdq{}}
        \PYG{n+na}{score=}\PYG{l+s}{\PYGZdq{}INFINITY\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/constraints\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}op\PYGZus{}defaults}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}meta\PYGZus{}attributes} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}op\PYGZus{}defaults\PYGZhy{}options\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}op\PYGZhy{}default\PYGZhy{}1\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}timeout\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}30s\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}/meta\PYGZus{}attributes\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/op\PYGZus{}defaults\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}rsc\PYGZus{}defaults}\PYG{n+nt}{\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}meta\PYGZus{}attributes} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}rsc\PYGZus{}defaults\PYGZhy{}options\PYGZdq{}}\PYG{n+nt}{\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}rsc\PYGZhy{}default\PYGZhy{}1\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}resource\PYGZhy{}stickiness\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}100\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
        \PYG{n+nt}{\PYGZlt{}nvpair} \PYG{n+na}{id=}\PYG{l+s}{\PYGZdq{}rsc\PYGZhy{}default\PYGZhy{}2\PYGZdq{}} \PYG{n+na}{name=}\PYG{l+s}{\PYGZdq{}migration\PYGZhy{}threshold\PYGZdq{}} \PYG{n+na}{value=}\PYG{l+s}{\PYGZdq{}10\PYGZdq{}}\PYG{n+nt}{/\PYGZgt{}}
      \PYG{n+nt}{\PYGZlt{}/meta\PYGZus{}attributes\PYGZgt{}}
    \PYG{n+nt}{\PYGZlt{}/rsc\PYGZus{}defaults\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}/configuration\PYGZgt{}}
  \PYG{n+nt}{\PYGZlt{}status}\PYG{n+nt}{/\PYGZgt{}}
\PYG{n+nt}{\PYGZlt{}/cib\PYGZgt{}}
\end{sphinxVerbatim}
\end{sphinxShadowBox}


\chapter{Index}
\label{\detokenize{index:index}}\begin{itemize}
\item {} 
\DUrole{xref,std,std-ref}{genindex}

\item {} 
\DUrole{xref,std,std-ref}{search}

\end{itemize}



\renewcommand{\indexname}{Index}
\printindex
\end{document}