38 #include <sys/types.h>
44 #include <qb/qbipc_common.h>
75 static uint8_t qdevice_can_operate = 1;
76 static void *qdevice_reg_conn = NULL;
77 static uint8_t qdevice_master_wins = 0;
79 static uint8_t two_node = 0;
81 static uint8_t wait_for_all = 0;
82 static uint8_t wait_for_all_status = 0;
85 static int lowest_node_id = -1;
86 static int highest_node_id = -1;
88 #define DEFAULT_LMS_WIN 10000
89 static uint8_t last_man_standing = 0;
92 static uint8_t allow_downscale = 0;
93 static uint32_t ev_barrier = 0;
95 static uint8_t ev_tracking = 0;
96 static uint32_t ev_tracking_barrier = 0;
97 static int ev_tracking_fd = -1;
112 struct qb_ipc_request_header header __attribute__((aligned(8)));
122 struct qb_ipc_request_header header __attribute__((aligned(8)));
128 struct qb_ipc_request_header header __attribute__((aligned(8)));
143 #define MESSAGE_REQ_EXEC_VOTEQUORUM_NODEINFO 0
144 #define MESSAGE_REQ_EXEC_VOTEQUORUM_RECONFIGURE 1
145 #define MESSAGE_REQ_EXEC_VOTEQUORUM_QDEVICE_REG 2
146 #define MESSAGE_REQ_EXEC_VOTEQUORUM_QDEVICE_RECONFIGURE 3
148 static void votequorum_exec_send_expectedvotes_notification(
void);
149 static int votequorum_exec_send_quorum_notification(
void *conn, uint64_t context);
151 #define VOTEQUORUM_RECONFIG_PARAM_EXPECTED_VOTES 1
152 #define VOTEQUORUM_RECONFIG_PARAM_NODE_VOTES 2
153 #define VOTEQUORUM_RECONFIG_PARAM_CANCEL_WFA 3
155 static int votequorum_exec_send_reconfigure(uint8_t
param,
unsigned int nodeid, uint32_t
value);
160 #define VOTEQUORUM_QDEVICE_OPERATION_UNREGISTER 0
161 #define VOTEQUORUM_QDEVICE_OPERATION_REGISTER 1
167 #define NODE_FLAGS_QUORATE 1
168 #define NODE_FLAGS_LEAVING 2
169 #define NODE_FLAGS_WFASTATUS 4
170 #define NODE_FLAGS_FIRST 8
171 #define NODE_FLAGS_QDEVICE_REGISTERED 16
172 #define NODE_FLAGS_QDEVICE_ALIVE 32
173 #define NODE_FLAGS_QDEVICE_CAST_VOTE 64
174 #define NODE_FLAGS_QDEVICE_MASTER_WINS 128
195 static uint8_t quorum;
196 static uint8_t cluster_is_quorate;
203 static struct list_head cluster_members_list;
207 static int quorum_members_entries = 0;
208 static int previous_quorum_members_entries = 0;
209 static int atb_nodelist_entries = 0;
216 static int cluster_nodes_entries = 0;
236 static int qdevice_timer_set = 0;
238 static int last_man_standing_timer_set = 0;
239 static int sync_nodeinfo_sent = 0;
240 static int sync_wait_for_poll_or_timeout = 0;
246 static int sync_in_progress = 0;
248 static void votequorum_sync_init (
249 const unsigned int *trans_list,
250 size_t trans_list_entries,
251 const unsigned int *member_list,
252 size_t member_list_entries,
255 static int votequorum_sync_process (
void);
256 static void votequorum_sync_activate (
void);
257 static void votequorum_sync_abort (
void);
266 static int votequorum_exec_exit_fn (
void);
267 static int votequorum_exec_send_nodeinfo(uint32_t
nodeid);
269 static void message_handler_req_exec_votequorum_nodeinfo (
272 static void exec_votequorum_nodeinfo_endian_convert (
void *message);
274 static void message_handler_req_exec_votequorum_reconfigure (
277 static void exec_votequorum_reconfigure_endian_convert (
void *message);
279 static void message_handler_req_exec_votequorum_qdevice_reg (
282 static void exec_votequorum_qdevice_reg_endian_convert (
void *message);
284 static void message_handler_req_exec_votequorum_qdevice_reconfigure (
287 static void exec_votequorum_qdevice_reconfigure_endian_convert (
void *message);
293 .exec_endian_convert_fn = exec_votequorum_nodeinfo_endian_convert
296 .exec_handler_fn = message_handler_req_exec_votequorum_reconfigure,
297 .exec_endian_convert_fn = exec_votequorum_reconfigure_endian_convert
300 .exec_handler_fn = message_handler_req_exec_votequorum_qdevice_reg,
301 .exec_endian_convert_fn = exec_votequorum_qdevice_reg_endian_convert
304 .exec_handler_fn = message_handler_req_exec_votequorum_qdevice_reconfigure,
305 .exec_endian_convert_fn = exec_votequorum_qdevice_reconfigure_endian_convert
313 static int quorum_lib_init_fn (
void *conn);
315 static int quorum_lib_exit_fn (
void *conn);
317 static void qdevice_timer_fn(
void *arg);
319 static void message_handler_req_lib_votequorum_getinfo (
void *conn,
320 const void *message);
322 static void message_handler_req_lib_votequorum_setexpected (
void *conn,
323 const void *message);
325 static void message_handler_req_lib_votequorum_setvotes (
void *conn,
326 const void *message);
328 static void message_handler_req_lib_votequorum_trackstart (
void *conn,
329 const void *message);
331 static void message_handler_req_lib_votequorum_trackstop (
void *conn,
332 const void *message);
334 static void message_handler_req_lib_votequorum_qdevice_register (
void *conn,
335 const void *message);
337 static void message_handler_req_lib_votequorum_qdevice_unregister (
void *conn,
338 const void *message);
340 static void message_handler_req_lib_votequorum_qdevice_update (
void *conn,
341 const void *message);
343 static void message_handler_req_lib_votequorum_qdevice_poll (
void *conn,
344 const void *message);
346 static void message_handler_req_lib_votequorum_qdevice_master_wins (
void *conn,
347 const void *message);
356 .lib_handler_fn = message_handler_req_lib_votequorum_setexpected,
360 .lib_handler_fn = message_handler_req_lib_votequorum_setvotes,
364 .lib_handler_fn = message_handler_req_lib_votequorum_trackstart,
368 .lib_handler_fn = message_handler_req_lib_votequorum_trackstop,
372 .lib_handler_fn = message_handler_req_lib_votequorum_qdevice_register,
376 .lib_handler_fn = message_handler_req_lib_votequorum_qdevice_unregister,
380 .lib_handler_fn = message_handler_req_lib_votequorum_qdevice_update,
384 .lib_handler_fn = message_handler_req_lib_votequorum_qdevice_poll,
388 .lib_handler_fn = message_handler_req_lib_votequorum_qdevice_master_wins,
394 .
name =
"corosync vote quorum service v1.0",
397 .private_data_size =
sizeof (
struct quorum_pd),
400 .lib_init_fn = quorum_lib_init_fn,
401 .lib_exit_fn = quorum_lib_exit_fn,
402 .lib_engine = quorum_lib_service,
404 .exec_init_fn = votequorum_exec_init_fn,
405 .exec_exit_fn = votequorum_exec_exit_fn,
406 .exec_engine = votequorum_exec_engine,
408 .sync_init = votequorum_sync_init,
409 .sync_process = votequorum_sync_process,
410 .sync_activate = votequorum_sync_activate,
411 .sync_abort = votequorum_sync_abort
416 return (&votequorum_service_engine);
421 .
name =
"corosync_votequorum",
431 #define max(a,b) (((a) > (b)) ? (a) : (b))
433 #define list_iterate(v, head) \
434 for (v = (head)->next; v != head; v = v->next)
436 static void node_add_ordered(
struct cluster_node *newnode)
452 list_add(&newnode->
list, &cluster_members_list);
471 cl = (
struct cluster_node *)&cluster_nodes[cluster_nodes_entries];
472 cluster_nodes_entries++;
493 node_add_ordered(cl);
502 static struct cluster_node *find_node_by_nodeid(
unsigned int nodeid)
531 static void get_lowest_node_id(
void)
543 (node->
node_id < lowest_node_id)) {
544 lowest_node_id = node->
node_id;
553 static void get_highest_node_id(
void)
565 (node->
node_id > highest_node_id)) {
566 highest_node_id = node->
node_id;
575 static int check_low_node_id_partition(
void)
586 (node->
node_id == lowest_node_id)) {
595 static int check_high_node_id_partition(
void)
606 (node->
node_id == highest_node_id)) {
615 static int is_in_nodelist(
int nodeid,
unsigned int *members,
int entries)
620 for (i=0; i<entries; i++) {
621 if (nodeid == members[i]) {
643 static int check_auto_tie_breaker(
void)
650 res = check_low_node_id_partition();
656 res = check_high_node_id_partition();
663 for (i=0; i < atb_nodelist_entries; i++) {
664 if (is_in_nodelist(atb_nodelist[i], quorum_members, quorum_members_entries)) {
671 for (j=0; j<i; j++) {
672 if (is_in_nodelist(atb_nodelist[j], previous_quorum_members, previous_quorum_members_entries)) {
699 static void parse_atb_string(
char *atb_string)
707 if (!strcmp(atb_string,
"lowest"))
710 if (!strcmp(atb_string,
"highest"))
713 if (atoi(atb_string)) {
715 atb_nodelist_entries = 0;
718 num = strtol(ptr, &ptr, 10);
721 atb_nodelist[atb_nodelist_entries++] = num;
725 if (atb_nodelist_entries) {
734 log_printf(
LOGSYS_LEVEL_WARNING,
"auto_tie_breaker_nodes is not valid. It must be 'lowest', 'highest' or a space-separated list of node IDs. auto_tie_breaker is disabled");
740 static int check_qdevice_master(
void)
761 static void decode_flags(uint32_t
flags)
766 "flags: quorate: %s Leaving: %s WFA Status: %s First: %s Qdevice: %s QdeviceAlive: %s QdeviceCastVote: %s QdeviceMasterWins: %s",
782 static int load_ev_tracking_barrier(
void)
785 char filename[PATH_MAX];
789 snprintf(filename,
sizeof(filename) - 1,
"%s/ev_tracking",
get_run_dir());
791 ev_tracking_fd = open(filename, O_RDWR, 0700);
792 if (ev_tracking_fd != -1) {
793 res = read (ev_tracking_fd, &ev_tracking_barrier,
sizeof(uint32_t));
794 if (res ==
sizeof (uint32_t)) {
800 ev_tracking_barrier = 0;
802 ev_tracking_fd = open (filename, O_CREAT|O_RDWR, 0700);
803 if (ev_tracking_fd != -1) {
804 res = write (ev_tracking_fd, &ev_tracking_barrier,
sizeof (uint32_t));
805 if ((res == -1) || (res !=
sizeof (uint32_t))) {
807 "Unable to write to %s", filename);
813 "Unable to create %s file", filename);
820 static void update_wait_for_all_status(uint8_t wfa_status)
824 wait_for_all_status = wfa_status;
825 if (wait_for_all_status) {
828 us->
flags &= ~NODE_FLAGS_WFASTATUS;
831 wait_for_all_status);
836 static void update_two_node(
void)
855 static void update_qdevice_can_operate(uint8_t status)
859 qdevice_can_operate = status;
860 icmap_set_uint8(
"runtime.votequorum.qdevice_can_operate", qdevice_can_operate);
865 static void update_qdevice_master_wins(uint8_t allow)
869 qdevice_master_wins = allow;
870 icmap_set_uint8(
"runtime.votequorum.qdevice_master_wins", qdevice_master_wins);
875 static void update_ev_tracking_barrier(uint32_t ev_t_barrier)
881 ev_tracking_barrier = ev_t_barrier;
882 icmap_set_uint32(
"runtime.votequorum.ev_tracking_barrier", ev_tracking_barrier);
884 if (lseek (ev_tracking_fd, 0, SEEK_SET) != 0) {
886 "Unable to update ev_tracking_barrier on disk data!!!");
891 res = write (ev_tracking_fd, &ev_tracking_barrier,
sizeof (uint32_t));
892 if (res !=
sizeof (uint32_t)) {
894 "Unable to update ev_tracking_barrier on disk data!!!");
896 fdatasync(ev_tracking_fd);
905 static int calculate_quorum(
int allow_decrease,
unsigned int max_expected,
unsigned int *ret_total_votes)
909 unsigned int total_votes = 0;
910 unsigned int highest_expected = 0;
911 unsigned int newquorum, q1, q2;
912 unsigned int total_nodes = 0;
916 if ((allow_downscale) && (allow_decrease) && (max_expected)) {
917 max_expected =
max(ev_barrier, max_expected);
932 total_votes += node->
votes;
939 total_votes += qdevice->
votes;
943 if (max_expected > 0) {
944 highest_expected = max_expected;
951 q1 = (highest_expected + 2) / 2;
952 q2 = (total_votes + 2) / 2;
953 newquorum =
max(q1, q2);
959 if (!allow_decrease) {
960 newquorum =
max(quorum, newquorum);
970 if (two_node && total_nodes <= 2) {
974 if (ret_total_votes) {
975 *ret_total_votes = total_votes;
982 static void are_we_quorate(
unsigned int total_votes)
985 int quorum_change = 0;
993 if ((wait_for_all) && (wait_for_all_status)) {
996 "Waiting for all cluster members. "
997 "Current votes: %d expected_votes: %d",
999 cluster_is_quorate = 0;
1002 update_wait_for_all_status(0);
1005 if (quorum > total_votes) {
1009 get_lowest_node_id();
1010 get_highest_node_id();
1013 if ((auto_tie_breaker !=
ATB_NONE) &&
1015 (check_auto_tie_breaker() == 1)) {
1019 if ((qdevice_master_wins) &&
1021 (check_qdevice_master() == 1)) {
1026 if (cluster_is_quorate && !quorate) {
1030 if (!cluster_is_quorate && quorate) {
1036 if (cluster_is_quorate) {
1039 us->
flags &= ~NODE_FLAGS_QUORATE;
1044 update_wait_for_all_status(0);
1046 update_wait_for_all_status(1);
1050 if ((quorum_change) &&
1051 (sync_in_progress == 0)) {
1052 quorum_callback(quorum_members, quorum_members_entries,
1053 cluster_is_quorate, &quorum_ringid);
1059 static void get_total_votes(
unsigned int *totalvotes,
unsigned int *current_members)
1061 unsigned int total_votes = 0;
1062 unsigned int cluster_members = 0;
1072 total_votes += node->
votes;
1076 if (qdevice->
votes) {
1077 total_votes += qdevice->
votes;
1081 *totalvotes = total_votes;
1082 *current_members = cluster_members;
1090 static void recalculate_quorum(
int allow_decrease,
int by_current_nodes)
1092 unsigned int total_votes = 0;
1093 unsigned int cluster_members = 0;
1097 get_total_votes(&total_votes, &cluster_members);
1099 if (!by_current_nodes) {
1100 cluster_members = 0;
1109 votequorum_exec_send_expectedvotes_notification();
1112 if ((ev_tracking) &&
1117 quorum = calculate_quorum(allow_decrease, cluster_members, &total_votes);
1118 are_we_quorate(total_votes);
1120 votequorum_exec_send_quorum_notification(NULL, 0L);
1129 static int votequorum_read_nodelist_configuration(uint32_t *
votes,
1134 const char *iter_key;
1136 uint32_t our_pos, node_pos;
1137 uint32_t nodecount = 0;
1138 uint32_t nodelist_expected_votes = 0;
1139 uint32_t node_votes = 0;
1146 "No nodelist defined or our node is not in the nodelist");
1154 res = sscanf(iter_key,
"nodelist.node.%u.%s", &node_pos, tmp_key);
1159 if (strcmp(tmp_key,
"ring0_addr") != 0) {
1170 nodelist_expected_votes = nodelist_expected_votes + node_votes;
1172 if (node_pos == our_pos) {
1173 *votes = node_votes;
1177 *expected_votes = nodelist_expected_votes;
1187 static int votequorum_qdevice_is_configured(uint32_t *qdevice_votes)
1189 char *qdevice_model = NULL;
1195 if (strlen(qdevice_model)) {
1197 *qdevice_votes = -1;
1205 update_qdevice_can_operate(1);
1209 free(qdevice_model);
1217 #define VOTEQUORUM_READCONFIG_STARTUP 0
1218 #define VOTEQUORUM_READCONFIG_RUNTIME 1
1220 static char *votequorum_readconfig(
int runtime)
1222 uint32_t node_votes = 0, qdevice_votes = 0;
1223 uint32_t node_expected_votes = 0, expected_votes = 0;
1224 uint32_t node_count = 0;
1226 int have_nodelist, have_qdevice;
1227 char *atb_string = NULL;
1238 have_nodelist = votequorum_read_nodelist_configuration(&node_votes, &node_count, &node_expected_votes);
1239 have_qdevice = votequorum_qdevice_is_configured(&qdevice_votes);
1246 if ((!have_nodelist) && (!expected_votes)) {
1248 error = (
char *)
"configuration error: nodelist or quorum.expected_votes must be configured!";
1261 if ((two_node) && (have_qdevice)) {
1263 error = (
char *)
"configuration error: two_node and quorum device cannot be configured at the same time!";
1267 if (us->
flags & NODE_FLAGS_QDEVICE_REGISTERED) {
1272 update_qdevice_can_operate(0);
1288 icmap_get_uint32(
"quorum.last_man_standing_window", &last_man_standing_window);
1297 "auto_tie_breaker_node: is meaningless if auto_tie_breaker is set to 0");
1301 if (atb && atb_string) {
1302 parse_atb_string(atb_string);
1307 if (allow_downscale) {
1312 if (load_ev_tracking_barrier() < 0) {
1314 return ((
char *)
"Unable to load ev_tracking file!");
1316 update_ev_tracking_barrier(ev_tracking_barrier);
1328 if ((have_qdevice) && (last_man_standing)) {
1330 error = (
char *)
"configuration error: quorum.device is not compatible with last_man_standing";
1335 update_qdevice_can_operate(0);
1339 if ((have_qdevice) && (auto_tie_breaker !=
ATB_NONE)) {
1341 error = (
char *)
"configuration error: quorum.device is not compatible with auto_tie_breaker";
1346 update_qdevice_can_operate(0);
1350 if ((have_qdevice) && (wait_for_all)) {
1352 error = (
char *)
"configuration error: quorum.device is not compatible with wait_for_all";
1357 update_qdevice_can_operate(0);
1361 if ((have_qdevice) && (allow_downscale)) {
1363 error = (
char *)
"configuration error: quorum.device is not compatible with allow_downscale";
1368 update_qdevice_can_operate(0);
1377 if ((expected_votes) && (have_qdevice) && (qdevice_votes == -1)) {
1379 error = (
char *)
"configuration error: quorum.device.votes must be specified when quorum.expected_votes is set";
1384 update_qdevice_can_operate(0);
1393 if ((have_qdevice) &&
1394 (qdevice_votes == -1) &&
1396 (node_count != node_expected_votes)) {
1398 error = (
char *)
"configuration error: quorum.device.votes must be specified when not all nodes votes 1";
1403 update_qdevice_can_operate(0);
1411 if ((qdevice_votes > 0) && (expected_votes)) {
1412 int delta = expected_votes - qdevice_votes;
1415 error = (
char *)
"configuration error: quorum.device.votes is too high or expected_votes is too low";
1420 update_qdevice_can_operate(0);
1429 if ((have_qdevice) &&
1430 (qdevice_votes == -1) &&
1431 (!expected_votes) &&
1433 (node_count == node_expected_votes)) {
1434 qdevice_votes = node_expected_votes - 1;
1435 node_expected_votes = node_expected_votes + qdevice_votes;
1441 log_printf(
LOGSYS_LEVEL_DEBUG,
"ev_tracking=%d, ev_tracking_barrier = %d: expected_votes = %d\n", ev_tracking, ev_tracking_barrier, expected_votes);
1444 expected_votes = ev_tracking_barrier;
1447 if (have_nodelist) {
1448 us->
votes = node_votes;
1455 if (expected_votes) {
1463 if (!have_qdevice) {
1467 if (qdevice_votes != -1) {
1468 qdevice->
votes = qdevice_votes;
1474 update_wait_for_all_status(1);
1482 static void votequorum_refresh_config(
1484 const char *key_name,
1489 int old_votes, old_expected_votes;
1499 if (
icmap_get_uint8(
"config.totemconfig_reload_in_progress", &reloading) ==
CS_OK && reloading) {
1504 if (strcmp(key_name,
"quorum.cancel_wait_for_all") == 0 &&
1512 old_votes = us->
votes;
1523 votequorum_exec_send_nodeinfo(us->
node_id);
1525 if (us->
votes != old_votes) {
1537 static void votequorum_exec_add_config_notification(
void)
1547 votequorum_refresh_config,
1549 &icmap_track_nodelist);
1553 votequorum_refresh_config,
1555 &icmap_track_quorum);
1559 votequorum_refresh_config,
1561 &icmap_track_reload);
1570 static int votequorum_exec_send_reconfigure(uint8_t
param,
unsigned int nodeid, uint32_t
value)
1573 struct iovec iov[1];
1597 static int votequorum_exec_send_nodeinfo(uint32_t nodeid)
1600 struct iovec iov[1];
1606 node = find_node_by_nodeid(nodeid);
1616 decode_flags(node->
flags);
1631 static int votequorum_exec_send_qdevice_reconfigure(
const char *
oldname,
const char *
newname)
1634 struct iovec iov[1];
1653 static int votequorum_exec_send_qdevice_reg(uint32_t
operation,
const char *qdevice_name_req)
1656 struct iovec iov[1];
1675 static int votequorum_exec_send_quorum_notification(
void *conn, uint64_t context)
1680 int cluster_members = 0;
1691 if (us->
flags & NODE_FLAGS_QDEVICE_REGISTERED) {
1695 size =
sizeof(
struct res_lib_votequorum_notification) + sizeof(struct
votequorum_node) * cluster_members;
1697 res_lib_votequorum_notification = (
struct res_lib_votequorum_notification *)&buf;
1698 res_lib_votequorum_notification->quorate = cluster_is_quorate;
1699 res_lib_votequorum_notification->node_list_entries = cluster_members;
1700 res_lib_votequorum_notification->ring_id.nodeid = quorum_ringid.
rep.
nodeid;
1701 res_lib_votequorum_notification->ring_id.seq = quorum_ringid.
seq;
1702 res_lib_votequorum_notification->context = context;
1705 res_lib_votequorum_notification->node_list[i].nodeid = node->
node_id;
1706 res_lib_votequorum_notification->node_list[i++].state = node->
state;
1708 if (us->
flags & NODE_FLAGS_QDEVICE_REGISTERED) {
1710 res_lib_votequorum_notification->node_list[i++].state = qdevice->
state;
1713 res_lib_votequorum_notification->header.size = size;
1714 res_lib_votequorum_notification->header.error =
CS_OK;
1736 static void votequorum_exec_send_expectedvotes_notification(
void)
1761 static void exec_votequorum_qdevice_reconfigure_endian_convert (
void *message)
1768 static void message_handler_req_exec_votequorum_qdevice_reconfigure (
1769 const void *message,
1770 unsigned int nodeid)
1778 req_exec_quorum_qdevice_reconfigure->
oldname,
1779 req_exec_quorum_qdevice_reconfigure->
newname);
1781 if (!strcmp(req_exec_quorum_qdevice_reconfigure->
oldname, qdevice_name)) {
1784 strcpy(qdevice_name, req_exec_quorum_qdevice_reconfigure->
newname);
1795 static void exec_votequorum_qdevice_reg_endian_convert (
void *message)
1806 static void message_handler_req_exec_votequorum_qdevice_reg (
1807 const void *message,
1808 unsigned int nodeid)
1812 int wipe_qdevice_name = 1;
1823 switch(req_exec_quorum_qdevice_reg->
operation)
1827 if (!strlen(qdevice_name)) {
1829 strcpy(qdevice_name, req_exec_quorum_qdevice_reg->
qdevice_name);
1840 if (us->
flags & NODE_FLAGS_QDEVICE_REGISTERED) {
1848 if (!qdevice_reg_conn) {
1857 if (!strlen(qdevice_name)) {
1858 strcpy(qdevice_name, req_exec_quorum_qdevice_reg->
qdevice_name);
1864 if ((!strncmp(req_exec_quorum_qdevice_reg->
qdevice_name,
1868 votequorum_exec_send_nodeinfo(us->
node_id);
1871 "A new qdevice with different name (new: %s old: %s) is trying to register!",
1872 req_exec_quorum_qdevice_reg->
qdevice_name, qdevice_name);
1880 qdevice_reg_conn = NULL;
1886 (node->
flags & NODE_FLAGS_QDEVICE_REGISTERED)) {
1887 wipe_qdevice_name = 0;
1891 if (wipe_qdevice_name) {
1900 static void exec_votequorum_nodeinfo_endian_convert (
void *message)
1914 static void message_handler_req_exec_votequorum_nodeinfo (
1915 const void *message,
1916 unsigned int sender_nodeid)
1925 int allow_downgrade = 0;
1927 unsigned int nodeid = req_exec_quorum_nodeinfo->
nodeid;
1934 req_exec_quorum_nodeinfo->
votes,
1936 req_exec_quorum_nodeinfo->
flags);
1939 decode_flags(req_exec_quorum_nodeinfo->
flags);
1942 node = find_node_by_nodeid(nodeid);
1944 node = allocate_node(nodeid);
1959 old_votes = node->
votes;
1961 old_state = node->
state;
1962 old_flags = node->
flags;
1966 struct cluster_node *sender_node = find_node_by_nodeid(sender_nodeid);
1968 assert(sender_node != NULL);
1970 if ((!cluster_is_quorate) &&
1971 (sender_node->
flags & NODE_FLAGS_QUORATE)) {
1972 node->
votes = req_exec_quorum_nodeinfo->
votes;
1980 node->
flags = req_exec_quorum_nodeinfo->
flags;
1981 node->
votes = req_exec_quorum_nodeinfo->
votes;
1984 if (node->
flags & NODE_FLAGS_LEAVING) {
1986 allow_downgrade = 1;
1990 if ((!cluster_is_quorate) &&
1991 (node->
flags & NODE_FLAGS_QUORATE)) {
1992 allow_downgrade = 1;
1996 if (node->
flags & NODE_FLAGS_QUORATE || (ev_tracking)) {
2002 if ((last_man_standing) && (node->
votes > 1)) {
2004 "cluster nodes votes are set to 1. Disabling LMS.");
2005 last_man_standing = 0;
2006 if (last_man_standing_timer_set) {
2008 last_man_standing_timer_set = 0;
2015 (node->
flags & NODE_FLAGS_FIRST) ||
2016 (old_votes != node->
votes) ||
2018 (old_flags != node->
flags) ||
2019 (old_state != node->
state)) {
2020 recalculate_quorum(allow_downgrade, by_node);
2023 if ((wait_for_all) &&
2024 (!(node->
flags & NODE_FLAGS_WFASTATUS)) &&
2025 (node->
flags & NODE_FLAGS_QUORATE)) {
2026 update_wait_for_all_status(0);
2032 static void exec_votequorum_reconfigure_endian_convert (
void *message)
2044 static void message_handler_req_exec_votequorum_reconfigure (
2045 const void *message,
2046 unsigned int nodeid)
2055 nodeid, req_exec_quorum_reconfigure->
nodeid);
2057 switch(req_exec_quorum_reconfigure->
param)
2066 votequorum_exec_send_expectedvotes_notification();
2067 update_ev_barrier(req_exec_quorum_reconfigure->
value);
2071 recalculate_quorum(1, 0);
2075 node = find_node_by_nodeid(req_exec_quorum_reconfigure->
nodeid);
2080 node->
votes = req_exec_quorum_reconfigure->
value;
2081 recalculate_quorum(1, 0);
2085 update_wait_for_all_status(0);
2087 req_exec_quorum_reconfigure->
nodeid);
2088 recalculate_quorum(0, 0);
2097 static int votequorum_exec_exit_fn (
void)
2107 if (allow_downscale) {
2109 ret = votequorum_exec_send_nodeinfo(us->
node_id);
2112 if ((ev_tracking) && (ev_tracking_fd != -1)) {
2113 close(ev_tracking_fd);
2130 list_init(&cluster_members_list);
2131 list_init(&trackers_list);
2134 memset(cluster_nodes, 0,
sizeof(cluster_nodes));
2142 return ((
char *)
"Could not allocate node.");
2153 return ((
char *)
"Could not allocate node.");
2166 recalculate_quorum(0, 0);
2171 votequorum_exec_add_config_notification();
2176 votequorum_exec_send_nodeinfo(us->
node_id);
2187 static void votequorum_last_man_standing_timer_fn(
void *arg)
2191 last_man_standing_timer_set = 0;
2192 if (cluster_is_quorate) {
2193 recalculate_quorum(1,1);
2199 static void votequorum_sync_init (
2200 const unsigned int *trans_list,
size_t trans_list_entries,
2201 const unsigned int *member_list,
size_t member_list_entries,
2211 sync_in_progress = 1;
2212 sync_nodeinfo_sent = 0;
2213 sync_wait_for_poll_or_timeout = 0;
2215 if (member_list_entries > 1) {
2216 us->
flags &= ~NODE_FLAGS_FIRST;
2225 for (i = 0; i < quorum_members_entries; i++) {
2227 for (j = 0; j < member_list_entries; j++) {
2228 if (quorum_members[i] == member_list[j]) {
2235 node = find_node_by_nodeid(quorum_members[i]);
2242 if (last_man_standing) {
2243 if (((member_list_entries >= quorum) && (left_nodes)) ||
2244 ((member_list_entries <= quorum) && (auto_tie_breaker !=
ATB_NONE) && (check_low_node_id_partition() == 1))) {
2245 if (last_man_standing_timer_set) {
2247 last_man_standing_timer_set = 0;
2249 corosync_api->
timer_add_duration((
unsigned long long)last_man_standing_window*1000000,
2250 NULL, votequorum_last_man_standing_timer_fn,
2251 &last_man_standing_timer);
2252 last_man_standing_timer_set = 1;
2256 memcpy(previous_quorum_members, quorum_members,
sizeof(
unsigned int) * quorum_members_entries);
2257 previous_quorum_members_entries = quorum_members_entries;
2259 memcpy(quorum_members, member_list,
sizeof(
unsigned int) * member_list_entries);
2260 quorum_members_entries = member_list_entries;
2261 memcpy(&quorum_ringid, ring_id,
sizeof(*ring_id));
2263 if (us->
flags & NODE_FLAGS_QDEVICE_REGISTERED && us->
flags & NODE_FLAGS_QDEVICE_ALIVE) {
2267 if (qdevice_timer_set) {
2270 corosync_api->
timer_add_duration((
unsigned long long)qdevice_sync_timeout*1000000, qdevice,
2271 qdevice_timer_fn, &qdevice_timer);
2272 qdevice_timer_set = 1;
2273 sync_wait_for_poll_or_timeout = 1;
2276 qdevice_name, qdevice_sync_timeout);
2282 static int votequorum_sync_process (
void)
2285 if (!sync_nodeinfo_sent) {
2286 votequorum_exec_send_nodeinfo(us->
node_id);
2288 if (strlen(qdevice_name)) {
2292 sync_nodeinfo_sent = 1;
2295 if (us->
flags & NODE_FLAGS_QDEVICE_REGISTERED && sync_wait_for_poll_or_timeout) {
2306 static void votequorum_sync_activate (
void)
2308 recalculate_quorum(0, 0);
2309 quorum_callback(quorum_members, quorum_members_entries,
2310 cluster_is_quorate, &quorum_ringid);
2311 sync_in_progress = 0;
2314 static void votequorum_sync_abort (
void)
2326 if (q_set_quorate_fn == NULL) {
2327 return ((
char *)
"Quorate function not set");
2331 quorum_callback = q_set_quorate_fn;
2334 &votequorum_service[0]);
2348 static int quorum_lib_init_fn (
void *conn)
2354 list_init (&pd->
list);
2361 static int quorum_lib_exit_fn (
void *conn)
2368 list_del (&quorum_pd->
list);
2369 list_init (&quorum_pd->
list);
2381 static void qdevice_timer_fn(
void *arg)
2385 if ((!(us->
flags & NODE_FLAGS_QDEVICE_ALIVE)) ||
2386 (!qdevice_timer_set)) {
2391 us->
flags &= ~NODE_FLAGS_QDEVICE_ALIVE;
2394 votequorum_exec_send_nodeinfo(us->
node_id);
2396 qdevice_timer_set = 0;
2397 sync_wait_for_poll_or_timeout = 0;
2406 static void message_handler_req_lib_votequorum_getinfo (
void *conn,
const void *message)
2411 unsigned int highest_expected = 0;
2412 unsigned int total_votes = 0;
2414 uint32_t nodeid = req_lib_votequorum_getinfo->
nodeid;
2424 node = find_node_by_nodeid(nodeid);
2435 total_votes += iternode->
votes;
2440 total_votes += qdevice->
votes;
2443 switch(node->
state) {
2470 if (cluster_is_quorate) {
2476 if (last_man_standing) {
2479 if (auto_tie_breaker !=
ATB_NONE) {
2482 if (allow_downscale) {
2490 if (node->
flags & NODE_FLAGS_QDEVICE_REGISTERED) {
2493 if (node->
flags & NODE_FLAGS_QDEVICE_ALIVE) {
2515 static void message_handler_req_lib_votequorum_setexpected (
void *conn,
const void *message)
2520 unsigned int newquorum;
2521 unsigned int total_votes;
2522 uint8_t allow_downscale_status = 0;
2526 allow_downscale_status = allow_downscale;
2527 allow_downscale = 0;
2532 newquorum = calculate_quorum(1, req_lib_votequorum_setexpected->
expected_votes, &total_votes);
2533 allow_downscale = allow_downscale_status;
2534 if (newquorum < total_votes / 2 ||
2535 newquorum > total_votes) {
2552 static void message_handler_req_lib_votequorum_setvotes (
void *conn,
const void *message)
2557 unsigned int newquorum;
2558 unsigned int total_votes;
2559 unsigned int saved_votes;
2565 nodeid = req_lib_votequorum_setvotes->
nodeid;
2566 node = find_node_by_nodeid(nodeid);
2575 saved_votes = node->
votes;
2576 node->
votes = req_lib_votequorum_setvotes->
votes;
2578 newquorum = calculate_quorum(1, 0, &total_votes);
2580 if (newquorum < total_votes / 2 ||
2581 newquorum > total_votes) {
2582 node->
votes = saved_votes;
2588 req_lib_votequorum_setvotes->
votes);
2599 static void message_handler_req_lib_votequorum_trackstart (
void *conn,
2600 const void *message)
2614 votequorum_exec_send_quorum_notification(conn, req_lib_votequorum_trackstart->
context);
2627 list_add (&quorum_pd->
list, &trackers_list);
2638 static void message_handler_req_lib_votequorum_trackstop (
void *conn,
2639 const void *message)
2650 list_del (&quorum_pd->
list);
2651 list_init (&quorum_pd->
list);
2664 static void message_handler_req_lib_votequorum_qdevice_register (
void *conn,
2665 const void *message)
2673 if (!qdevice_can_operate) {
2674 log_printf(
LOGSYS_LEVEL_INFO,
"Registration of quorum device is disabled by incorrect corosync.conf. See logs for more information");
2679 if (us->
flags & NODE_FLAGS_QDEVICE_REGISTERED) {
2680 if ((!strncmp(req_lib_votequorum_qdevice_register->
name,
2685 "A new qdevice with different name (new: %s old: %s) is trying to re-register!",
2686 req_lib_votequorum_qdevice_register->
name, qdevice_name);
2691 if (qdevice_reg_conn != NULL) {
2693 "Registration request already in progress");
2697 qdevice_reg_conn = conn;
2699 req_lib_votequorum_qdevice_register->
name) != 0) {
2701 "Unable to send qdevice registration request to cluster");
2703 qdevice_reg_conn = NULL;
2720 static void message_handler_req_lib_votequorum_qdevice_unregister (
void *conn,
2721 const void *message)
2729 if (us->
flags & NODE_FLAGS_QDEVICE_REGISTERED) {
2734 if (qdevice_timer_set) {
2736 qdevice_timer_set = 0;
2737 sync_wait_for_poll_or_timeout = 0;
2739 us->
flags &= ~NODE_FLAGS_QDEVICE_REGISTERED;
2740 us->
flags &= ~NODE_FLAGS_QDEVICE_ALIVE;
2743 votequorum_exec_send_nodeinfo(us->
node_id);
2745 req_lib_votequorum_qdevice_unregister->
name);
2759 static void message_handler_req_lib_votequorum_qdevice_update (
void *conn,
2760 const void *message)
2768 if (us->
flags & NODE_FLAGS_QDEVICE_REGISTERED) {
2773 votequorum_exec_send_qdevice_reconfigure(req_lib_votequorum_qdevice_update->
oldname,
2774 req_lib_votequorum_qdevice_update->
newname);
2788 static void message_handler_req_lib_votequorum_qdevice_poll (
void *conn,
2789 const void *message)
2798 if (!qdevice_can_operate) {
2803 if (us->
flags & NODE_FLAGS_QDEVICE_REGISTERED) {
2804 if (!(req_lib_votequorum_qdevice_poll->ring_id.nodeid == quorum_ringid.
rep.
nodeid &&
2805 req_lib_votequorum_qdevice_poll->ring_id.seq == quorum_ringid.
seq)) {
2807 "ring id (%u.%"PRIu64
"). Ignoring poll call.",
2808 req_lib_votequorum_qdevice_poll->ring_id.nodeid, req_lib_votequorum_qdevice_poll->ring_id.seq,
2818 if (qdevice_timer_set) {
2820 qdevice_timer_set = 0;
2823 oldflags = us->
flags;
2827 if (req_lib_votequorum_qdevice_poll->
cast_vote) {
2833 if (us->
flags != oldflags) {
2834 votequorum_exec_send_nodeinfo(us->
node_id);
2837 corosync_api->
timer_add_duration((
unsigned long long)qdevice_timeout*1000000, qdevice,
2838 qdevice_timer_fn, &qdevice_timer);
2839 qdevice_timer_set = 1;
2840 sync_wait_for_poll_or_timeout = 0;
2854 static void message_handler_req_lib_votequorum_qdevice_master_wins (
void *conn,
2855 const void *message)
2860 uint32_t oldflags = us->
flags;
2864 if (!qdevice_can_operate) {
2869 if (us->
flags & NODE_FLAGS_QDEVICE_REGISTERED) {
2875 if (req_lib_votequorum_qdevice_master_wins->
allow) {
2881 if (us->
flags != oldflags) {
2882 votequorum_exec_send_nodeinfo(us->
node_id);
2885 update_qdevice_master_wins(req_lib_votequorum_qdevice_master_wins->
allow);
char name[VOTEQUORUM_QDEVICE_MAX_NAME_LEN]
char newname[VOTEQUORUM_QDEVICE_MAX_NAME_LEN]
#define VOTEQUORUM_INFO_QUORATE
char oldname[VOTEQUORUM_QDEVICE_MAX_NAME_LEN]
#define VOTEQUORUM_READCONFIG_STARTUP
const char * get_run_dir(void)
const char * icmap_iter_next(icmap_iter_t iter, size_t *value_len, icmap_value_types_t *type)
#define NODE_FLAGS_WFASTATUS
void(* lib_handler_fn)(void *conn, const void *msg)
#define LOGSYS_LEVEL_INFO
#define NODE_FLAGS_QUORATE
void(* timer_delete)(corosync_timer_handle_t timer_handle)
#define VOTEQUORUM_QDEVICE_DEFAULT_SYNC_TIMEOUT
void icmap_iter_finalize(icmap_iter_t iter)
#define VOTEQUORUM_QDEVICE_OPERATION_UNREGISTER
#define MESSAGE_REQ_EXEC_VOTEQUORUM_RECONFIGURE
int(* timer_add_duration)(unsigned long long nanoseconds_in_future, void *data, void(*timer_nf)(void *data), corosync_timer_handle_t *handle)
#define list_iterate(v, head)
char * votequorum_init(struct corosync_api_v1 *api, quorum_set_quorate_fn_t q_set_quorate_fn)
#define VOTEQUORUM_RECONFIG_PARAM_CANCEL_WFA
char name[VOTEQUORUM_QDEVICE_MAX_NAME_LEN]
char qdevice_name[VOTEQUORUM_QDEVICE_MAX_NAME_LEN]
#define NODE_FLAGS_QDEVICE_MASTER_WINS
unsigned int highest_expected
#define VOTEQUORUM_INFO_LAST_MAN_STANDING
struct message_header header
#define VOTEQUORUM_INFO_WAIT_FOR_ALL
#define NODE_FLAGS_QDEVICE_CAST_VOTE
char name[VOTEQUORUM_QDEVICE_MAX_NAME_LEN]
#define MESSAGE_REQ_EXEC_VOTEQUORUM_QDEVICE_RECONFIGURE
#define VOTEQUORUM_INFO_TWONODE
char qdevice_name[VOTEQUORUM_QDEVICE_MAX_NAME_LEN]
#define VOTEQUORUM_INFO_QDEVICE_REGISTERED
#define log_printf(level, format, args...)
void(* exec_handler_fn)(const void *msg, unsigned int nodeid)
unsigned int expected_votes
#define VOTEQUORUM_QDEVICE_NODEID
#define VOTEQUORUM_INFO_QDEVICE_MASTER_WINS
#define VOTEQUORUM_NODESTATE_MEMBER
#define SERVICE_ID_MAKE(a, b)
#define ICMAP_TRACK_DELETE
#define ICMAP_KEYNAME_MAXLEN
#define VOTEQUORUM_QDEVICE_OPERATION_REGISTER
cs_error_t icmap_get_uint8(const char *key_name, uint8_t *u8)
#define VOTEQUORUM_INFO_ALLOW_DOWNSCALE
#define LOGSYS_LEVEL_WARNING
#define ICMAP_TRACK_MODIFY
#define VOTEQUORUM_INFO_QDEVICE_ALIVE
void *(* ipc_private_data_get)(void *conn)
cs_error_t icmap_set_uint32(const char *key_name, uint32_t value)
#define CS_TRACK_CHANGES_ONLY
char name[VOTEQUORUM_QDEVICE_MAX_NAME_LEN]
struct totem_ip_address rep
#define COROSYNC_LIB_FLOW_CONTROL_NOT_REQUIRED
unsigned char track_flags
#define LOGSYS_LEVEL_DEBUG
LOGSYS_DECLARE_SUBSYS("VOTEQ")
int(* ipc_dispatch_send)(void *conn, const void *msg, size_t mlen)
cs_error_t icmap_get_uint32(const char *key_name, uint32_t *u32)
int(* totem_mcast)(const struct iovec *iovec, unsigned int iov_len, unsigned int guarantee)
#define VOTEQUORUM_INFO_AUTO_TIE_BREAKER
struct corosync_service_engine * votequorum_get_service_engine_ver0(void)
int(* ipc_response_send)(void *conn, const void *msg, size_t mlen)
char * corosync_service_link_and_init(struct corosync_api_v1 *corosync_api, struct default_service *service)
Link and initialize a service.
char newname[VOTEQUORUM_QDEVICE_MAX_NAME_LEN]
#define VOTEQUORUM_NODESTATE_LEAVING
#define PROCESSOR_COUNT_MAX
unsigned int qdevice_votes
void(* error_memory_failure)(void) __attribute__((noreturn))
#define MESSAGE_REQ_EXEC_VOTEQUORUM_QDEVICE_REG
#define VOTEQUORUM_READCONFIG_RUNTIME
#define MESSAGE_REQ_EXEC_VOTEQUORUM_NODEINFO
#define VOTEQUORUM_RECONFIG_PARAM_NODE_VOTES
#define VOTEQUORUM_QDEVICE_MAX_NAME_LEN
qb_loop_timer_handle corosync_timer_handle_t
cs_error_t icmap_get_string(const char *key_name, char **str)
#define LOGSYS_LEVEL_CRIT
char oldname[VOTEQUORUM_QDEVICE_MAX_NAME_LEN]
#define NODE_FLAGS_LEAVING
#define list_entry(ptr, type, member)
#define COROSYNC_LIB_FLOW_CONTROL_REQUIRED
#define LOGSYS_LEVEL_NOTICE
cs_error_t icmap_set_uint8(const char *key_name, uint8_t value)
#define VOTEQUORUM_NODESTATE_DEAD
#define VOTEQUORUM_INFO_QDEVICE_CAST_VOTE
#define VOTEQUORUM_QDEVICE_DEFAULT_TIMEOUT
icmap_iter_t icmap_iter_init(const char *prefix)
struct memb_ring_id ring_id
uint64_t tracking_context
void(* quorum_set_quorate_fn_t)(const unsigned int *view_list, size_t view_list_entries, int quorate, struct memb_ring_id *)
unsigned int(* totem_nodeid_get)(void)
#define VOTEQUORUM_RECONFIG_PARAM_EXPECTED_VOTES
#define NODE_FLAGS_QDEVICE_ALIVE
qb_map_iter_t * icmap_iter_t
#define NODE_FLAGS_QDEVICE_REGISTERED
cs_error_t icmap_track_add(const char *key_name, int32_t track_type, icmap_notify_fn_t notify_fn, void *user_data, icmap_track_t *icmap_track)
struct qb_ipc_request_header header __attribute__((aligned(8)))
#define ICMAP_TRACK_PREFIX
unsigned int expected_votes