based refactors: Simplify IPC channels and cleanup

daemons/based/based_callbacks.c

@@ -28,7 +28,7 @@
 #include <crm/common/ipc.h>         // crm_ipc_*, pcmk_ipc_*
 #include <crm/common/logging.h>     // CRM_LOG_ASSERT, CRM_CHECK
 #include <crm/common/mainloop.h>    // mainloop_*
-#include <crm/common/results.h>     // CRM_EX_OK, crm_exit_t, pcmk_rc_*
+#include <crm/common/results.h>     // pcmk_rc_*
 #include <crm/common/xml.h>         // PCMK_XA_*, PCMK_XE_*
 #include <crm/crm.h>                // CRM_OP_*
 
@@ -39,6 +39,85 @@ static long long ping_seq = 0;
 static char *ping_digest = NULL;
 static bool ping_modified_since = false;
 
+/*!
+ * \internal
+ * \brief Request CIB digests from all peer nodes
+ *
+ * This is used as a callback that runs 5 seconds after we modify the CIB on the
+ * DC. It sends a ping request to all cluster nodes. They will respond by
+ * sending their current digests and version info, which we will validate in
+ * process_ping_reply(). If their digest doesn't match, we'll sync our own CIB
+ * to them. This helps ensure consistency across the cluster after a CIB update.
+ *
+ * \param[in] data  Ignored
+ *
+ * \return \c G_SOURCE_REMOVE (to destroy the timeout)
+ *
+ * \note It's not clear why we wait 5 seconds rather than sending the ping
+ *       request immediately after a performing a modifying op. Perhaps it's to
+ *       avoid overwhelming other nodes with ping requests when there are a lot
+ *       of modifying requests in a short period. The timer restarts after
+ *       every successful modifying op, so we send ping requests **at most**
+ *       every 5 seconds. Or perhaps it's a remnant of legacy mode (pre-1.1.12).
+ *       In any case, the other nodes shouldn't need time to process the
+ *       modifying op before responding to the ping request. The ping request is
+ *       sent after the op is sent, so it should also be received after the op
+ *       is received.
+ */
+static gboolean
+digest_timer_cb(void *data)
+{
+    xmlNode *ping = NULL;
+
+    if (!based_get_local_node_dc()) {
+        // Only the DC sends a ping
+        return G_SOURCE_REMOVE;
+    }
+
+    if (++ping_seq < 0) {
+        ping_seq = 0;
+    }
+
+    g_clear_pointer(&ping_digest, free);
+    ping_modified_since = false;
+
+    ping = pcmk__xe_create(NULL, PCMK__XE_PING);
+    pcmk__xe_set(ping, PCMK__XA_T, PCMK__VALUE_CIB);
+    pcmk__xe_set(ping, PCMK__XA_CIB_OP, CRM_OP_PING);
+    pcmk__xe_set_ll(ping, PCMK__XA_CIB_PING_ID, ping_seq);
+    pcmk__xe_set(ping, PCMK_XA_CRM_FEATURE_SET, CRM_FEATURE_SET);
+
+    pcmk__trace("Requesting peer digests (%lld)", ping_seq);
+    pcmk__cluster_send_message(NULL, pcmk_ipc_based, ping);
+
+    pcmk__xml_free(ping);
+    return G_SOURCE_REMOVE;
+}
+
+/*!
+ * \internal
+ * \brief Initialize data structures used for CIB manager callbacks
+ */
+void
+based_callbacks_init(void)
+{
+    if (digest_timer == NULL) {
+        digest_timer = mainloop_timer_add("based_digest_timer", 5000, false,
+                                          digest_timer_cb, NULL);
+    }
+}
+
+/*!
+ * \internal
+ * \brief Free data structures used for CIB manager callbacks
+ */
+void
+based_callbacks_cleanup(void)
+{
+    g_clear_pointer(&digest_timer, mainloop_timer_del);
+    g_clear_pointer(&ping_digest, free);
+}
+
 /*!
  * \internal
  * \brief Create reply XML for a CIB request
@@ -148,61 +227,6 @@ do_local_notify(const xmlNode *xml, const char *client_id, bool sync_reply,
     }
 }
 
-/*!
- * \internal
- * \brief Request CIB digests from all peer nodes
- *
- * This is used as a callback that runs 5 seconds after we modify the CIB on the
- * DC. It sends a ping request to all cluster nodes. They will respond by
- * sending their current digests and version info, which we will validate in
- * process_ping_reply(). If their digest doesn't match, we'll sync our own CIB
- * to them. This helps ensure consistency across the cluster after a CIB update.
- *
- * \param[in] data  Ignored
- *
- * \return \c G_SOURCE_REMOVE (to destroy the timeout)
- *
- * \note It's not clear why we wait 5 seconds rather than sending the ping
- *       request immediately after a performing a modifying op. Perhaps it's to
- *       avoid overwhelming other nodes with ping requests when there are a lot
- *       of modifying requests in a short period. The timer restarts after
- *       every successful modifying op, so we send ping requests **at most**
- *       every 5 seconds. Or perhaps it's a remnant of legacy mode (pre-1.1.12).
- *       In any case, the other nodes shouldn't need time to process the
- *       modifying op before responding to the ping request. The ping request is
- *       sent after the op is sent, so it should also be received after the op
- *       is received.
- */
-static gboolean
-digest_timer_cb(void *data)
-{
-    xmlNode *ping = NULL;
-
-    if (!based_get_local_node_dc()) {
-        // Only the DC sends a ping
-        return G_SOURCE_REMOVE;
-    }
-
-    if (++ping_seq < 0) {
-        ping_seq = 0;
-    }
-
-    g_clear_pointer(&ping_digest, free);
-    ping_modified_since = false;
-
-    ping = pcmk__xe_create(NULL, PCMK__XE_PING);
-    pcmk__xe_set(ping, PCMK__XA_T, PCMK__VALUE_CIB);
-    pcmk__xe_set(ping, PCMK__XA_CIB_OP, CRM_OP_PING);
-    pcmk__xe_set_ll(ping, PCMK__XA_CIB_PING_ID, ping_seq);
-    pcmk__xe_set(ping, PCMK_XA_CRM_FEATURE_SET, CRM_FEATURE_SET);
-
-    pcmk__trace("Requesting peer digests (%lld)", ping_seq);
-    pcmk__cluster_send_message(NULL, pcmk_ipc_based, ping);
-
-    pcmk__xml_free(ping);
-    return G_SOURCE_REMOVE;
-}
-
 /*!
  * \internal
  * \brief Process a reply to a \c CRM_OP_PING request
@@ -542,11 +566,6 @@ based_perform_op_rw(xmlNode *request, const cib__operation_t *operation,
         ping_modified_since = true;
     }
 
-    if (digest_timer == NULL) {
-        digest_timer = mainloop_timer_add("based_digest_timer", 5000, false,
-                                          digest_timer_cb, NULL);
-    }
-
     mainloop_timer_start(digest_timer);
 
 done:
@@ -812,35 +831,3 @@ based_process_request(xmlNode *request, bool privileged,
     pcmk__xml_free(reply);
     return rc;
 }
-
-/*!
- * \internal
- * \brief Close remote sockets, free the global CIB and quit
- *
- * \param[in] exit_status  Exit code
- */
-void
-based_terminate(crm_exit_t exit_status)
-{
-    based_ipc_cleanup();
-    based_remote_cleanup();
-
-    g_clear_pointer(&digest_timer, mainloop_timer_del);
-    g_clear_pointer(&ping_digest, free);
-    g_clear_pointer(&based_cib, pcmk__xml_free);
-
-    // Exit immediately on error
-    if (exit_status != CRM_EX_OK) {
-        crm_exit(exit_status);
-        return;
-    }
-
-    based_cluster_disconnect();
-
-    if ((mainloop != NULL) && g_main_loop_is_running(mainloop)) {
-        g_main_loop_quit(mainloop);
-        return;
-    }
-
-    crm_exit(CRM_EX_OK);
-}

daemons/based/based_callbacks.h

@@ -15,10 +15,11 @@
 #include <libxml/tree.h>                // xmlNode
 
 #include <crm/common/internal.h>        // pcmk__client_t
-#include <crm/common/results.h>         // crm_exit_t
+
+void based_callbacks_init(void);
+void based_callbacks_cleanup(void);
 
 int based_process_request(xmlNode *request, bool privileged,
                           const pcmk__client_t *client);
-void based_terminate(crm_exit_t exit_status);
 
 #endif // BASED_CALLBACKS__H

daemons/based/based_corosync.c

@@ -15,7 +15,7 @@
 #include <stdlib.h>                 // free
 
 #include <corosync/cpg.h>           // cpg_*
-#include <glib.h>                   // gpointer
+#include <glib.h>                   // g_clear_pointer
 #include <libxml/tree.h>            // xmlNode
 
 #include <crm_config.h>             // SUPPORT_COROSYNC

daemons/based/based_io.c

@@ -31,7 +31,7 @@
 #include <crm/cib/util.h>           // createEmptyCib
 #include <crm/common/internal.h>    // pcmk__assert_asprintf, PCMK__XE_*, etc.
 #include <crm/common/logging.h>     // CRM_CHECK
-#include <crm/common/mainloop.h>    // mainloop_add_signal
+#include <crm/common/mainloop.h>    // mainloop_*
 #include <crm/common/results.h>     // pcmk_legacy2rc, pcmk_rc_*
 #include <crm/common/util.h>        // pcmk_common_cleanup
 #include <crm/common/xml.h>         // PCMK_XA_*, PCMK_XE_*
@@ -176,7 +176,7 @@ based_enable_writes(int nsig)
 
 /*!
  * \internal
- * \brief Initialize data structures for \c pacemaker-based I/O
+ * \brief Initialize data structures used for CIB manager I/O
  */
 void
 based_io_init(void)
@@ -198,6 +198,16 @@ based_io_init(void)
     write_trigger = mainloop_add_trigger(G_PRIORITY_LOW, write_cib_async, NULL);
 }
 
+/*!
+ * \internal
+ * \brief Free data structures used for CIB manager I/O
+ */
+void
+based_io_cleanup(void)
+{
+    g_clear_pointer(&write_trigger, mainloop_destroy_trigger);
+}
+
 /*!
  * \internal
  * \brief Rename a CIB or digest file after digest mismatch

daemons/based/based_io.h

@@ -15,6 +15,8 @@
 #include <libxml/tree.h>                // xmlNode
 
 void based_io_init(void);
+void based_io_cleanup(void);
+
 void based_enable_writes(int nsig);
 xmlNode *based_read_cib(void);
 int based_activate_cib(xmlNode *new_cib, bool to_disk, const char *op);

daemons/based/based_ipc.c

@@ -30,7 +30,6 @@
 
 static qb_ipcs_service_t *ipcs_ro = NULL;
 static qb_ipcs_service_t *ipcs_rw = NULL;
-static qb_ipcs_service_t *ipcs_shm = NULL;
 
 /*!
  * \internal
@@ -305,7 +304,7 @@ static struct qb_ipcs_service_handlers ipc_rw_callbacks = {
 void
 based_ipc_init(void)
 {
-    pcmk__serve_based_ipc(&ipcs_ro, &ipcs_rw, &ipcs_shm, &ipc_ro_callbacks,
+    pcmk__serve_based_ipc(&ipcs_ro, &ipcs_rw, &ipc_ro_callbacks,
                           &ipc_rw_callbacks);
 }
 
@@ -322,9 +321,6 @@ based_ipc_cleanup(void)
     pcmk__drop_all_clients(ipcs_rw);
     g_clear_pointer(&ipcs_rw, qb_ipcs_destroy);
 
-    pcmk__drop_all_clients(ipcs_shm);
-    g_clear_pointer(&ipcs_shm, qb_ipcs_destroy);
-
     /* Drop remote clients here because they're part of the IPC client table and
      * must be dropped before \c pcmk__client_cleanup()
      */