Run an RPC protocol over the LXC controller monitor

This defines a new RPC protocol to be used between the LXC
controller and the libvirtd LXC driver. There is only a
single RPC message defined thus far, an asynchronous "EXIT"
event that is emitted just before the LXC controller process
exits. This provides the LXC driver with details about how
the container shutdown - normally, or abnormally (crashed),
thus allowing the driver to emit better libvirt events.

Emitting the event in the LXC controller requires a few
little tricks with the RPC service. Simply calling the
virNetServiceClientSendMessage does not work, since this
merely queues the message for asynchronous processing.
In addition the main event loop is no longer running at
the point the event is emitted, so no I/O is processed.

Thus after invoking virNetServiceClientSendMessage it is
necessary to mark the client as being in "delayed close"
mode. Then the event loop is run again, until the client
completes its close - this happens only after the queued
message has been fully transmitted. The final complexity
is that it is not safe to run virNetServerQuit() from the
client close callback, since that is invoked from a
context where the server is locked. Thus a zero-second
timer is used to trigger shutdown of the event loop,
causing the controller to finally exit.

* src/Makefile.am: Add rules for generating RPC protocol
  files and dispatch methods
* src/lxc/lxc_controller.c: Emit an RPC event immediately
  before exiting
* src/lxc/lxc_domain.h: Record the shutdown reason
  given by the controller
* src/lxc/lxc_monitor.c, src/lxc/lxc_monitor.h: Register
  RPC program and event handler. Add callback to let
  driver receive EXIT event.
* src/lxc/lxc_process.c: Use monitor exit event to decide
  what kind of domain event to emit
* src/lxc/lxc_protocol.x: Define wire protocol for LXC
  controller monitor.

Signed-off-by: Daniel P. Berrange <berrange@redhat.com>

6 files changed, 221 insertions, 2 deletions

diff --git a/src/lxc/lxc_controller.c b/src/lxc/lxc_controller.c
index e39c2365c..e388302a8 100644
--- a/src/lxc/lxc_controller.c
+++ b/src/lxc/lxc_controller.c
@@ -59,6 +59,7 @@
 #include "lxc_conf.h"
 #include "lxc_container.h"
 #include "lxc_cgroup.h"
+#include "lxc_protocol.h"
 #include "virnetdev.h"
 #include "virnetdevveth.h"
 #include "memory.h"
@@ -121,10 +122,25 @@ struct _virLXCController {
 
     /* Server socket */
     virNetServerPtr server;
+    virNetServerClientPtr client;
+    virNetServerProgramPtr prog;
+    bool inShutdown;
+    int timerShutdown;
 };
 
+#include "lxc_controller_dispatch.h"
+
 static void virLXCControllerFree(virLXCControllerPtr ctrl);
 
+static void virLXCControllerQuitTimer(int timer ATTRIBUTE_UNUSED, void *opaque)
+{
+    virLXCControllerPtr ctrl = opaque;
+
+    VIR_DEBUG("Triggering event loop quit");
+    virNetServerQuit(ctrl->server);
+}
+
+
 static virLXCControllerPtr virLXCControllerNew(const char *name)
 {
     virLXCControllerPtr ctrl = NULL;
@@ -134,6 +150,8 @@ static virLXCControllerPtr virLXCControllerNew(const char *name)
     if (VIR_ALLOC(ctrl) < 0)
         goto no_memory;
 
+    ctrl->timerShutdown = -1;
+
     if (!(ctrl->name = strdup(name)))
         goto no_memory;
 
@@ -150,6 +168,11 @@ static virLXCControllerPtr virLXCControllerNew(const char *name)
                                            0)) == NULL)
         goto error;
 
+    if ((ctrl->timerShutdown = virEventAddTimeout(-1,
+                                                  virLXCControllerQuitTimer, ctrl,
+                                                  NULL)) < 0)
+        goto error;
+
 cleanup:
     VIR_FREE(configFile);
     virCapabilitiesFree(caps);
@@ -238,6 +261,9 @@ static void virLXCControllerFree(virLXCControllerPtr ctrl)
     virDomainDefFree(ctrl->def);
     VIR_FREE(ctrl->name);
 
+    if (ctrl->timerShutdown != -1)
+        virEventRemoveTimeout(ctrl->timerShutdown);
+
     virNetServerFree(ctrl->server);
 
     VIR_FREE(ctrl);
@@ -539,12 +565,28 @@ static int virLXCControllerSetupResourceLimits(virLXCControllerPtr ctrl)
 }
 
 
+static void virLXCControllerClientCloseHook(virNetServerClientPtr client)
+{
+    virLXCControllerPtr ctrl = virNetServerClientGetPrivateData(client);
+
+    VIR_DEBUG("Client %p has closed", client);
+    if (ctrl->client == client)
+        ctrl->client = NULL;
+    if (ctrl->inShutdown) {
+        VIR_DEBUG("Arm timer to quit event loop");
+        virEventUpdateTimeout(ctrl->timerShutdown, 0);
+    }
+}
+
 static int virLXCControllerClientHook(virNetServerPtr server ATTRIBUTE_UNUSED,
                                       virNetServerClientPtr client,
                                       void *opaque)
 {
     virLXCControllerPtr ctrl = opaque;
     virNetServerClientSetPrivateData(client, ctrl, NULL);
+    virNetServerClientSetCloseHook(client, virLXCControllerClientCloseHook);
+    VIR_DEBUG("Got new client %p", client);
+    ctrl->client = client;
     return 0;
 }
 
@@ -581,6 +623,12 @@ static int virLXCControllerSetupServer(virLXCControllerPtr ctrl)
     virNetServerServiceFree(svc);
     svc = NULL;
 
+    if (!(ctrl->prog = virNetServerProgramNew(VIR_LXC_PROTOCOL_PROGRAM,
+                                              VIR_LXC_PROTOCOL_PROGRAM_VERSION,
+                                              virLXCProtocolProcs,
+                                              virLXCProtocolNProcs)))
+        goto error;
+
     virNetServerUpdateServices(ctrl->server, true);
     VIR_FREE(sockpath);
     return 0;
@@ -1219,6 +1267,79 @@ virLXCControllerSetupConsoles(virLXCControllerPtr ctrl,
 }
 
 
+static void
+virLXCControllerEventSend(virLXCControllerPtr ctrl,
+                          int procnr,
+                          xdrproc_t proc,
+                          void *data)
+{
+    virNetMessagePtr msg;
+
+    if (!ctrl->client)
+        return;
+
+    VIR_DEBUG("Send event %d client=%p", procnr, ctrl->client);
+    if (!(msg = virNetMessageNew(false)))
+        goto error;
+
+    msg->header.prog = virNetServerProgramGetID(ctrl->prog);
+    msg->header.vers = virNetServerProgramGetVersion(ctrl->prog);
+    msg->header.proc = procnr;
+    msg->header.type = VIR_NET_MESSAGE;
+    msg->header.serial = 1;
+    msg->header.status = VIR_NET_OK;
+
+    if (virNetMessageEncodeHeader(msg) < 0)
+        goto error;
+
+    if (virNetMessageEncodePayload(msg, proc, data) < 0)
+        goto error;
+
+    VIR_DEBUG("Queue event %d %zu", procnr, msg->bufferLength);
+    virNetServerClientSendMessage(ctrl->client, msg);
+
+    xdr_free(proc, data);
+    return;
+
+error:
+    virNetMessageFree(msg);
+    xdr_free(proc, data);
+}
+
+
+static int
+virLXCControllerEventSendExit(virLXCControllerPtr ctrl,
+                              int exitstatus)
+{
+    virLXCProtocolExitEventMsg msg;
+
+    VIR_DEBUG("Exit status %d", exitstatus);
+    memset(&msg, 0, sizeof(msg));
+    switch (exitstatus) {
+    case 0:
+        msg.status = VIR_LXC_PROTOCOL_EXIT_STATUS_SHUTDOWN;
+        break;
+    default:
+        msg.status = VIR_LXC_PROTOCOL_EXIT_STATUS_ERROR;
+        break;
+    }
+
+    virLXCControllerEventSend(ctrl,
+                              VIR_LXC_PROTOCOL_PROC_EXIT_EVENT,
+                              (xdrproc_t)xdr_virLXCProtocolExitEventMsg,
+                              (void*)&msg);
+
+    if (ctrl->client) {
+        VIR_DEBUG("Waiting for client to complete dispatch");
+        ctrl->inShutdown = true;
+        virNetServerClientDelayedClose(ctrl->client);
+        virNetServerRun(ctrl->server);
+    }
+    VIR_DEBUG("Client has gone away");
+    return 0;
+}
+
+
 static int
 virLXCControllerRun(virLXCControllerPtr ctrl)
 {
@@ -1306,6 +1427,8 @@ virLXCControllerRun(virLXCControllerPtr ctrl)
 
     rc = virLXCControllerMain(ctrl);
 
+    virLXCControllerEventSendExit(ctrl, rc);
+
 cleanup:
     VIR_FORCE_CLOSE(control[0]);
     VIR_FORCE_CLOSE(control[1]);
@@ -1527,5 +1650,5 @@ cleanup:
 
     virLXCControllerFree(ctrl);
 
-    return rc ? EXIT_FAILURE : EXIT_SUCCESS;
+    return rc < 0? EXIT_FAILURE : EXIT_SUCCESS;
 }
diff --git a/src/lxc/lxc_domain.h b/src/lxc/lxc_domain.h
index 318c78eb6..f4a838412 100644
--- a/src/lxc/lxc_domain.h
+++ b/src/lxc/lxc_domain.h
@@ -32,6 +32,7 @@ typedef virLXCDomainObjPrivate *virLXCDomainObjPrivatePtr;
 struct _virLXCDomainObjPrivate {
     virLXCMonitorPtr monitor;
     bool doneStopEvent;
+    int stopReason;
 };
 
 void virLXCDomainSetPrivateDataHooks(virCapsPtr caps);
diff --git a/src/lxc/lxc_monitor.c b/src/lxc/lxc_monitor.c
index da170ae6a..a2a25995f 100644
--- a/src/lxc/lxc_monitor.c
+++ b/src/lxc/lxc_monitor.c
@@ -22,6 +22,8 @@
 
 #include "lxc_monitor.h"
 #include "lxc_conf.h"
+#include "lxc_protocol.h"
+#include "lxc_monitor_dispatch.h"
 
 #include "memory.h"
 
@@ -41,9 +43,35 @@ struct _virLXCMonitor {
     virLXCMonitorCallbacksPtr cb;
 
     virNetClientPtr client;
+    virNetClientProgramPtr program;
 };
 
 static void virLXCMonitorFree(virLXCMonitorPtr mon);
+static void
+virLXCMonitorHandleEventExit(virNetClientProgramPtr prog,
+                             virNetClientPtr client,
+                             void *evdata, void *opaque);
+
+static virNetClientProgramEvent virLXCProtocolEvents[] = {
+    { VIR_LXC_PROTOCOL_PROC_EXIT_EVENT,
+      virLXCMonitorHandleEventExit,
+      sizeof(virLXCProtocolExitEventMsg),
+      (xdrproc_t)xdr_virLXCProtocolExitEventMsg },
+};
+
+
+static void
+virLXCMonitorHandleEventExit(virNetClientProgramPtr prog ATTRIBUTE_UNUSED,
+                             virNetClientPtr client ATTRIBUTE_UNUSED,
+                             void *evdata, void *opaque)
+{
+    virLXCMonitorPtr mon = opaque;
+    virLXCProtocolExitEventMsg *msg = evdata;
+
+    VIR_DEBUG("Event exit %d", msg->status);
+    if (mon->cb->exitNotify)
+        mon->cb->exitNotify(mon, msg->status, mon->vm);
+}
 
 
 static void virLXCMonitorEOFNotify(virNetClientPtr client ATTRIBUTE_UNUSED,
@@ -54,6 +82,7 @@ static void virLXCMonitorEOFNotify(virNetClientPtr client ATTRIBUTE_UNUSED,
     virLXCMonitorCallbackEOFNotify eofNotify;
     virDomainObjPtr vm;
 
+    VIR_DEBUG("EOF notify");
     virLXCMonitorLock(mon);
     eofNotify = mon->cb->eofNotify;
     vm = mon->vm;
@@ -101,6 +130,17 @@ virLXCMonitorPtr virLXCMonitorNew(virDomainObjPtr vm,
         goto error;
 
 
+    if (!(mon->program = virNetClientProgramNew(VIR_LXC_PROTOCOL_PROGRAM,
+                                                VIR_LXC_PROTOCOL_PROGRAM_VERSION,
+                                                virLXCProtocolEvents,
+                                                ARRAY_CARDINALITY(virLXCProtocolEvents),
+                                                mon)))
+        goto error;
+
+    if (virNetClientAddProgram(mon->client,
+                               mon->program) < 0)
+        goto error;
+
     mon->vm = vm;
     mon->cb = cb;
 
@@ -130,6 +170,7 @@ static void virLXCMonitorFree(virLXCMonitorPtr mon)
     if (mon->cb && mon->cb->destroy)
         (mon->cb->destroy)(mon, mon->vm);
     virMutexDestroy(&mon->lock);
+    virNetClientProgramFree(mon->program);
     VIR_FREE(mon);
 }
 
diff --git a/src/lxc/lxc_monitor.h b/src/lxc/lxc_monitor.h
index d3b6387b1..53301f1f2 100644
--- a/src/lxc/lxc_monitor.h
+++ b/src/lxc/lxc_monitor.h
@@ -22,6 +22,7 @@
 # define __LXC_MONITOR_H__
 
 # include "domain_conf.h"
+# include "lxc_protocol.h"
 
 typedef struct _virLXCMonitor virLXCMonitor;
 typedef virLXCMonitor *virLXCMonitorPtr;
@@ -34,9 +35,14 @@ typedef void (*virLXCMonitorCallbackDestroy)(virLXCMonitorPtr mon,
 typedef void (*virLXCMonitorCallbackEOFNotify)(virLXCMonitorPtr mon,
                                                virDomainObjPtr vm);
 
+typedef void (*virLXCMonitorCallbackExitNotify)(virLXCMonitorPtr mon,
+                                                virLXCProtocolExitStatus status,
+                                                virDomainObjPtr vm);
+
 struct _virLXCMonitorCallbacks {
     virLXCMonitorCallbackDestroy destroy;
     virLXCMonitorCallbackEOFNotify eofNotify;
+    virLXCMonitorCallbackExitNotify exitNotify;
 };
 
 virLXCMonitorPtr virLXCMonitorNew(virDomainObjPtr vm,
diff --git a/src/lxc/lxc_process.c b/src/lxc/lxc_process.c
index 89c2b214b..5ffdff5c6 100644
--- a/src/lxc/lxc_process.c
+++ b/src/lxc/lxc_process.c
@@ -167,6 +167,9 @@ static void virLXCProcessCleanup(virLXCDriverPtr driver,
     virLXCDomainObjPrivatePtr priv = vm->privateData;
     virNetDevVPortProfilePtr vport = NULL;
 
+    VIR_DEBUG("Stopping VM name=%s pid=%d reason=%d",
+              vm->def->name, (int)vm->pid, (int)reason);
+
     /* now that we know it's stopped call the hook if present */
     if (virHookPresent(VIR_HOOK_DRIVER_LXC)) {
         char *xml = virDomainDefFormat(vm->def, 0);
@@ -509,7 +512,7 @@ static void virLXCProcessMonitorEOFNotify(virLXCMonitorPtr mon ATTRIBUTE_UNUSED,
     if (!priv->doneStopEvent) {
         event = virDomainEventNewFromObj(vm,
                                          VIR_DOMAIN_EVENT_STOPPED,
-                                         VIR_DOMAIN_EVENT_STOPPED_SHUTDOWN);
+                                         priv->stopReason);
         virDomainAuditStop(vm, "shutdown");
     } else {
         VIR_DEBUG("Stop event has already been sent");
@@ -528,10 +531,31 @@ static void virLXCProcessMonitorEOFNotify(virLXCMonitorPtr mon ATTRIBUTE_UNUSED,
     }
 }
 
+static void virLXCProcessMonitorExitNotify(virLXCMonitorPtr mon ATTRIBUTE_UNUSED,
+                                           virLXCProtocolExitStatus status,
+                                           virDomainObjPtr vm)
+{
+    virLXCDomainObjPrivatePtr priv = vm->privateData;
+
+    switch (status) {
+    case VIR_LXC_PROTOCOL_EXIT_STATUS_SHUTDOWN:
+        priv->stopReason = VIR_DOMAIN_EVENT_STOPPED_SHUTDOWN;
+        break;
+    case VIR_LXC_PROTOCOL_EXIT_STATUS_ERROR:
+        priv->stopReason = VIR_DOMAIN_EVENT_STOPPED_FAILED;
+        break;
+    default:
+        priv->stopReason = VIR_DOMAIN_EVENT_STOPPED_FAILED;
+        break;
+    }
+    VIR_DEBUG("Domain shutoff reason %d (from status %d)",
+              priv->stopReason, status);
+}
 
 static virLXCMonitorCallbacks monitorCallbacks = {
     .eofNotify = virLXCProcessMonitorEOFNotify,
     .destroy = virLXCProcessMonitorDestroy,
+    .exitNotify = virLXCProcessMonitorExitNotify,
 };
 
 
@@ -573,6 +597,8 @@ int virLXCProcessStop(virLXCDriverPtr driver,
     virCgroupPtr group = NULL;
     int rc;
 
+    VIR_DEBUG("Stopping VM name=%s pid=%d reason=%d",
+              vm->def->name, (int)vm->pid, (int)reason);
     if (vm->pid <= 0) {
         virReportError(VIR_ERR_INTERNAL_ERROR,
                        _("Invalid PID %d for container"), vm->pid);
@@ -1006,6 +1032,7 @@ int virLXCProcessStart(virConnectPtr conn,
         goto cleanup;
     }
 
+    priv->stopReason = VIR_DOMAIN_EVENT_STOPPED_FAILED;
     vm->def->id = vm->pid;
     virDomainObjSetState(vm, VIR_DOMAIN_RUNNING, reason);
     priv->doneStopEvent = false;
diff --git a/src/lxc/lxc_protocol.x b/src/lxc/lxc_protocol.x
new file mode 100644
index 000000000..4fdbe34e7
--- /dev/null
+++ b/src/lxc/lxc_protocol.x
@@ -0,0 +1,21 @@
+/* -*- c -*-
+ * Define wire protocol for communication between the
+ * LXC driver in libvirtd, and the LXC controller in
+ * the libvirt_lxc helper program.
+ */
+
+enum virLXCProtocolExitStatus {
+    VIR_LXC_PROTOCOL_EXIT_STATUS_ERROR,
+    VIR_LXC_PROTOCOL_EXIT_STATUS_SHUTDOWN
+};
+
+struct virLXCProtocolExitEventMsg {
+    enum virLXCProtocolExitStatus status;
+};
+
+const VIR_LXC_PROTOCOL_PROGRAM = 0x12341234;
+const VIR_LXC_PROTOCOL_PROGRAM_VERSION = 1;
+
+enum virLXCProtocolProcedure {
+    VIR_LXC_PROTOCOL_PROC_EXIT_EVENT = 1 /* skipgen skipgen */
+};