engine: alert when a host reports instances CloudStack does not know about

A warn line only helps someone already reading the log. Raise an
ALERT_TYPE_SYNC alert as well, so the condition reaches whoever watches
alerts.

The alert follows the same rule as the log: raised when the set of unknown
instances on a host changes, not on every report.

Nothing is stopped automatically. An unknown instance may be a domain an
operator put on the host deliberately.

Signed-off-by: Brad House <bhouse@nexthop.ai>

Author: bhouse-nexthop

engine/orchestration/src/main/java/com/cloud/vm/VirtualMachinePowerStateSyncImpl.java

@@ -36,6 +36,7 @@
 import org.apache.logging.log4j.Logger;
 
 import com.cloud.agent.api.HostVmStateReportEntry;
+import com.cloud.alert.AlertManager;
 import com.cloud.configuration.ManagementServiceConfiguration;
 import com.cloud.host.Host;
 import com.cloud.host.HostVO;
@@ -50,6 +51,9 @@ public class VirtualMachinePowerStateSyncImpl implements VirtualMachinePowerStat
     @Inject VMInstanceDao _instanceDao;
     @Inject HostDao hostDao;
     @Inject ManagementServiceConfiguration mgmtServiceConf;
+    @Inject AlertManager _alertMgr;
+
+    protected static final String UNKNOWN_INSTANCES_ALERT_SUBJECT = "Instances running on a host that CloudStack has no record of";
 
     private final Map<Long, Set<String>> unknownInstancesByHost = new ConcurrentHashMap<>();
 
@@ -245,9 +249,18 @@ protected boolean reportUnknownInstances(long hostId, Set<String> unknownInstanc
             return true;
         }
         unknownInstancesByHost.put(hostId, unknownInstanceNames);
+        HostVO host = hostCache.get(hostId);
+        String names = String.join(", ", unknownInstanceNames);
         logger.warn("Host reports {} instance(s) that do not exist in CloudStack DB, they are running unmanaged. " +
                         "host: {}, instances: [{}]",
-                unknownInstanceNames.size(), hostCache.get(hostId), String.join(", ", unknownInstanceNames));
+                unknownInstanceNames.size(), host, names);
+        if (host != null) {
+            _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_SYNC, host.getDataCenterId(), host.getPodId(),
+                    UNKNOWN_INSTANCES_ALERT_SUBJECT,
+                    String.format("Host %s reports %d instance(s) that do not exist in CloudStack: %s. They are "
+                                    + "running unmanaged and may still be holding addresses and storage.",
+                            host.getName(), unknownInstanceNames.size(), names));
+        }
         return true;
     }