engine: cover the stop paths that decide whether an address is freed

These are the branches that decide whether a still-running instance keeps its
addresses, and they had no tests.

`VirtualMachineManagerImplTest`:

| test                                          | checks                                        |
|-----------------------------------------------|-----------------------------------------------|
| missingReportUsesUnforcedStopAndKeepsResources | a missing report stops unforced, and a failed stop does not release resources |
| powerOffKeepsForcedStop                        | a PowerOff report still stops forced          |
| ensure...skipsExternal                         | External instances are left to their extension |
| ensure...noHostIdDoesNothing                   | nothing is sent when there is no host to send to |
| ensure...fallsBackToLastHostId                 | the last known host is used when `host_id` is cleared |
| ensure...agentUnavailableIsSwallowed           | an unreachable host does not break the expunge |

`VirtualMachinePowerStateSyncImplTest` gains the AlertManager mock the alert
needs.

Signed-off-by: Brad House <bhouse@nexthop.ai>

Author: bhouse-nexthop

engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java

@@ -24,6 +24,7 @@
 import static org.junit.Assert.assertThrows;
 import static org.junit.Assert.assertTrue;
 import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
 import static org.mockito.ArgumentMatchers.anyList;
 import static org.mockito.ArgumentMatchers.anyLong;
 import static org.mockito.ArgumentMatchers.anyString;
@@ -2049,4 +2050,85 @@ public void testUnmanageSuccessKvm() throws Exception {
         }
     }
 
+    @Test
+    public void testEnsureInstanceIsStoppedOnLastKnownHost_skipsExternal() throws Exception {
+        when(vmInstanceMock.getHypervisorType()).thenReturn(HypervisorType.External);
+        virtualMachineManagerImpl.ensureInstanceIsStoppedOnLastKnownHost(vmInstanceMock);
+        verify(agentManagerMock, never()).send(anyLong(), any(StopCommand.class));
+    }
+
+    @Test
+    public void testEnsureInstanceIsStoppedOnLastKnownHost_noHostIdDoesNothing() throws Exception {
+        when(vmInstanceMock.getHypervisorType()).thenReturn(HypervisorType.KVM);
+        when(vmInstanceMock.getHostId()).thenReturn(null);
+        when(vmInstanceMock.getLastHostId()).thenReturn(null);
+        virtualMachineManagerImpl.ensureInstanceIsStoppedOnLastKnownHost(vmInstanceMock);
+        verify(agentManagerMock, never()).send(anyLong(), any(StopCommand.class));
+    }
+
+    @Test
+    public void testEnsureInstanceIsStoppedOnLastKnownHost_fallsBackToLastHostId() throws Exception {
+        StopCommand stopCommand = mock(StopCommand.class);
+        when(vmInstanceMock.getHypervisorType()).thenReturn(HypervisorType.KVM);
+        when(vmInstanceMock.getHostId()).thenReturn(null);
+        when(vmInstanceMock.getLastHostId()).thenReturn(7L);
+        doReturn(stopCommand).when(virtualMachineManagerImpl).buildStopCommand(any(), any(), eq(false));
+        com.cloud.agent.api.Answer answer = mock(com.cloud.agent.api.Answer.class);
+        when(answer.getResult()).thenReturn(true);
+        when(agentManagerMock.send(eq(7L), any(StopCommand.class))).thenReturn(answer);
+
+        virtualMachineManagerImpl.ensureInstanceIsStoppedOnLastKnownHost(vmInstanceMock);
+
+        verify(agentManagerMock, times(1)).send(eq(7L), any(StopCommand.class));
+    }
+
+    @Test
+    public void testEnsureInstanceIsStoppedOnLastKnownHost_agentUnavailableIsSwallowed() throws Exception {
+        StopCommand stopCommand = mock(StopCommand.class);
+        when(vmInstanceMock.getHypervisorType()).thenReturn(HypervisorType.KVM);
+        when(vmInstanceMock.getHostId()).thenReturn(7L);
+        doReturn(stopCommand).when(virtualMachineManagerImpl).buildStopCommand(any(), any(), eq(false));
+        when(agentManagerMock.send(anyLong(), any(StopCommand.class)))
+                .thenThrow(new AgentUnavailableException("down", 7L));
+
+        // must not propagate, expunge continues
+        virtualMachineManagerImpl.ensureInstanceIsStoppedOnLastKnownHost(vmInstanceMock);
+
+        verify(agentManagerMock, times(1)).send(anyLong(), any(StopCommand.class));
+    }
+
+    /**
+     * A missing report is weak evidence, so the stop must not be forced: sendStop() answers success for an
+     * unreachable host when forced, which would free the addresses of an instance that is still running.
+     */
+    @Test
+    public void testHandlePowerOffReport_missingReportUsesUnforcedStopAndKeepsResources() {
+        when(vmInstanceMock.getState()).thenReturn(VirtualMachine.State.Migrating);
+        when(vmInstanceMock.getPowerState()).thenReturn(VirtualMachine.PowerState.PowerReportMissing);
+        doReturn(mock(VirtualMachineGuru.class)).when(virtualMachineManagerImpl).getVmGuru(any());
+        doReturn(new Pair<>(false, "host did not answer")).when(virtualMachineManagerImpl)
+                .sendStop(any(), any(), eq(false), eq(true));
+
+        virtualMachineManagerImpl.handlePowerOffReportWithNoPendingJobsOnVM(vmInstanceMock);
+
+        verify(virtualMachineManagerImpl, times(1)).sendStop(any(), any(), eq(false), eq(true));
+        verify(virtualMachineManagerImpl, never()).releaseVmResources(any(), anyBoolean());
+    }
+
+    /**
+     * A PowerOff report is the host stating the instance is down, so that path keeps its forced stop.
+     */
+    @Test
+    public void testHandlePowerOffReport_powerOffKeepsForcedStop() {
+        when(vmInstanceMock.getState()).thenReturn(VirtualMachine.State.Migrating);
+        when(vmInstanceMock.getPowerState()).thenReturn(VirtualMachine.PowerState.PowerOff);
+        doReturn(mock(VirtualMachineGuru.class)).when(virtualMachineManagerImpl).getVmGuru(any());
+        doReturn(new Pair<>(false, "host did not answer")).when(virtualMachineManagerImpl)
+                .sendStop(any(), any(), eq(true), eq(true));
+
+        virtualMachineManagerImpl.handlePowerOffReportWithNoPendingJobsOnVM(vmInstanceMock);
+
+        verify(virtualMachineManagerImpl, times(1)).sendStop(any(), any(), eq(true), eq(true));
+        verify(virtualMachineManagerImpl, never()).releaseVmResources(any(), anyBoolean());
+    }
 }

engine/orchestration/src/test/java/com/cloud/vm/VirtualMachinePowerStateSyncImplTest.java

@@ -36,6 +36,7 @@
 import org.mockito.junit.MockitoJUnitRunner;
 
 import com.cloud.agent.api.HostVmStateReportEntry;
+import com.cloud.alert.AlertManager;
 import com.cloud.configuration.ManagementServiceConfiguration;
 import com.cloud.host.HostVO;
 import com.cloud.host.dao.HostDao;
@@ -51,6 +52,8 @@ public class VirtualMachinePowerStateSyncImplTest {
     HostDao hostDao;
     @Mock
     ManagementServiceConfiguration mgmtServiceConf;
+    @Mock
+    AlertManager alertManager;
 
     @InjectMocks
     VirtualMachinePowerStateSyncImpl virtualMachinePowerStateSync = new VirtualMachinePowerStateSyncImpl();