Get Diagnostics: Download logs and diagnostics data from SSVM, CPVM, Router

api/src/main/java/com/cloud/agent/api/to/DataObjectType.java

@@ -19,5 +19,5 @@
 package com.cloud.agent.api.to;
 
 public enum DataObjectType {
-    VOLUME, SNAPSHOT, TEMPLATE
+    VOLUME, SNAPSHOT, TEMPLATE, ARCHIVE
 }

api/src/main/java/com/cloud/storage/Storage.java

@@ -16,11 +16,11 @@
 // under the License.
 package com.cloud.storage;
 
-import org.apache.commons.lang.NotImplementedException;
-
 import java.util.ArrayList;
 import java.util.List;
 
+import org.apache.commons.lang.NotImplementedException;
+
 public class Storage {
     public static enum ImageFormat {
         QCOW2(true, true, false, "qcow2"),
@@ -33,6 +33,7 @@ public static enum ImageFormat {
         VMDK(true, true, false, "vmdk"),
         VDI(true, true, false, "vdi"),
         TAR(false, false, false, "tar"),
+        ZIP(false, false, false, "zip"),
         DIR(false, false, false, "dir");
 
         private final boolean supportThinProvisioning;

api/src/main/java/org/apache/cloudstack/api/ApiConstants.java

@@ -742,6 +742,7 @@ public class ApiConstants {
     public static final String STDERR = "stderr";
     public static final String EXITCODE = "exitcode";
     public static final String TARGET_ID = "targetid";
+    public static final String FILES = "files";
     public static final String VOLUME_IDS = "volumeids";
 
     public enum HostDetails {

api/src/main/java/org/apache/cloudstack/api/command/admin/diagnostics/GetDiagnosticsDataCmd.java

@@ -0,0 +1,157 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.command.admin.diagnostics;
+
+import java.util.List;
+
+import javax.inject.Inject;
+
+import org.apache.cloudstack.acl.RoleType;
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiArgValidator;
+import org.apache.cloudstack.api.ApiCommandJobType;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.BaseAsyncCmd;
+import org.apache.cloudstack.api.BaseCmd;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.ServerApiException;
+import org.apache.cloudstack.api.response.SystemVmResponse;
+import org.apache.cloudstack.api.response.diagnostics.GetDiagnosticsDataResponse;
+import org.apache.cloudstack.context.CallContext;
+import org.apache.cloudstack.diagnostics.DiagnosticsService;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.validator.routines.UrlValidator;
+
+import com.cloud.event.EventTypes;
+import com.cloud.exception.InsufficientCapacityException;
+import com.cloud.exception.ResourceUnavailableException;
+import com.cloud.user.Account;
+import com.cloud.utils.exception.CloudRuntimeException;
+import com.cloud.vm.VirtualMachine;
+
+@APICommand(name = GetDiagnosticsDataCmd.APINAME,
+        responseObject = GetDiagnosticsDataResponse.class,
+        entityType = {VirtualMachine.class},
+        responseHasSensitiveInfo = false,
+        requestHasSensitiveInfo = false,
+        description = "Get diagnostics and files from system VMs",
+        since = "4.14.0.0",
+        authorized = {RoleType.Admin})
+public class GetDiagnosticsDataCmd extends BaseAsyncCmd {
+    public static final String APINAME = "getDiagnosticsData";
+
+    @Inject
+    private DiagnosticsService diagnosticsService;
+
+    /////////////////////////////////////////////////////
+    //////////////// API parameters /////////////////////
+    /////////////////////////////////////////////////////
+    @Parameter(name = ApiConstants.TARGET_ID,
+            type = BaseCmd.CommandType.UUID,
+            entityType = SystemVmResponse.class,
+            required = true,
+            validations = {ApiArgValidator.PositiveNumber},
+            description = "The ID of the system VM instance to retrieve diagnostics data files from")
+    private Long id;
+
+    @Parameter(name = ApiConstants.FILES,
+            type = BaseCmd.CommandType.LIST,
+            collectionType = BaseCmd.CommandType.STRING,
+            description = "A comma separated list of diagnostics data files to be retrieved. Defaults are taken from global settings if none has been provided.")
+    private List<String> filesList;
+
+    /////////////////////////////////////////////////////
+    /////////////////// Accessors ///////////////////////
+    /////////////////////////////////////////////////////
+
+    public Long getId() {
+        return id;
+    }
+
+    public List<String> getFilesList() {
+        return filesList;
+    }
+
+    /////////////////////////////////////////////////////
+    /////////////////// Implementation //////////////////
+    /////////////////////////////////////////////////////
+
+    @Override
+    public String getCommandName() {
+        return APINAME.toLowerCase() + BaseCmd.RESPONSE_SUFFIX;
+    }
+
+    @Override
+    public long getEntityOwnerId() {
+        Account account = CallContext.current().getCallingAccount();
+        if (account != null) {
+            return account.getId();
+        }
+        return Account.ACCOUNT_ID_SYSTEM;
+    }
+
+    @Override
+    public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException {
+        try {
+            String downloadUrl = diagnosticsService.getDiagnosticsDataCommand(this);
+            UrlValidator urlValidator = new UrlValidator();
+            if (StringUtils.isEmpty(downloadUrl)) {
+                throw new CloudRuntimeException("Failed to retrieve diagnostics files");
+            }
+            GetDiagnosticsDataResponse response = new GetDiagnosticsDataResponse();
+            if (urlValidator.isValid(downloadUrl)){
+                response.setUrl(downloadUrl);
+                response.setObjectName("diagnostics");
+                response.setResponseName(getCommandName());
+                this.setResponseObject(response);
+            } else {
+                throw new CloudRuntimeException("failed to generate valid download url: " + downloadUrl);
+            }
+        } catch (ServerApiException e) {
+            throw new CloudRuntimeException("Internal exception caught while retrieving diagnostics files: ", e);
+        }
+    }
+
+    @Override
+    public String getEventType() {
+        VirtualMachine.Type vmType = _entityMgr.findById(VirtualMachine.class, getId()).getType();
+        String eventType = "";
+        switch (vmType) {
+            case ConsoleProxy:
+                eventType = EventTypes.EVENT_PROXY_DIAGNOSTICS;
+                break;
+            case SecondaryStorageVm:
+                eventType = EventTypes.EVENT_SSVM_DIAGNOSTICS;
+                break;
+            case DomainRouter:
+                eventType = EventTypes.EVENT_ROUTER_DIAGNOSTICS;
+                break;
+        }
+        return eventType;
+    }
+
+    @Override
+    public String getEventDescription() {
+        return "Getting diagnostics data files from system vm: " + this._uuidMgr.getUuid(VirtualMachine.class, getId());
+    }
+
+    @Override
+    public ApiCommandJobType getInstanceType() {
+        return ApiCommandJobType.SystemVm;
+    }
+
+}

api/src/main/java/org/apache/cloudstack/api/response/diagnostics/GetDiagnosticsDataResponse.java

@@ -0,0 +1,40 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.response.diagnostics;
+
+import com.cloud.serializer.Param;
+import com.cloud.vm.VirtualMachine;
+import com.google.gson.annotations.SerializedName;
+
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.BaseResponse;
+import org.apache.cloudstack.api.EntityReference;
+
+@EntityReference(value = VirtualMachine.class)
+public class GetDiagnosticsDataResponse extends BaseResponse {
+    @SerializedName(ApiConstants.URL)
+    @Param(description = "Storage URL to download retrieve diagnostics data files")
+    private String url;
+
+    public String getUrl() {
+        return url;
+    }
+
+    public void setUrl(String url) {
+        this.url = url;
+    }
+}

api/src/main/java/org/apache/cloudstack/diagnostics/DiagnosticsService.java

@@ -18,12 +18,16 @@
 //
 package org.apache.cloudstack.diagnostics;
 
-import org.apache.cloudstack.api.command.admin.diagnostics.RunDiagnosticsCmd;
-
 import java.util.Map;
 
+import org.apache.cloudstack.api.command.admin.diagnostics.GetDiagnosticsDataCmd;
+import org.apache.cloudstack.api.command.admin.diagnostics.RunDiagnosticsCmd;
+
 public interface DiagnosticsService {
 
+    String DIAGNOSTICS_DIRECTORY = "diagnostics";
+
     Map<String, String> runDiagnosticsCommand(RunDiagnosticsCmd cmd);
 
+    String getDiagnosticsDataCommand(GetDiagnosticsDataCmd getDiagnosticsDataCmd);
 }

core/src/main/java/com/cloud/agent/resource/virtualnetwork/VRScripts.java

@@ -70,4 +70,6 @@ public class VRScripts {
     public static final String VR_CFG = "vr_cfg.sh";
 
     public static final String DIAGNOSTICS = "diagnostics.py";
+    public static final String RETRIEVE_DIAGNOSTICS = "get_diagnostics_files.py";
+    public static final String VR_FILE_CLEANUP = "cleanup.sh";
 }

core/src/main/java/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java

@@ -23,8 +23,11 @@
 import java.net.InetSocketAddress;
 import java.nio.channels.SocketChannel;
 
+import org.apache.cloudstack.diagnostics.DeleteFileInVrCommand;
 import org.apache.cloudstack.diagnostics.DiagnosticsAnswer;
 import org.apache.cloudstack.diagnostics.DiagnosticsCommand;
+import org.apache.cloudstack.diagnostics.PrepareFilesAnswer;
+import org.apache.cloudstack.diagnostics.PrepareFilesCommand;
 import org.joda.time.Duration;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -196,7 +199,11 @@ private Answer executeQueryCommand(NetworkElementCommand cmd) {
         } else if (cmd instanceof GetRouterAlertsCommand) {
             return execute((GetRouterAlertsCommand)cmd);
         } else if (cmd instanceof DiagnosticsCommand) {
-            return execute((DiagnosticsCommand)cmd);
+            return execute((DiagnosticsCommand) cmd);
+        } else if (cmd instanceof PrepareFilesCommand) {
+            return execute((PrepareFilesCommand) cmd);
+        } else if (cmd instanceof DeleteFileInVrCommand) {
+            return execute((DeleteFileInVrCommand)cmd);
         } else {
             s_logger.error("Unknown query command in VirtualRoutingResource!");
             return Answer.createUnsupportedCommandAnswer(cmd);
@@ -306,6 +313,24 @@ private Answer execute(DiagnosticsCommand cmd) {
         return new DiagnosticsAnswer(cmd, result.isSuccess(), result.getDetails());
     }
 
+    private Answer execute(PrepareFilesCommand cmd) {
+        String fileList = String.join(" ", cmd.getFilesToRetrieveList());
+        _eachTimeout = Duration.standardSeconds(cmd.getTimeout());
+        final ExecutionResult result = _vrDeployer.executeInVR(cmd.getRouterAccessIp(), VRScripts.RETRIEVE_DIAGNOSTICS, fileList, _eachTimeout);
+        if (result.isSuccess()) {
+            return new PrepareFilesAnswer(cmd, true, result.getDetails());
+        }
+        return new PrepareFilesAnswer(cmd, false, result.getDetails());
+    }
+
+    private Answer execute(DeleteFileInVrCommand cmd) {
+        ExecutionResult result = _vrDeployer.executeInVR(cmd.getRouterAccessIp(), VRScripts.VR_FILE_CLEANUP, cmd.getFileName());
+        if (result.isSuccess()) {
+            return new Answer(cmd, result.isSuccess(), result.getDetails());
+        }
+        return new Answer(cmd, result.isSuccess(), result.getDetails());
+    }
+
     private Answer execute(GetDomRVersionCmd cmd) {
         final ExecutionResult result = _vrDeployer.executeInVR(cmd.getRouterAccessIp(), VRScripts.VERSION, null);
         if (!result.isSuccess()) {

core/src/main/java/org/apache/cloudstack/diagnostics/CopyToSecondaryStorageAnswer.java

@@ -0,0 +1,26 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.diagnostics;
+
+import com.cloud.agent.api.Answer;
+
+public class CopyToSecondaryStorageAnswer extends Answer {
+
+    public CopyToSecondaryStorageAnswer(CopyToSecondaryStorageCommand command, boolean success, String details) {
+        super(command, success, details);
+    }
+}

core/src/main/java/org/apache/cloudstack/diagnostics/CopyToSecondaryStorageCommand.java

@@ -0,0 +1,53 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.diagnostics;
+
+import org.apache.cloudstack.storage.command.StorageSubSystemCommand;
+
+public class CopyToSecondaryStorageCommand extends StorageSubSystemCommand {
+    private String secondaryStorageUrl;
+    private String systemVmIp;
+    private String fileName;
+
+    public CopyToSecondaryStorageCommand(String secondaryStorageUrl, String systemVmIp, String fileName) {
+        this.secondaryStorageUrl = secondaryStorageUrl;
+        this.systemVmIp = systemVmIp;
+        this.fileName = fileName;
+    }
+
+    public String getSecondaryStorageUrl() {
+        return secondaryStorageUrl;
+    }
+
+    public String getSystemVmIp() {
+        return systemVmIp;
+    }
+
+    public String getFileName() {
+        return fileName;
+    }
+
+    @Override
+    public boolean executeInSequence() {
+        return false;
+    }
+
+    @Override
+    public void setExecuteInSequence(boolean inSeq) {
+
+    }
+}