mirror of
https://github.com/portainer/portainer.git
synced 2025-07-19 05:19:39 +02:00
feat(support): collect system info bundle to assist support troubleshooting [r8s-157] (#154)
This commit is contained in:
Malcolm Lockyer
GitHub
parent
17648d12fe
commit
783ab253af
17 changed files with 1367 additions and 440 deletions
File diff suppressed because it is too large
Load diff
|
|
@ -672,6 +672,7 @@
|
||||||
{
|
{
|
||||||
"Docker": {
|
"Docker": {
|
||||||
"ContainerCount": 0,
|
"ContainerCount": 0,
|
||||||
|
"DiagnosticsData": {},
|
||||||
"DockerSnapshotRaw": {
|
"DockerSnapshotRaw": {
|
||||||
"Containers": null,
|
"Containers": null,
|
||||||
"Images": null,
|
"Images": null,
|
||||||
|
|
|
||||||
|
|
@ -1,20 +1,9 @@
|
||||||
package docker
|
package docker
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
portainer "github.com/portainer/portainer/api"
|
portainer "github.com/portainer/portainer/api"
|
||||||
dockerclient "github.com/portainer/portainer/api/docker/client"
|
dockerclient "github.com/portainer/portainer/api/docker/client"
|
||||||
"github.com/portainer/portainer/api/docker/consts"
|
"github.com/portainer/portainer/pkg/snapshot"
|
||||||
|
|
||||||
"github.com/docker/docker/api/types"
|
|
||||||
"github.com/docker/docker/api/types/container"
|
|
||||||
_container "github.com/docker/docker/api/types/container"
|
|
||||||
"github.com/docker/docker/api/types/volume"
|
|
||||||
"github.com/docker/docker/client"
|
|
||||||
"github.com/rs/zerolog/log"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// Snapshotter represents a service used to create environment(endpoint) snapshots
|
// Snapshotter represents a service used to create environment(endpoint) snapshots
|
||||||
|
|
@ -37,247 +26,5 @@ func (snapshotter *Snapshotter) CreateSnapshot(endpoint *portainer.Endpoint) (*p
|
||||||
}
|
}
|
||||||
defer cli.Close()
|
defer cli.Close()
|
||||||
|
|
||||||
return snapshot(cli, endpoint)
|
return snapshot.CreateDockerSnapshot(cli)
|
||||||
}
|
|
||||||
|
|
||||||
func snapshot(cli *client.Client, endpoint *portainer.Endpoint) (*portainer.DockerSnapshot, error) {
|
|
||||||
if _, err := cli.Ping(context.Background()); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
snapshot := &portainer.DockerSnapshot{
|
|
||||||
StackCount: 0,
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := snapshotInfo(snapshot, cli); err != nil {
|
|
||||||
log.Warn().Str("environment", endpoint.Name).Err(err).Msg("unable to snapshot engine information")
|
|
||||||
}
|
|
||||||
|
|
||||||
if snapshot.Swarm {
|
|
||||||
if err := snapshotSwarmServices(snapshot, cli); err != nil {
|
|
||||||
log.Warn().Str("environment", endpoint.Name).Err(err).Msg("unable to snapshot Swarm services")
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := snapshotNodes(snapshot, cli); err != nil {
|
|
||||||
log.Warn().Str("environment", endpoint.Name).Err(err).Msg("unable to snapshot Swarm nodes")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := snapshotContainers(snapshot, cli); err != nil {
|
|
||||||
log.Warn().Str("environment", endpoint.Name).Err(err).Msg("unable to snapshot containers")
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := snapshotImages(snapshot, cli); err != nil {
|
|
||||||
log.Warn().Str("environment", endpoint.Name).Err(err).Msg("unable to snapshot images")
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := snapshotVolumes(snapshot, cli); err != nil {
|
|
||||||
log.Warn().Str("environment", endpoint.Name).Err(err).Msg("unable to snapshot volumes")
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := snapshotNetworks(snapshot, cli); err != nil {
|
|
||||||
log.Warn().Str("environment", endpoint.Name).Err(err).Msg("unable to snapshot networks")
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := snapshotVersion(snapshot, cli); err != nil {
|
|
||||||
log.Warn().Str("environment", endpoint.Name).Err(err).Msg("unable to snapshot engine version")
|
|
||||||
}
|
|
||||||
|
|
||||||
snapshot.Time = time.Now().Unix()
|
|
||||||
|
|
||||||
return snapshot, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func snapshotInfo(snapshot *portainer.DockerSnapshot, cli *client.Client) error {
|
|
||||||
info, err := cli.Info(context.Background())
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
snapshot.Swarm = info.Swarm.ControlAvailable
|
|
||||||
snapshot.DockerVersion = info.ServerVersion
|
|
||||||
snapshot.TotalCPU = info.NCPU
|
|
||||||
snapshot.TotalMemory = info.MemTotal
|
|
||||||
snapshot.SnapshotRaw.Info = info
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func snapshotNodes(snapshot *portainer.DockerSnapshot, cli *client.Client) error {
|
|
||||||
nodes, err := cli.NodeList(context.Background(), types.NodeListOptions{})
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
var nanoCpus int64
|
|
||||||
var totalMem int64
|
|
||||||
|
|
||||||
for _, node := range nodes {
|
|
||||||
nanoCpus += node.Description.Resources.NanoCPUs
|
|
||||||
totalMem += node.Description.Resources.MemoryBytes
|
|
||||||
}
|
|
||||||
|
|
||||||
snapshot.TotalCPU = int(nanoCpus / 1e9)
|
|
||||||
snapshot.TotalMemory = totalMem
|
|
||||||
snapshot.NodeCount = len(nodes)
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func snapshotSwarmServices(snapshot *portainer.DockerSnapshot, cli *client.Client) error {
|
|
||||||
stacks := make(map[string]struct{})
|
|
||||||
|
|
||||||
services, err := cli.ServiceList(context.Background(), types.ServiceListOptions{})
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, service := range services {
|
|
||||||
for k, v := range service.Spec.Labels {
|
|
||||||
if k == "com.docker.stack.namespace" {
|
|
||||||
stacks[v] = struct{}{}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
snapshot.ServiceCount = len(services)
|
|
||||||
snapshot.StackCount += len(stacks)
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func snapshotContainers(snapshot *portainer.DockerSnapshot, cli *client.Client) error {
|
|
||||||
containers, err := cli.ContainerList(context.Background(), container.ListOptions{All: true})
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
stacks := make(map[string]struct{})
|
|
||||||
gpuUseSet := make(map[string]struct{})
|
|
||||||
gpuUseAll := false
|
|
||||||
|
|
||||||
for _, container := range containers {
|
|
||||||
if container.State == "running" {
|
|
||||||
// Snapshot GPUs
|
|
||||||
response, err := cli.ContainerInspect(context.Background(), container.ID)
|
|
||||||
if err != nil {
|
|
||||||
// Inspect a container will fail when the container runs on a different
|
|
||||||
// Swarm node, so it is better to log the error instead of return error
|
|
||||||
// when the Swarm mode is enabled
|
|
||||||
if !snapshot.Swarm {
|
|
||||||
return err
|
|
||||||
} else {
|
|
||||||
if !strings.Contains(err.Error(), "No such container") {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
// It is common to have containers running on different Swarm nodes,
|
|
||||||
// so we just log the error in the debug level
|
|
||||||
log.Debug().Str("container", container.ID).Err(err).Msg("unable to inspect container in other Swarm nodes")
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
var gpuOptions *_container.DeviceRequest = nil
|
|
||||||
for _, deviceRequest := range response.HostConfig.Resources.DeviceRequests {
|
|
||||||
if deviceRequest.Driver == "nvidia" || deviceRequest.Capabilities[0][0] == "gpu" {
|
|
||||||
gpuOptions = &deviceRequest
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if gpuOptions != nil {
|
|
||||||
if gpuOptions.Count == -1 {
|
|
||||||
gpuUseAll = true
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, id := range gpuOptions.DeviceIDs {
|
|
||||||
gpuUseSet[id] = struct{}{}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for k, v := range container.Labels {
|
|
||||||
if k == consts.ComposeStackNameLabel {
|
|
||||||
stacks[v] = struct{}{}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
gpuUseList := make([]string, 0, len(gpuUseSet))
|
|
||||||
for gpuUse := range gpuUseSet {
|
|
||||||
gpuUseList = append(gpuUseList, gpuUse)
|
|
||||||
}
|
|
||||||
|
|
||||||
snapshot.GpuUseAll = gpuUseAll
|
|
||||||
snapshot.GpuUseList = gpuUseList
|
|
||||||
|
|
||||||
stats := CalculateContainerStats(containers)
|
|
||||||
|
|
||||||
snapshot.ContainerCount = stats.Total
|
|
||||||
snapshot.RunningContainerCount = stats.Running
|
|
||||||
snapshot.StoppedContainerCount = stats.Stopped
|
|
||||||
snapshot.HealthyContainerCount = stats.Healthy
|
|
||||||
snapshot.UnhealthyContainerCount = stats.Unhealthy
|
|
||||||
snapshot.StackCount += len(stacks)
|
|
||||||
|
|
||||||
for _, container := range containers {
|
|
||||||
snapshot.SnapshotRaw.Containers = append(snapshot.SnapshotRaw.Containers, portainer.DockerContainerSnapshot{Container: container})
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func snapshotImages(snapshot *portainer.DockerSnapshot, cli *client.Client) error {
|
|
||||||
images, err := cli.ImageList(context.Background(), types.ImageListOptions{})
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
snapshot.ImageCount = len(images)
|
|
||||||
snapshot.SnapshotRaw.Images = images
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func snapshotVolumes(snapshot *portainer.DockerSnapshot, cli *client.Client) error {
|
|
||||||
volumes, err := cli.VolumeList(context.Background(), volume.ListOptions{})
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
snapshot.VolumeCount = len(volumes.Volumes)
|
|
||||||
snapshot.SnapshotRaw.Volumes = volumes
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func snapshotNetworks(snapshot *portainer.DockerSnapshot, cli *client.Client) error {
|
|
||||||
networks, err := cli.NetworkList(context.Background(), types.NetworkListOptions{})
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
snapshot.SnapshotRaw.Networks = networks
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func snapshotVersion(snapshot *portainer.DockerSnapshot, cli *client.Client) error {
|
|
||||||
version, err := cli.ServerVersion(context.Background())
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
snapshot.SnapshotRaw.Version = version
|
|
||||||
snapshot.IsPodman = isPodman(version)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// isPodman checks if the version is for Podman by checking if any of the components contain "podman".
|
|
||||||
// If it's podman, a component name should be "Podman Engine"
|
|
||||||
func isPodman(version types.Version) bool {
|
|
||||||
for _, component := range version.Components {
|
|
||||||
if strings.Contains(strings.ToLower(component.Name), "podman") {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,15 +1,9 @@
|
||||||
package kubernetes
|
package kubernetes
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
portainer "github.com/portainer/portainer/api"
|
portainer "github.com/portainer/portainer/api"
|
||||||
"github.com/portainer/portainer/api/kubernetes/cli"
|
"github.com/portainer/portainer/api/kubernetes/cli"
|
||||||
|
"github.com/portainer/portainer/pkg/snapshot"
|
||||||
"github.com/rs/zerolog/log"
|
|
||||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
||||||
"k8s.io/client-go/kubernetes"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type Snapshotter struct {
|
type Snapshotter struct {
|
||||||
|
|
@ -30,55 +24,5 @@ func (snapshotter *Snapshotter) CreateSnapshot(endpoint *portainer.Endpoint) (*p
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
return snapshot(client, endpoint)
|
return snapshot.CreateKubernetesSnapshot(client)
|
||||||
}
|
|
||||||
|
|
||||||
func snapshot(cli *kubernetes.Clientset, endpoint *portainer.Endpoint) (*portainer.KubernetesSnapshot, error) {
|
|
||||||
res := cli.RESTClient().Get().AbsPath("/healthz").Do(context.TODO())
|
|
||||||
if res.Error() != nil {
|
|
||||||
return nil, res.Error()
|
|
||||||
}
|
|
||||||
|
|
||||||
snapshot := &portainer.KubernetesSnapshot{}
|
|
||||||
|
|
||||||
err := snapshotVersion(snapshot, cli)
|
|
||||||
if err != nil {
|
|
||||||
log.Warn().Str("endpoint", endpoint.Name).Err(err).Msg("unable to snapshot cluster version")
|
|
||||||
}
|
|
||||||
|
|
||||||
err = snapshotNodes(snapshot, cli)
|
|
||||||
if err != nil {
|
|
||||||
log.Warn().Str("endpoint", endpoint.Name).Err(err).Msg("unable to snapshot cluster nodes")
|
|
||||||
}
|
|
||||||
|
|
||||||
snapshot.Time = time.Now().Unix()
|
|
||||||
return snapshot, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func snapshotVersion(snapshot *portainer.KubernetesSnapshot, cli *kubernetes.Clientset) error {
|
|
||||||
versionInfo, err := cli.ServerVersion()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
snapshot.KubernetesVersion = versionInfo.GitVersion
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func snapshotNodes(snapshot *portainer.KubernetesSnapshot, cli *kubernetes.Clientset) error {
|
|
||||||
nodeList, err := cli.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
var totalCPUs, totalMemory int64
|
|
||||||
for _, node := range nodeList.Items {
|
|
||||||
totalCPUs += node.Status.Capacity.Cpu().Value()
|
|
||||||
totalMemory += node.Status.Capacity.Memory().Value()
|
|
||||||
}
|
|
||||||
|
|
||||||
snapshot.TotalCPU = totalCPUs
|
|
||||||
snapshot.TotalMemory = totalMemory
|
|
||||||
snapshot.NodeCount = len(nodeList.Items)
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -185,6 +185,16 @@ type (
|
||||||
// CustomTemplatePlatform represents a custom template platform
|
// CustomTemplatePlatform represents a custom template platform
|
||||||
CustomTemplatePlatform int
|
CustomTemplatePlatform int
|
||||||
|
|
||||||
|
// DiagnosticsData represents the diagnostics data for an environment
|
||||||
|
// this contains the logs, telnet, traceroute, dns and proxy information
|
||||||
|
// which will be part of the DockerSnapshot and KubernetesSnapshot structs
|
||||||
|
DiagnosticsData struct {
|
||||||
|
Log string `json:"Log,omitempty"`
|
||||||
|
Telnet map[string]string `json:"Telnet,omitempty"`
|
||||||
|
DNS map[string]string `json:"DNS,omitempty"`
|
||||||
|
Proxy map[string]string `json:"Proxy,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
// DockerHub represents all the required information to connect and use the
|
// DockerHub represents all the required information to connect and use the
|
||||||
// Docker Hub
|
// Docker Hub
|
||||||
DockerHub struct {
|
DockerHub struct {
|
||||||
|
|
@ -217,6 +227,7 @@ type (
|
||||||
GpuUseAll bool `json:"GpuUseAll"`
|
GpuUseAll bool `json:"GpuUseAll"`
|
||||||
GpuUseList []string `json:"GpuUseList"`
|
GpuUseList []string `json:"GpuUseList"`
|
||||||
IsPodman bool `json:"IsPodman"`
|
IsPodman bool `json:"IsPodman"`
|
||||||
|
DiagnosticsData *DiagnosticsData `json:"DiagnosticsData"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// DockerContainerSnapshot is an extent of Docker's Container struct
|
// DockerContainerSnapshot is an extent of Docker's Container struct
|
||||||
|
|
@ -641,6 +652,7 @@ type (
|
||||||
NodeCount int `json:"NodeCount"`
|
NodeCount int `json:"NodeCount"`
|
||||||
TotalCPU int64 `json:"TotalCPU"`
|
TotalCPU int64 `json:"TotalCPU"`
|
||||||
TotalMemory int64 `json:"TotalMemory"`
|
TotalMemory int64 `json:"TotalMemory"`
|
||||||
|
DiagnosticsData *DiagnosticsData `json:"DiagnosticsData"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// KubernetesConfiguration represents the configuration of a Kubernetes environment(endpoint)
|
// KubernetesConfiguration represents the configuration of a Kubernetes environment(endpoint)
|
||||||
|
|
|
||||||
4
go.mod
4
go.mod
|
|
@ -10,6 +10,7 @@ require (
|
||||||
github.com/aws/aws-sdk-go-v2 v1.24.1
|
github.com/aws/aws-sdk-go-v2 v1.24.1
|
||||||
github.com/aws/aws-sdk-go-v2/credentials v1.16.16
|
github.com/aws/aws-sdk-go-v2/credentials v1.16.16
|
||||||
github.com/aws/aws-sdk-go-v2/service/ecr v1.24.1
|
github.com/aws/aws-sdk-go-v2/service/ecr v1.24.1
|
||||||
|
github.com/aws/smithy-go v1.19.0
|
||||||
github.com/cbroglie/mustache v1.4.0
|
github.com/cbroglie/mustache v1.4.0
|
||||||
github.com/compose-spec/compose-go/v2 v2.0.2
|
github.com/compose-spec/compose-go/v2 v2.0.2
|
||||||
github.com/containers/image/v5 v5.30.1
|
github.com/containers/image/v5 v5.30.1
|
||||||
|
|
@ -41,7 +42,7 @@ require (
|
||||||
github.com/robfig/cron/v3 v3.0.1
|
github.com/robfig/cron/v3 v3.0.1
|
||||||
github.com/rs/zerolog v1.29.0
|
github.com/rs/zerolog v1.29.0
|
||||||
github.com/segmentio/encoding v0.3.6
|
github.com/segmentio/encoding v0.3.6
|
||||||
github.com/stretchr/testify v1.9.0
|
github.com/stretchr/testify v1.10.0
|
||||||
github.com/urfave/negroni v1.0.0
|
github.com/urfave/negroni v1.0.0
|
||||||
github.com/viney-shih/go-lock v1.1.1
|
github.com/viney-shih/go-lock v1.1.1
|
||||||
go.etcd.io/bbolt v1.3.10
|
go.etcd.io/bbolt v1.3.10
|
||||||
|
|
@ -86,7 +87,6 @@ require (
|
||||||
github.com/aws/aws-sdk-go-v2/service/sso v1.18.7 // indirect
|
github.com/aws/aws-sdk-go-v2/service/sso v1.18.7 // indirect
|
||||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.7 // indirect
|
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.7 // indirect
|
||||||
github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 // indirect
|
github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 // indirect
|
||||||
github.com/aws/smithy-go v1.19.0 // indirect
|
|
||||||
github.com/beorn7/perks v1.0.1 // indirect
|
github.com/beorn7/perks v1.0.1 // indirect
|
||||||
github.com/buger/goterm v1.0.4 // indirect
|
github.com/buger/goterm v1.0.4 // indirect
|
||||||
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
|
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
|
||||||
|
|
|
||||||
4
go.sum
4
go.sum
|
|
@ -618,8 +618,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
|
||||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||||
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||||
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
|
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
|
||||||
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||||
github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI=
|
github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI=
|
||||||
github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
|
github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
|
||||||
github.com/theupdateframework/notary v0.7.0 h1:QyagRZ7wlSpjT5N2qQAh/pN+DVqgekv4DzbAiAiEL3c=
|
github.com/theupdateframework/notary v0.7.0 h1:QyagRZ7wlSpjT5N2qQAh/pN+DVqgekv4DzbAiAiEL3c=
|
||||||
|
|
|
||||||
30
pkg/edge/utils.go
Normal file
30
pkg/edge/utils.go
Normal file
|
|
@ -0,0 +1,30 @@
|
||||||
|
package edge
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/base64"
|
||||||
|
"errors"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// GetPortainerURLFromEdgeKey returns the portainer URL from an edge key
|
||||||
|
// format: <portainer_instance_url>|<tunnel_server_addr>|<tunnel_server_fingerprint>|<endpoint_id>
|
||||||
|
func GetPortainerURLFromEdgeKey(edgeKey string) (string, error) {
|
||||||
|
decodedKey, err := base64.RawStdEncoding.DecodeString(edgeKey)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
keyInfo := strings.Split(string(decodedKey), "|")
|
||||||
|
|
||||||
|
if len(keyInfo) != 4 {
|
||||||
|
return "", errors.New("invalid key format")
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = strconv.Atoi(keyInfo[3])
|
||||||
|
if err != nil {
|
||||||
|
return "", errors.New("invalid key format")
|
||||||
|
}
|
||||||
|
|
||||||
|
return keyInfo[0], nil
|
||||||
|
}
|
||||||
29
pkg/edge/utils_test.go
Normal file
29
pkg/edge/utils_test.go
Normal file
|
|
@ -0,0 +1,29 @@
|
||||||
|
package edge
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestGetPortainerURLFromEdgeKey(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
edgeKey string
|
||||||
|
expected string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "ValidEdgeKey",
|
||||||
|
edgeKey: "aHR0cHM6Ly9wb3J0YWluZXIuaW98cG9ydGFpbmVyLmlvOjgwMDB8YXNkZnwx",
|
||||||
|
expected: "https://portainer.io",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
result, err := GetPortainerURLFromEdgeKey(tt.edgeKey)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
assert.Equal(t, tt.expected, result)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -1,6 +1,14 @@
|
||||||
package endpoints
|
package endpoints
|
||||||
|
|
||||||
import portainer "github.com/portainer/portainer/api"
|
import (
|
||||||
|
portainer "github.com/portainer/portainer/api"
|
||||||
|
)
|
||||||
|
|
||||||
|
// IsRegularAgentEndpoint returns true if this is a regular agent endpoint
|
||||||
|
func IsRegularAgentEndpoint(endpoint *portainer.Endpoint) bool {
|
||||||
|
return endpoint.Type == portainer.AgentOnDockerEnvironment ||
|
||||||
|
endpoint.Type == portainer.AgentOnKubernetesEnvironment
|
||||||
|
}
|
||||||
|
|
||||||
// IsEdgeEndpoint returns true if this is an Edge endpoint
|
// IsEdgeEndpoint returns true if this is an Edge endpoint
|
||||||
func IsEdgeEndpoint(endpoint *portainer.Endpoint) bool {
|
func IsEdgeEndpoint(endpoint *portainer.Endpoint) bool {
|
||||||
|
|
|
||||||
|
|
@ -7,6 +7,50 @@ import (
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func TestIsRegularAgentEndpoint(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
endpoint *portainer.Endpoint
|
||||||
|
expected bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "AgentOnDockerEnvironment",
|
||||||
|
endpoint: &portainer.Endpoint{
|
||||||
|
Type: portainer.AgentOnDockerEnvironment,
|
||||||
|
},
|
||||||
|
expected: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "AgentOnKubernetesEnvironment",
|
||||||
|
endpoint: &portainer.Endpoint{
|
||||||
|
Type: portainer.AgentOnKubernetesEnvironment,
|
||||||
|
},
|
||||||
|
expected: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "EdgeAgentOnDockerEnvironment",
|
||||||
|
endpoint: &portainer.Endpoint{
|
||||||
|
Type: portainer.EdgeAgentOnDockerEnvironment,
|
||||||
|
},
|
||||||
|
expected: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "EdgeAgentOnKubernetesEnvironment",
|
||||||
|
endpoint: &portainer.Endpoint{
|
||||||
|
Type: portainer.EdgeAgentOnKubernetesEnvironment,
|
||||||
|
},
|
||||||
|
expected: false,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
result := IsRegularAgentEndpoint(tt.endpoint)
|
||||||
|
assert.Equal(t, tt.expected, result)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestIsEdgeEndpoint(t *testing.T) {
|
func TestIsEdgeEndpoint(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
|
|
|
||||||
136
pkg/networking/diagnostics.go
Normal file
136
pkg/networking/diagnostics.go
Normal file
|
|
@ -0,0 +1,136 @@
|
||||||
|
package networking
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/tls"
|
||||||
|
"fmt"
|
||||||
|
"net"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/segmentio/encoding/json"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ProbeDNSConnection probes a DNS connection and returns a JSON string with the DNS lookup status and IP addresses.
|
||||||
|
// ignores errors for the dns lookup since we want to know if the host is reachable
|
||||||
|
func ProbeDNSConnection(url string) string {
|
||||||
|
_, host, _ := parseURL(url)
|
||||||
|
result := map[string]interface{}{
|
||||||
|
"operation": "dns lookup",
|
||||||
|
"remote_address": host,
|
||||||
|
"connected_at": time.Now().Format(time.RFC3339),
|
||||||
|
"status": "dns lookup successful",
|
||||||
|
"resolved_ips": []net.IP{},
|
||||||
|
}
|
||||||
|
|
||||||
|
ipAddresses, err := net.LookupIP(host)
|
||||||
|
if err != nil {
|
||||||
|
result["status"] = fmt.Sprintf("dns lookup failed: %s", err)
|
||||||
|
} else {
|
||||||
|
result["resolved_ips"] = ipAddresses
|
||||||
|
}
|
||||||
|
|
||||||
|
jsonData, _ := json.Marshal(result)
|
||||||
|
return string(jsonData)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ProbeTelnetConnection probes a telnet connection and returns a JSON string with the telnet connection status, local and remote addresses.
|
||||||
|
// ignores errors for the telnet connection since we want to know if the host is reachable
|
||||||
|
func ProbeTelnetConnection(url string) string {
|
||||||
|
network, host, port := parseURL(url)
|
||||||
|
if network == "https" || network == "http" {
|
||||||
|
network = "tcp"
|
||||||
|
}
|
||||||
|
|
||||||
|
address := fmt.Sprintf("%s:%s", host, port)
|
||||||
|
result := map[string]string{
|
||||||
|
"operation": "telnet connection",
|
||||||
|
"local_address": "unknown",
|
||||||
|
"remote_address": "unknown",
|
||||||
|
"network": network,
|
||||||
|
"status": "connected to " + address,
|
||||||
|
"connected_at": time.Now().Format(time.RFC3339),
|
||||||
|
}
|
||||||
|
|
||||||
|
connection, err := net.DialTimeout(network, address, 5*time.Second)
|
||||||
|
if err != nil {
|
||||||
|
result["status"] = fmt.Sprintf("failed to connect to %s: %s", address, err)
|
||||||
|
} else {
|
||||||
|
defer connection.Close()
|
||||||
|
result["local_address"] = connection.LocalAddr().String()
|
||||||
|
result["remote_address"] = connection.RemoteAddr().String()
|
||||||
|
}
|
||||||
|
|
||||||
|
jsonData, _ := json.Marshal(result)
|
||||||
|
return string(jsonData)
|
||||||
|
}
|
||||||
|
|
||||||
|
// DetectProxy probes a target URL and returns a JSON string with the proxy detection status, local and remote addresses.
|
||||||
|
// ignores errors for the http request since we want to know if the host is reachable
|
||||||
|
func DetectProxy(url string) string {
|
||||||
|
client := &http.Client{
|
||||||
|
Transport: &http.Transport{
|
||||||
|
TLSClientConfig: &tls.Config{
|
||||||
|
InsecureSkipVerify: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
Timeout: 10 * time.Second,
|
||||||
|
}
|
||||||
|
|
||||||
|
result := map[string]string{
|
||||||
|
"operation": "proxy detection",
|
||||||
|
"local_address": "unknown",
|
||||||
|
"remote_address": "unknown",
|
||||||
|
"network": "https",
|
||||||
|
"status": "no proxy detected",
|
||||||
|
"connected_at": time.Now().Format(time.RFC3339),
|
||||||
|
}
|
||||||
|
|
||||||
|
resp, err := client.Get(url)
|
||||||
|
if err != nil {
|
||||||
|
result["status"] = fmt.Sprintf("failed to make request: %s", err)
|
||||||
|
} else {
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.Request != nil {
|
||||||
|
result["local_address"] = resp.Request.Host
|
||||||
|
result["remote_address"] = resp.Request.RemoteAddr
|
||||||
|
}
|
||||||
|
|
||||||
|
if resp.Header.Get("Via") != "" || resp.Header.Get("X-Forwarded-For") != "" || resp.Header.Get("Proxy-Connection") != "" {
|
||||||
|
result["status"] = "proxy detected via headers"
|
||||||
|
} else if resp.TLS != nil && len(resp.TLS.PeerCertificates) > 0 {
|
||||||
|
cert := resp.TLS.PeerCertificates[0]
|
||||||
|
if cert.IsCA || strings.Contains(strings.ToLower(cert.Issuer.CommonName), "proxy") {
|
||||||
|
result["status"] = "proxy detected via certificate"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
jsonData, _ := json.Marshal(result)
|
||||||
|
return string(jsonData)
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseURL parses a raw URL and returns the network, host and port
|
||||||
|
// it also ensures the network is tcp and the port is set to the default for the network
|
||||||
|
func parseURL(rawURL string) (network, host, port string) {
|
||||||
|
u, err := url.Parse(rawURL)
|
||||||
|
if err != nil {
|
||||||
|
return "", "", ""
|
||||||
|
}
|
||||||
|
|
||||||
|
network = u.Scheme
|
||||||
|
host = u.Hostname()
|
||||||
|
port = u.Port()
|
||||||
|
|
||||||
|
if port == "" {
|
||||||
|
if network == "https" {
|
||||||
|
port = "443"
|
||||||
|
} else if network == "http" {
|
||||||
|
port = "80"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return network, host, port
|
||||||
|
}
|
||||||
164
pkg/networking/diagnostics_test.go
Normal file
164
pkg/networking/diagnostics_test.go
Normal file
|
|
@ -0,0 +1,164 @@
|
||||||
|
package networking
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Response structs for each function
|
||||||
|
type dnsResponse struct {
|
||||||
|
Operation string `json:"operation"`
|
||||||
|
ResolvedIPs []string `json:"resolved_ips"`
|
||||||
|
RemoteAddr string `json:"remote_address"`
|
||||||
|
ConnectedAt string `json:"connected_at"`
|
||||||
|
Status string `json:"status"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type telnetResponse struct {
|
||||||
|
Operation string `json:"operation"`
|
||||||
|
LocalAddr string `json:"local_address"`
|
||||||
|
RemoteAddr string `json:"remote_address"`
|
||||||
|
Network string `json:"network"`
|
||||||
|
Status string `json:"status"`
|
||||||
|
ConnectedAt string `json:"connected_at"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type proxyResponse struct {
|
||||||
|
Operation string `json:"operation"`
|
||||||
|
LocalAddr string `json:"local_address"`
|
||||||
|
RemoteAddr string `json:"remote_address"`
|
||||||
|
Network string `json:"network"`
|
||||||
|
Status string `json:"status"`
|
||||||
|
ConnectedAt string `json:"connected_at"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProbeDNSConnection(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
host string
|
||||||
|
wantSuccess bool
|
||||||
|
statusContains string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "Valid domain",
|
||||||
|
host: "https://api.portainer.io",
|
||||||
|
wantSuccess: true,
|
||||||
|
statusContains: "dns lookup successful",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "Invalid domain",
|
||||||
|
host: "https://nonexistent.domain.invalid",
|
||||||
|
wantSuccess: false,
|
||||||
|
statusContains: "dns lookup failed",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
response := ProbeDNSConnection(tt.host)
|
||||||
|
|
||||||
|
var result dnsResponse
|
||||||
|
if err := json.Unmarshal([]byte(response), &result); err != nil {
|
||||||
|
t.Fatalf("Invalid JSON response: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !strings.Contains(result.Status, tt.statusContains) {
|
||||||
|
t.Errorf("Status should contain '%s', got: %s", tt.statusContains, result.Status)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProbeTelnetConnection(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
url string
|
||||||
|
wantSuccess bool
|
||||||
|
statusContains string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "Valid connection",
|
||||||
|
url: "https://api.portainer.io",
|
||||||
|
wantSuccess: true,
|
||||||
|
statusContains: "connected to",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "Invalid port",
|
||||||
|
url: "https://api.portainer.io:99999",
|
||||||
|
wantSuccess: false,
|
||||||
|
statusContains: "failed to connect",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
response := ProbeTelnetConnection(tt.url)
|
||||||
|
|
||||||
|
var result telnetResponse
|
||||||
|
if err := json.Unmarshal([]byte(response), &result); err != nil {
|
||||||
|
t.Fatalf("Invalid JSON response: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
validateCommonFields(t, result.Operation, "telnet connection", result.ConnectedAt)
|
||||||
|
if !strings.Contains(result.Status, tt.statusContains) {
|
||||||
|
t.Errorf("Status should contain '%s', got: %s", tt.statusContains, result.Status)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDetectProxy(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
url string
|
||||||
|
wantSuccess bool
|
||||||
|
statusContains string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "Valid URL",
|
||||||
|
url: "https://api.portainer.io",
|
||||||
|
wantSuccess: true,
|
||||||
|
statusContains: "proxy",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "Invalid URL",
|
||||||
|
url: "https://nonexistent.domain.invalid",
|
||||||
|
wantSuccess: false,
|
||||||
|
statusContains: "failed to make request",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
response := DetectProxy(tt.url)
|
||||||
|
|
||||||
|
var result proxyResponse
|
||||||
|
if err := json.Unmarshal([]byte(response), &result); err != nil {
|
||||||
|
t.Fatalf("Invalid JSON response: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
validateCommonFields(t, result.Operation, "proxy detection", result.ConnectedAt)
|
||||||
|
if result.Network != "https" {
|
||||||
|
t.Errorf("Expected network https, got %s", result.Network)
|
||||||
|
}
|
||||||
|
if !strings.Contains(result.Status, tt.statusContains) {
|
||||||
|
t.Errorf("Status should contain '%s', got: %s", tt.statusContains, result.Status)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Helper function to validate common fields across all responses
|
||||||
|
func validateCommonFields(t *testing.T, operation, expectedOperation, connectedAt string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
if operation != expectedOperation {
|
||||||
|
t.Errorf("Expected operation '%s', got '%s'", expectedOperation, operation)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := time.Parse(time.RFC3339, connectedAt); err != nil {
|
||||||
|
t.Errorf("Invalid connected_at timestamp: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
372
pkg/snapshot/docker.go
Normal file
372
pkg/snapshot/docker.go
Normal file
|
|
@ -0,0 +1,372 @@
|
||||||
|
package snapshot
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/segmentio/encoding/json"
|
||||||
|
|
||||||
|
"github.com/docker/docker/api/types"
|
||||||
|
"github.com/docker/docker/api/types/container"
|
||||||
|
_container "github.com/docker/docker/api/types/container"
|
||||||
|
"github.com/docker/docker/api/types/image"
|
||||||
|
"github.com/docker/docker/api/types/volume"
|
||||||
|
"github.com/docker/docker/client"
|
||||||
|
"github.com/docker/docker/pkg/stdcopy"
|
||||||
|
portainer "github.com/portainer/portainer/api"
|
||||||
|
"github.com/portainer/portainer/api/docker/consts"
|
||||||
|
edgeutils "github.com/portainer/portainer/pkg/edge"
|
||||||
|
networkingutils "github.com/portainer/portainer/pkg/networking"
|
||||||
|
"github.com/rs/zerolog/log"
|
||||||
|
)
|
||||||
|
|
||||||
|
func CreateDockerSnapshot(cli *client.Client) (*portainer.DockerSnapshot, error) {
|
||||||
|
if _, err := cli.Ping(context.Background()); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
dockerSnapshot := &portainer.DockerSnapshot{
|
||||||
|
StackCount: 0,
|
||||||
|
}
|
||||||
|
|
||||||
|
err := dockerSnapshotInfo(dockerSnapshot, cli)
|
||||||
|
if err != nil {
|
||||||
|
log.Warn().Err(err).Msg("unable to snapshot engine information")
|
||||||
|
}
|
||||||
|
|
||||||
|
if dockerSnapshot.Swarm {
|
||||||
|
err = dockerSnapshotSwarmServices(dockerSnapshot, cli)
|
||||||
|
if err != nil {
|
||||||
|
log.Warn().Err(err).Msg("unable to snapshot Swarm services")
|
||||||
|
}
|
||||||
|
|
||||||
|
err = dockerSnapshotNodes(dockerSnapshot, cli)
|
||||||
|
if err != nil {
|
||||||
|
log.Warn().Err(err).Msg("unable to snapshot Swarm nodes")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
err = dockerSnapshotContainers(dockerSnapshot, cli)
|
||||||
|
if err != nil {
|
||||||
|
log.Warn().Err(err).Msg("unable to snapshot containers")
|
||||||
|
}
|
||||||
|
|
||||||
|
err = dockerSnapshotImages(dockerSnapshot, cli)
|
||||||
|
if err != nil {
|
||||||
|
log.Warn().Err(err).Msg("unable to snapshot images")
|
||||||
|
}
|
||||||
|
|
||||||
|
err = dockerSnapshotVolumes(dockerSnapshot, cli)
|
||||||
|
if err != nil {
|
||||||
|
log.Warn().Err(err).Msg("unable to snapshot volumes")
|
||||||
|
}
|
||||||
|
|
||||||
|
err = dockerSnapshotNetworks(dockerSnapshot, cli)
|
||||||
|
if err != nil {
|
||||||
|
log.Warn().Err(err).Msg("unable to snapshot networks")
|
||||||
|
}
|
||||||
|
|
||||||
|
err = dockerSnapshotVersion(dockerSnapshot, cli)
|
||||||
|
if err != nil {
|
||||||
|
log.Warn().Err(err).Msg("unable to snapshot engine version")
|
||||||
|
}
|
||||||
|
|
||||||
|
dockerSnapshot.Time = time.Now().Unix()
|
||||||
|
|
||||||
|
return dockerSnapshot, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func dockerSnapshotInfo(snapshot *portainer.DockerSnapshot, cli *client.Client) error {
|
||||||
|
info, err := cli.Info(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
snapshot.Swarm = info.Swarm.ControlAvailable
|
||||||
|
snapshot.DockerVersion = info.ServerVersion
|
||||||
|
snapshot.TotalCPU = info.NCPU
|
||||||
|
snapshot.TotalMemory = info.MemTotal
|
||||||
|
snapshot.SnapshotRaw.Info = info
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func dockerSnapshotNodes(snapshot *portainer.DockerSnapshot, cli *client.Client) error {
|
||||||
|
nodes, err := cli.NodeList(context.Background(), types.NodeListOptions{})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
var nanoCpus int64
|
||||||
|
var totalMem int64
|
||||||
|
|
||||||
|
for _, node := range nodes {
|
||||||
|
nanoCpus += node.Description.Resources.NanoCPUs
|
||||||
|
totalMem += node.Description.Resources.MemoryBytes
|
||||||
|
}
|
||||||
|
|
||||||
|
snapshot.TotalCPU = int(nanoCpus / 1e9)
|
||||||
|
snapshot.TotalMemory = totalMem
|
||||||
|
snapshot.NodeCount = len(nodes)
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func dockerSnapshotSwarmServices(snapshot *portainer.DockerSnapshot, cli *client.Client) error {
|
||||||
|
stacks := make(map[string]struct{})
|
||||||
|
|
||||||
|
services, err := cli.ServiceList(context.Background(), types.ServiceListOptions{})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, service := range services {
|
||||||
|
for k, v := range service.Spec.Labels {
|
||||||
|
if k == "com.docker.stack.namespace" {
|
||||||
|
stacks[v] = struct{}{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
snapshot.ServiceCount = len(services)
|
||||||
|
snapshot.StackCount += len(stacks)
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func dockerSnapshotContainers(snapshot *portainer.DockerSnapshot, cli *client.Client) error {
|
||||||
|
containers, err := cli.ContainerList(context.Background(), container.ListOptions{All: true})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
stacks := make(map[string]struct{})
|
||||||
|
gpuUseSet := make(map[string]struct{})
|
||||||
|
gpuUseAll := false
|
||||||
|
|
||||||
|
for _, container := range containers {
|
||||||
|
if container.State == "running" {
|
||||||
|
// Snapshot GPUs
|
||||||
|
response, err := cli.ContainerInspect(context.Background(), container.ID)
|
||||||
|
if err != nil {
|
||||||
|
// Inspect a container will fail when the container runs on a different
|
||||||
|
// Swarm node, so it is better to log the error instead of return error
|
||||||
|
// when the Swarm mode is enabled
|
||||||
|
if !snapshot.Swarm {
|
||||||
|
return err
|
||||||
|
} else {
|
||||||
|
if !strings.Contains(err.Error(), "No such container") {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// It is common to have containers running on different Swarm nodes,
|
||||||
|
// so we just log the error in the debug level
|
||||||
|
log.Debug().Str("container", container.ID).Err(err).Msg("unable to inspect container in other Swarm nodes")
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
var gpuOptions *_container.DeviceRequest = nil
|
||||||
|
for _, deviceRequest := range response.HostConfig.Resources.DeviceRequests {
|
||||||
|
if deviceRequest.Driver == "nvidia" || deviceRequest.Capabilities[0][0] == "gpu" {
|
||||||
|
gpuOptions = &deviceRequest
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if gpuOptions != nil {
|
||||||
|
if gpuOptions.Count == -1 {
|
||||||
|
gpuUseAll = true
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, id := range gpuOptions.DeviceIDs {
|
||||||
|
gpuUseSet[id] = struct{}{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for k, v := range container.Labels {
|
||||||
|
if k == consts.ComposeStackNameLabel {
|
||||||
|
stacks[v] = struct{}{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
gpuUseList := make([]string, 0, len(gpuUseSet))
|
||||||
|
for gpuUse := range gpuUseSet {
|
||||||
|
gpuUseList = append(gpuUseList, gpuUse)
|
||||||
|
}
|
||||||
|
|
||||||
|
snapshot.GpuUseAll = gpuUseAll
|
||||||
|
snapshot.GpuUseList = gpuUseList
|
||||||
|
|
||||||
|
stats := calculateContainerStats(containers)
|
||||||
|
|
||||||
|
snapshot.ContainerCount = stats.Total
|
||||||
|
snapshot.RunningContainerCount = stats.Running
|
||||||
|
snapshot.StoppedContainerCount = stats.Stopped
|
||||||
|
snapshot.HealthyContainerCount = stats.Healthy
|
||||||
|
snapshot.UnhealthyContainerCount = stats.Unhealthy
|
||||||
|
snapshot.StackCount += len(stacks)
|
||||||
|
|
||||||
|
for _, container := range containers {
|
||||||
|
snapshot.SnapshotRaw.Containers = append(snapshot.SnapshotRaw.Containers, portainer.DockerContainerSnapshot{Container: container})
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func dockerSnapshotImages(snapshot *portainer.DockerSnapshot, cli *client.Client) error {
|
||||||
|
images, err := cli.ImageList(context.Background(), image.ListOptions{})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
snapshot.ImageCount = len(images)
|
||||||
|
snapshot.SnapshotRaw.Images = images
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func dockerSnapshotVolumes(snapshot *portainer.DockerSnapshot, cli *client.Client) error {
|
||||||
|
volumes, err := cli.VolumeList(context.Background(), volume.ListOptions{})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
snapshot.VolumeCount = len(volumes.Volumes)
|
||||||
|
snapshot.SnapshotRaw.Volumes = volumes
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func dockerSnapshotNetworks(snapshot *portainer.DockerSnapshot, cli *client.Client) error {
|
||||||
|
networks, err := cli.NetworkList(context.Background(), types.NetworkListOptions{})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
snapshot.SnapshotRaw.Networks = networks
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func dockerSnapshotVersion(snapshot *portainer.DockerSnapshot, cli *client.Client) error {
|
||||||
|
version, err := cli.ServerVersion(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
snapshot.SnapshotRaw.Version = version
|
||||||
|
snapshot.IsPodman = isPodman(version)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// DockerSnapshotDiagnostics returns the diagnostics data for the agent
|
||||||
|
func DockerSnapshotDiagnostics(cli *client.Client, edgeKey string) (*portainer.DiagnosticsData, error) {
|
||||||
|
containerID := os.Getenv("HOSTNAME")
|
||||||
|
snapshot := &portainer.DockerSnapshot{
|
||||||
|
DiagnosticsData: &portainer.DiagnosticsData{
|
||||||
|
DNS: make(map[string]string),
|
||||||
|
Telnet: make(map[string]string),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
err := dockerSnapshotContainerErrorLogs(snapshot, cli, containerID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if edgeKey != "" {
|
||||||
|
url, err := edgeutils.GetPortainerURLFromEdgeKey(edgeKey)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to get portainer URL from edge key: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
snapshot.DiagnosticsData.DNS["edge-to-portainer"] = networkingutils.ProbeDNSConnection(url)
|
||||||
|
snapshot.DiagnosticsData.Telnet["edge-to-portainer"] = networkingutils.ProbeTelnetConnection(url)
|
||||||
|
}
|
||||||
|
|
||||||
|
return snapshot.DiagnosticsData, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// DockerSnapshotContainerErrorLogs returns the 5 most recent error logs of the agent container
|
||||||
|
// this will primarily be used for agent snapshot
|
||||||
|
func dockerSnapshotContainerErrorLogs(snapshot *portainer.DockerSnapshot, cli *client.Client, containerId string) error {
|
||||||
|
if containerId == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
rd, err := cli.ContainerLogs(context.Background(), containerId, container.LogsOptions{
|
||||||
|
ShowStdout: false,
|
||||||
|
ShowStderr: true,
|
||||||
|
Tail: "5",
|
||||||
|
Timestamps: true,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to get container logs: %w", err)
|
||||||
|
}
|
||||||
|
defer rd.Close()
|
||||||
|
|
||||||
|
var stdOut, stdErr bytes.Buffer
|
||||||
|
_, err = stdcopy.StdCopy(&stdErr, &stdOut, rd)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to copy error logs: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var logs []map[string]string
|
||||||
|
jsonLogs, err := json.Marshal(logs)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to marshal logs to JSON: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
snapshot.DiagnosticsData.Log = string(jsonLogs)
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// isPodman checks if the version is for Podman by checking if any of the components contain "podman".
|
||||||
|
// If it's podman, a component name should be "Podman Engine"
|
||||||
|
func isPodman(version types.Version) bool {
|
||||||
|
for _, component := range version.Components {
|
||||||
|
if strings.Contains(strings.ToLower(component.Name), "podman") {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
type ContainerStats struct {
|
||||||
|
Running int
|
||||||
|
Stopped int
|
||||||
|
Healthy int
|
||||||
|
Unhealthy int
|
||||||
|
Total int
|
||||||
|
}
|
||||||
|
|
||||||
|
func calculateContainerStats(containers []types.Container) ContainerStats {
|
||||||
|
var running, stopped, healthy, unhealthy int
|
||||||
|
for _, container := range containers {
|
||||||
|
switch container.State {
|
||||||
|
case "running":
|
||||||
|
running++
|
||||||
|
case "healthy":
|
||||||
|
running++
|
||||||
|
healthy++
|
||||||
|
case "unhealthy":
|
||||||
|
running++
|
||||||
|
unhealthy++
|
||||||
|
case "exited", "stopped":
|
||||||
|
stopped++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return ContainerStats{
|
||||||
|
Running: running,
|
||||||
|
Stopped: stopped,
|
||||||
|
Healthy: healthy,
|
||||||
|
Unhealthy: unhealthy,
|
||||||
|
Total: len(containers),
|
||||||
|
}
|
||||||
|
}
|
||||||
1
pkg/snapshot/docker_test.go
Normal file
1
pkg/snapshot/docker_test.go
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
package snapshot
|
||||||
150
pkg/snapshot/kubernetes.go
Normal file
150
pkg/snapshot/kubernetes.go
Normal file
|
|
@ -0,0 +1,150 @@
|
||||||
|
package snapshot
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/segmentio/encoding/json"
|
||||||
|
|
||||||
|
"github.com/aws/smithy-go/ptr"
|
||||||
|
portainer "github.com/portainer/portainer/api"
|
||||||
|
edgeutils "github.com/portainer/portainer/pkg/edge"
|
||||||
|
networkingutils "github.com/portainer/portainer/pkg/networking"
|
||||||
|
"github.com/rs/zerolog/log"
|
||||||
|
corev1 "k8s.io/api/core/v1"
|
||||||
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||||
|
"k8s.io/client-go/kubernetes"
|
||||||
|
)
|
||||||
|
|
||||||
|
func CreateKubernetesSnapshot(cli *kubernetes.Clientset) (*portainer.KubernetesSnapshot, error) {
|
||||||
|
kubernetesSnapshot := &portainer.KubernetesSnapshot{}
|
||||||
|
|
||||||
|
err := kubernetesSnapshotVersion(kubernetesSnapshot, cli)
|
||||||
|
if err != nil {
|
||||||
|
log.Warn().Err(err).Msg("unable to snapshot cluster version")
|
||||||
|
}
|
||||||
|
|
||||||
|
err = kubernetesSnapshotNodes(kubernetesSnapshot, cli)
|
||||||
|
if err != nil {
|
||||||
|
log.Warn().Err(err).Msg("unable to snapshot cluster nodes")
|
||||||
|
}
|
||||||
|
|
||||||
|
kubernetesSnapshot.Time = time.Now().Unix()
|
||||||
|
return kubernetesSnapshot, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func kubernetesSnapshotVersion(snapshot *portainer.KubernetesSnapshot, cli *kubernetes.Clientset) error {
|
||||||
|
versionInfo, err := cli.ServerVersion()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
snapshot.KubernetesVersion = versionInfo.GitVersion
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func kubernetesSnapshotNodes(snapshot *portainer.KubernetesSnapshot, cli *kubernetes.Clientset) error {
|
||||||
|
nodeList, err := cli.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
var totalCPUs, totalMemory int64
|
||||||
|
for _, node := range nodeList.Items {
|
||||||
|
totalCPUs += node.Status.Capacity.Cpu().Value()
|
||||||
|
totalMemory += node.Status.Capacity.Memory().Value()
|
||||||
|
}
|
||||||
|
|
||||||
|
snapshot.TotalCPU = totalCPUs
|
||||||
|
snapshot.TotalMemory = totalMemory
|
||||||
|
snapshot.NodeCount = len(nodeList.Items)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// KubernetesSnapshotDiagnostics returns the diagnostics data for the agent
|
||||||
|
func KubernetesSnapshotDiagnostics(cli *kubernetes.Clientset, edgeKey string) (*portainer.DiagnosticsData, error) {
|
||||||
|
podID := os.Getenv("HOSTNAME")
|
||||||
|
snapshot := &portainer.KubernetesSnapshot{
|
||||||
|
DiagnosticsData: &portainer.DiagnosticsData{
|
||||||
|
DNS: make(map[string]string),
|
||||||
|
Telnet: make(map[string]string),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
err := kubernetesSnapshotPodErrorLogs(snapshot, cli, "portainer", podID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to snapshot pod error logs: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if edgeKey != "" {
|
||||||
|
url, err := edgeutils.GetPortainerURLFromEdgeKey(edgeKey)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to get portainer URL from edge key: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
snapshot.DiagnosticsData.DNS["edge-to-portainer"] = networkingutils.ProbeDNSConnection(url)
|
||||||
|
snapshot.DiagnosticsData.Telnet["edge-to-portainer"] = networkingutils.ProbeTelnetConnection(url)
|
||||||
|
}
|
||||||
|
|
||||||
|
return snapshot.DiagnosticsData, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// KubernetesSnapshotPodErrorLogs returns 0 to 10 lines of the most recent error logs of the agent container
|
||||||
|
// this will primarily be used for agent snapshot
|
||||||
|
func kubernetesSnapshotPodErrorLogs(snapshot *portainer.KubernetesSnapshot, cli *kubernetes.Clientset, namespace, podID string) error {
|
||||||
|
if namespace == "" || podID == "" {
|
||||||
|
return errors.New("both namespace and podID are required to capture pod error logs in the snapshot")
|
||||||
|
}
|
||||||
|
|
||||||
|
logsStream, err := cli.CoreV1().Pods(namespace).GetLogs(podID, &corev1.PodLogOptions{TailLines: ptr.Int64(10), Timestamps: true}).Stream(context.TODO())
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to stream logs: %w", err)
|
||||||
|
}
|
||||||
|
defer logsStream.Close()
|
||||||
|
|
||||||
|
logBytes, err := io.ReadAll(logsStream)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to read error logs: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
logs := filterLogsByPattern(logBytes, []string{"error", "err", "level=error", "exception", "fatal", "panic"})
|
||||||
|
|
||||||
|
jsonLogs, err := json.Marshal(logs)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to marshal logs: %w", err)
|
||||||
|
}
|
||||||
|
snapshot.DiagnosticsData.Log = string(jsonLogs)
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// filterLogsByPattern filters the logs by the given patterns and returns a list of logs that match the patterns
|
||||||
|
// the logs are returned as a list of maps with the keys "timestamp" and "message"
|
||||||
|
func filterLogsByPattern(logBytes []byte, patterns []string) []map[string]string {
|
||||||
|
logs := []map[string]string{}
|
||||||
|
for _, line := range strings.Split(strings.TrimSpace(string(logBytes)), "\n") {
|
||||||
|
if line == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if parts := strings.SplitN(line, " ", 2); len(parts) == 2 {
|
||||||
|
messageLower := strings.ToLower(parts[1])
|
||||||
|
for _, pattern := range patterns {
|
||||||
|
if strings.Contains(messageLower, pattern) {
|
||||||
|
logs = append(logs, map[string]string{
|
||||||
|
"timestamp": parts[0],
|
||||||
|
"message": parts[1],
|
||||||
|
})
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return logs
|
||||||
|
}
|
||||||
1
pkg/snapshot/kubernetes_test.go
Normal file
1
pkg/snapshot/kubernetes_test.go
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
package snapshot
|
||||||
Loading…
Add table
Add a link
Reference in a new issue