feat(edge/stacks): increase status transparency [EE-5554] (#9094)

pkg/libstack/compose/internal/composeplugin/composeplugin.go

@@ -3,6 +3,7 @@ package composeplugin
 import (
 	"bytes"
 	"context"
+	"fmt"
 	"os"
 	"os/exec"
 	"strings"
@@ -160,7 +161,11 @@ func (wrapper *PluginWrapper) command(command composeCommand, options libstack.O
 			Err(err).
 			Msg("docker compose command failed")
 
-		return nil, errors.New(errOutput)
+		if errOutput != "" {
+			return nil, errors.New(errOutput)
+		}
+
+		return nil, fmt.Errorf("docker compose command failed: %w", err)
 	}
 
 	return output, nil

pkg/libstack/compose/internal/composeplugin/composeplugin_test.go

@@ -2,7 +2,6 @@ package composeplugin
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"log"
 	"os"
@@ -16,9 +15,9 @@ import (
 )
 
 func checkPrerequisites(t *testing.T) {
-	if _, err := os.Stat("docker-compose"); errors.Is(err, os.ErrNotExist) {
-		t.Fatal("docker-compose binary not found, please run download.sh and re-run this test suite")
-	}
+	// if _, err := os.Stat("docker-compose"); errors.Is(err, os.ErrNotExist) {
+	// 	t.Fatal("docker-compose binary not found, please run download.sh and re-run this test suite")
+	// }
 }
 
 func setup(t *testing.T) libstack.Deployer {
@@ -118,7 +117,11 @@ func createFile(dir, fileName, content string) (string, error) {
 		return "", err
 	}
 
-	f.WriteString(content)
+	_, err = f.WriteString(content)
+	if err != nil {
+		return "", err
+	}
+
 	f.Close()
 
 	return filePath, nil

pkg/libstack/compose/internal/composeplugin/status.go

@@ -0,0 +1,171 @@
+package composeplugin
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"github.com/portainer/portainer/pkg/libstack"
+	"github.com/rs/zerolog/log"
+)
+
+type publisher struct {
+	URL           string
+	TargetPort    int
+	PublishedPort int
+	Protocol      string
+}
+
+type service struct {
+	ID         string
+	Name       string
+	Image      string
+	Command    string
+	Project    string
+	Service    string
+	Created    int64
+	State      string
+	Status     string
+	Health     string
+	ExitCode   int
+	Publishers []publisher
+}
+
+// docker container state can be one of "created", "running", "paused", "restarting", "removing", "exited", or "dead"
+func getServiceStatus(service service) (libstack.Status, string) {
+	log.Debug().
+		Str("service", service.Name).
+		Str("state", service.State).
+		Int("exitCode", service.ExitCode).
+		Msg("getServiceStatus")
+
+	switch service.State {
+	case "created", "restarting", "paused":
+		return libstack.StatusStarting, ""
+	case "running":
+		return libstack.StatusRunning, ""
+	case "removing":
+		return libstack.StatusRemoving, ""
+	case "exited", "dead":
+		if service.ExitCode != 0 {
+			return libstack.StatusError, fmt.Sprintf("service %s exited with code %d", service.Name, service.ExitCode)
+		}
+
+		return libstack.StatusRemoved, ""
+	default:
+		return libstack.StatusUnknown, ""
+	}
+}
+
+func aggregateStatuses(services []service) (libstack.Status, string) {
+	servicesCount := len(services)
+
+	if servicesCount == 0 {
+		log.Debug().
+			Msg("no services found")
+
+		return libstack.StatusRemoved, ""
+	}
+
+	statusCounts := make(map[libstack.Status]int)
+	errorMessage := ""
+	for _, service := range services {
+		status, serviceError := getServiceStatus(service)
+		if serviceError != "" {
+			errorMessage = serviceError
+		}
+		statusCounts[status]++
+	}
+
+	log.Debug().
+		Interface("statusCounts", statusCounts).
+		Str("errorMessage", errorMessage).
+		Msg("check_status")
+
+	switch {
+	case errorMessage != "":
+		return libstack.StatusError, errorMessage
+	case statusCounts[libstack.StatusStarting] > 0:
+		return libstack.StatusStarting, ""
+	case statusCounts[libstack.StatusRemoving] > 0:
+		return libstack.StatusRemoving, ""
+	case statusCounts[libstack.StatusRunning] == servicesCount:
+		return libstack.StatusRunning, ""
+	case statusCounts[libstack.StatusStopped] == servicesCount:
+		return libstack.StatusStopped, ""
+	case statusCounts[libstack.StatusRemoved] == servicesCount:
+		return libstack.StatusRemoved, ""
+	default:
+		return libstack.StatusUnknown, ""
+	}
+
+}
+
+func (wrapper *PluginWrapper) WaitForStatus(ctx context.Context, name string, status libstack.Status) <-chan string {
+	errorMessageCh := make(chan string)
+
+	go func() {
+		for {
+			select {
+			case <-ctx.Done():
+				errorMessageCh <- fmt.Sprintf("failed to wait for status: %s", ctx.Err().Error())
+			default:
+			}
+
+			time.Sleep(1 * time.Second)
+
+			output, err := wrapper.command(newCommand([]string{"ps", "-a", "--format", "json"}, nil), libstack.Options{
+				ProjectName: name,
+			})
+			if len(output) == 0 {
+				log.Debug().
+					Str("project_name", name).
+					Msg("no output from docker compose ps")
+				continue
+			}
+
+			if err != nil {
+				log.Debug().
+					Str("project_name", name).
+					Err(err).
+					Msg("error from docker compose ps")
+				continue
+			}
+
+			var services []service
+			err = json.Unmarshal(output, &services)
+			if err != nil {
+				log.Debug().
+					Str("project_name", name).
+					Err(err).
+					Msg("failed to parse docker compose output")
+				continue
+			}
+
+			if len(services) == 0 && status == libstack.StatusRemoved {
+				errorMessageCh <- ""
+				return
+			}
+
+			aggregateStatus, errorMessage := aggregateStatuses(services)
+			if aggregateStatus == status {
+				errorMessageCh <- ""
+				return
+			}
+
+			if errorMessage != "" {
+				errorMessageCh <- errorMessage
+				return
+			}
+
+			log.Debug().
+				Str("project_name", name).
+				Str("status", string(aggregateStatus)).
+				Msg("waiting for status")
+
+		}
+	}()
+
+	return errorMessageCh
+}

pkg/libstack/compose/internal/composeplugin/status_integration_test.go

@@ -0,0 +1,116 @@
+package composeplugin
+
+import (
+	"context"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/portainer/portainer/pkg/libstack"
+)
+
+/*
+
+1. starting = docker compose file that runs several services, one of them should be with status starting
+2. running = docker compose file that runs successfully and returns status running
+3. removing = run docker compose config, remove the stack, and return removing status
+4. failed = run a valid docker compose file, but one of the services should fail to start (so "docker compose up" should run successfully, but one of the services should do something like `CMD ["exit", "1"]
+5. removed = remove a compose stack and return status removed
+
+*/
+
+func ensureIntegrationTest(t *testing.T) {
+	if _, ok := os.LookupEnv("INTEGRATION_TEST"); !ok {
+		t.Skip("skip an integration test")
+	}
+}
+
+func TestComposeProjectStatus(t *testing.T) {
+	ensureIntegrationTest(t)
+
+	testCases := []struct {
+		TestName              string
+		ComposeFile           string
+		ExpectedStatus        libstack.Status
+		ExpectedStatusMessage bool
+	}{
+
+		{
+			TestName:       "running",
+			ComposeFile:    "status_test_files/running.yml",
+			ExpectedStatus: libstack.StatusRunning,
+		},
+
+		{
+			TestName:              "failed",
+			ComposeFile:           "status_test_files/failed.yml",
+			ExpectedStatus:        libstack.StatusError,
+			ExpectedStatusMessage: true,
+		},
+	}
+
+	w := setup(t)
+	ctx := context.Background()
+
+	for _, testCase := range testCases {
+		t.Run(testCase.TestName, func(t *testing.T) {
+			projectName := testCase.TestName
+			err := w.Deploy(ctx, []string{testCase.ComposeFile}, libstack.DeployOptions{
+				Options: libstack.Options{
+					ProjectName: projectName,
+				},
+			})
+			if err != nil {
+				t.Fatalf("[test: %s] Failed to deploy compose file: %v", testCase.TestName, err)
+			}
+
+			time.Sleep(5 * time.Second)
+
+			status, statusMessage, err := waitForStatus(w, ctx, projectName, libstack.StatusRunning)
+			if err != nil {
+				t.Fatalf("[test: %s] Failed to get compose project status: %v", testCase.TestName, err)
+			}
+
+			if status != testCase.ExpectedStatus {
+				t.Fatalf("[test: %s] Expected status: %s, got: %s", testCase.TestName, testCase.ExpectedStatus, status)
+			}
+
+			if testCase.ExpectedStatusMessage && statusMessage == "" {
+				t.Fatalf("[test: %s] Expected status message but got empty", testCase.TestName)
+			}
+
+			err = w.Remove(ctx, projectName, nil, libstack.Options{})
+			if err != nil {
+				t.Fatalf("[test: %s] Failed to remove compose project: %v", testCase.TestName, err)
+			}
+
+			time.Sleep(20 * time.Second)
+
+			status, statusMessage, err = waitForStatus(w, ctx, projectName, libstack.StatusRemoved)
+			if err != nil {
+				t.Fatalf("[test: %s] Failed to get compose project status: %v", testCase.TestName, err)
+			}
+
+			if status != libstack.StatusRemoved {
+				t.Fatalf("[test: %s] Expected stack to be removed, got %s", testCase.TestName, status)
+			}
+
+			if statusMessage != "" {
+				t.Fatalf("[test: %s] Expected empty status message: %s, got: %s", "", testCase.TestName, statusMessage)
+			}
+		})
+	}
+}
+
+func waitForStatus(deployer libstack.Deployer, ctx context.Context, stackName string, requiredStatus libstack.Status) (libstack.Status, string, error) {
+	ctx, cancel := context.WithTimeout(ctx, 1*time.Minute)
+	defer cancel()
+
+	statusCh := deployer.WaitForStatus(ctx, stackName, requiredStatus)
+	result := <-statusCh
+	if result == "" {
+		return requiredStatus, "", nil
+	}
+
+	return libstack.StatusError, result, nil
+}

pkg/libstack/compose/internal/composeplugin/status_test_files/failed.yml

@@ -0,0 +1,7 @@
+version: '3'
+services:
+  web:
+    image: nginx:latest
+  failing-service:
+    image: busybox
+    command: ["false"]

pkg/libstack/compose/internal/composeplugin/status_test_files/running.yml

@@ -0,0 +1,4 @@
+version: '3'
+services:
+  web:
+    image: nginx:latest