1
0
Fork 0
mirror of https://github.com/portainer/portainer.git synced 2025-07-20 05:49:40 +02:00

fix(libstack): correctly load COMPOSE_* env vars [BE-11474] (#536)

This commit is contained in:
Devon Steenberg 2025-03-25 08:57:23 +13:00 committed by GitHub
parent 5d1cd670e9
commit 34235199dd
7 changed files with 952 additions and 87 deletions

View file

@ -2,14 +2,23 @@ package compose
import (
"context"
"errors"
"log"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"testing"
"github.com/compose-spec/compose-go/v2/consts"
"github.com/compose-spec/compose-go/v2/types"
"github.com/docker/cli/cli/command"
cmdcompose "github.com/docker/compose/v2/cmd/compose"
"github.com/docker/compose/v2/pkg/api"
"github.com/google/go-cmp/cmp"
"github.com/portainer/portainer/pkg/libstack"
zerolog "github.com/rs/zerolog/log"
"github.com/stretchr/testify/require"
)
@ -349,3 +358,735 @@ networks:
})
}
}
func Test_DeployWithRemoveOrphans(t *testing.T) {
const projectName = "compose_remove_orphans_test"
const composeFileContent = `services:
service-1:
image: alpine:latest
service-2:
image: alpine:latest`
const modifiedFileContent = `services:
service-2:
image: alpine:latest`
service1ContainerName := projectName + "-service-1"
service2ContainerName := projectName + "-service-2"
w := NewComposeDeployer()
dir := t.TempDir()
composeFilepath := createFile(t, dir, "docker-compose.yml", composeFileContent)
modifiedComposeFilepath := createFile(t, dir, "docker-compose-modified.yml", modifiedFileContent)
filepaths := []string{composeFilepath}
modifiedFilepaths := []string{modifiedComposeFilepath}
ctx := context.Background()
testCases := []struct {
name string
options libstack.DeployOptions
}{
{
name: "Remove Orphans in env",
options: libstack.DeployOptions{
Options: libstack.Options{
ProjectName: projectName,
Env: []string{cmdcompose.ComposeRemoveOrphans + "=true"},
},
},
},
{
name: "Remove Orphans in options",
options: libstack.DeployOptions{
Options: libstack.Options{
ProjectName: projectName,
},
RemoveOrphans: true,
},
},
{
name: "Remove Orphans in options and env",
options: libstack.DeployOptions{
Options: libstack.Options{
ProjectName: projectName,
Env: []string{cmdcompose.ComposeRemoveOrphans + "=true"},
},
RemoveOrphans: false,
},
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
options := tc.options.Options
err := w.Validate(ctx, filepaths, options)
require.NoError(t, err)
err = w.Pull(ctx, filepaths, options)
require.NoError(t, err)
require.False(t, containerExists(service1ContainerName))
require.False(t, containerExists(service2ContainerName))
err = w.Deploy(ctx, filepaths, tc.options)
require.NoError(t, err)
defer func() {
err = w.Remove(ctx, projectName, filepaths, libstack.RemoveOptions{})
require.NoError(t, err)
require.False(t, containerExists(service1ContainerName))
require.False(t, containerExists(service2ContainerName))
}()
require.True(t, containerExists(service1ContainerName))
require.True(t, containerExists(service2ContainerName))
waitResult := w.WaitForStatus(ctx, projectName, libstack.StatusCompleted)
require.Empty(t, waitResult.ErrorMsg)
require.Equal(t, libstack.StatusCompleted, waitResult.Status)
err = w.Validate(ctx, modifiedFilepaths, options)
require.NoError(t, err)
err = w.Pull(ctx, modifiedFilepaths, options)
require.NoError(t, err)
require.True(t, containerExists(service1ContainerName))
require.True(t, containerExists(service2ContainerName))
err = w.Deploy(ctx, modifiedFilepaths, tc.options)
require.NoError(t, err)
require.False(t, containerExists(service1ContainerName))
require.True(t, containerExists(service2ContainerName))
waitResult = w.WaitForStatus(ctx, projectName, libstack.StatusCompleted)
require.Empty(t, waitResult.ErrorMsg)
require.Equal(t, libstack.StatusCompleted, waitResult.Status)
})
}
}
func Test_DeployWithIgnoreOrphans(t *testing.T) {
var logOutput strings.Builder
oldLogger := zerolog.Logger
zerolog.Logger = zerolog.Output(&logOutput)
defer func() {
zerolog.Logger = oldLogger
}()
const projectName = "compose_ignore_orphans_test"
const composeFileContent = `services:
service-1:
image: alpine:latest
service-2:
image: alpine:latest`
const modifiedFileContent = `services:
service-2:
image: alpine:latest`
service1ContainerName := projectName + "-service-1"
service2ContainerName := projectName + "-service-2"
w := NewComposeDeployer()
dir := t.TempDir()
composeFilepath := createFile(t, dir, "docker-compose.yml", composeFileContent)
modifiedComposeFilepath := createFile(t, dir, "docker-compose-modified.yml", modifiedFileContent)
filepaths := []string{composeFilepath}
modifiedFilepaths := []string{modifiedComposeFilepath}
options := libstack.Options{
ProjectName: projectName,
Env: []string{cmdcompose.ComposeIgnoreOrphans + "=true"},
}
ctx := context.Background()
err := w.Validate(ctx, filepaths, options)
require.NoError(t, err)
err = w.Pull(ctx, filepaths, options)
require.NoError(t, err)
require.False(t, containerExists(service1ContainerName))
require.False(t, containerExists(service2ContainerName))
err = w.Deploy(ctx, filepaths, libstack.DeployOptions{Options: options})
require.NoError(t, err)
defer func() {
err = w.Remove(ctx, projectName, filepaths, libstack.RemoveOptions{})
require.NoError(t, err)
require.False(t, containerExists(service1ContainerName))
require.False(t, containerExists(service2ContainerName))
}()
require.True(t, containerExists(service1ContainerName))
require.True(t, containerExists(service2ContainerName))
waitResult := w.WaitForStatus(ctx, projectName, libstack.StatusCompleted)
require.Empty(t, waitResult.ErrorMsg)
require.Equal(t, libstack.StatusCompleted, waitResult.Status)
err = w.Validate(ctx, modifiedFilepaths, options)
require.NoError(t, err)
err = w.Pull(ctx, modifiedFilepaths, options)
require.NoError(t, err)
require.True(t, containerExists(service1ContainerName))
require.True(t, containerExists(service2ContainerName))
err = w.Deploy(ctx, modifiedFilepaths, libstack.DeployOptions{Options: options})
require.NoError(t, err)
require.True(t, containerExists(service1ContainerName))
require.True(t, containerExists(service2ContainerName))
waitResult = w.WaitForStatus(ctx, projectName, libstack.StatusCompleted)
require.Empty(t, waitResult.ErrorMsg)
require.Equal(t, libstack.StatusCompleted, waitResult.Status)
logString := logOutput.String()
require.False(t, strings.Contains(logString, "Found orphan containers ([compose_ignore_orphans_test-service-1-1])"))
}
func Test_MaxConcurrency(t *testing.T) {
const projectName = "compose_max_concurrency_test"
const composeFileContent = `services:
service-1:
image: alpine:latest`
w := ComposeDeployer{
createComposeServiceFn: createMockComposeService,
}
dir := t.TempDir()
composeFilepath := createFile(t, dir, "docker-compose.yml", composeFileContent)
expectedMaxConcurrency := 4
filepaths := []string{composeFilepath}
options := libstack.Options{
ProjectName: projectName,
Env: []string{cmdcompose.ComposeParallelLimit + "=" + strconv.Itoa(expectedMaxConcurrency)},
}
ctx := context.Background()
err := w.Validate(ctx, filepaths, options)
require.NoError(t, err)
w.withComposeService(ctx, filepaths, options, func(service api.Service, _ *types.Project) error {
if mockS, ok := service.(*mockComposeService); ok {
require.Equal(t, mockS.maxConcurrency, expectedMaxConcurrency)
} else {
t.Fatalf("Expected mockComposeService but got %T", service)
}
return nil
})
}
func Test_createProject(t *testing.T) {
ctx := context.Background()
dir := t.TempDir()
projectName := "create-project-test"
defer func() {
err := os.RemoveAll(dir)
if err != nil {
t.Fatalf("Failed to remove temp dir: %v", err)
}
}()
createTestLibstackOptions := func(workingDir, projectName string, env []string, envFilepath string) libstack.Options {
return libstack.Options{
WorkingDir: workingDir,
ProjectName: projectName,
Env: env,
EnvFilePath: envFilepath,
}
}
testSimpleComposeConfig := `services:
nginx:
container_name: nginx
image: nginx:latest`
expectedSimpleComposeProject := func(workingDirectory string, envOverrides map[string]string) *types.Project {
env := types.Mapping{consts.ComposeProjectName: "create-project-test"}
env = env.Merge(envOverrides)
if workingDirectory == "" {
workingDirectory = dir
}
if !filepath.IsAbs(workingDirectory) {
absWorkingDir, err := filepath.Abs(workingDirectory)
if err != nil {
t.Fatalf("Failed to get absolute path of working directory (%s): %v", workingDirectory, err)
}
workingDirectory = absWorkingDir
}
return &types.Project{
Name: projectName,
WorkingDir: workingDirectory,
Services: types.Services{
"nginx": {
Name: "nginx",
ContainerName: "nginx",
Environment: types.MappingWithEquals{},
Image: "nginx:latest",
Networks: map[string]*types.ServiceNetworkConfig{"default": nil},
},
},
Networks: types.Networks{"default": {Name: "create-project-test_default"}},
ComposeFiles: []string{
dir + "/docker-compose.yml",
},
Environment: env,
DisabledServices: types.Services{},
Profiles: []string{""},
}
}
testComposeProfilesConfig := `services:
nginx:
container_name: nginx
image: nginx:latest
profiles: ['web1']
apache:
container_name: apache
image: httpd:latest
profiles: ['web2']`
expectedComposeProfilesProject := func(envOverrides map[string]string) *types.Project {
env := types.Mapping{consts.ComposeProfiles: "web1", consts.ComposeProjectName: "create-project-test"}
env = env.Merge(envOverrides)
return &types.Project{
Name: projectName,
WorkingDir: dir,
Services: types.Services{
"nginx": {
Name: "nginx",
Profiles: []string{"web1"},
ContainerName: "nginx",
Environment: types.MappingWithEquals{},
Image: "nginx:latest",
Networks: map[string]*types.ServiceNetworkConfig{"default": nil},
},
},
Networks: types.Networks{"default": {Name: "create-project-test_default"}},
ComposeFiles: []string{
dir + "/docker-compose.yml",
},
Environment: env,
DisabledServices: types.Services{
"apache": {
Name: "apache",
Profiles: []string{"web2"},
ContainerName: "apache",
Environment: nil,
Image: "httpd:latest",
Networks: map[string]*types.ServiceNetworkConfig{"default": nil},
},
},
Profiles: []string{"web1"},
}
}
testcases := []struct {
name string
filesToCreate map[string]string
configFilepaths []string
options libstack.Options
expectedProject *types.Project
}{
{
name: "Compose profiles in env",
filesToCreate: map[string]string{
"docker-compose.yml": testComposeProfilesConfig,
},
configFilepaths: []string{dir + "/docker-compose.yml"},
options: createTestLibstackOptions(dir, projectName, []string{consts.ComposeProfiles + "=web1"}, ""),
expectedProject: expectedComposeProfilesProject(nil),
},
{
name: "Compose profiles in env file",
filesToCreate: map[string]string{
"docker-compose.yml": testComposeProfilesConfig,
"stack.env": consts.ComposeProfiles + "=web1",
},
configFilepaths: []string{dir + "/docker-compose.yml"},
options: createTestLibstackOptions(dir, projectName, nil, dir+"/stack.env"),
expectedProject: expectedComposeProfilesProject(nil),
},
{
name: "Compose profiles in env file in COMPOSE_ENV_FILES",
filesToCreate: map[string]string{
"docker-compose.yml": testComposeProfilesConfig,
"stack.env": consts.ComposeProfiles + "=web1",
},
configFilepaths: []string{dir + "/docker-compose.yml"},
options: createTestLibstackOptions(dir, projectName, []string{cmdcompose.ComposeEnvFiles + "=" + dir + "/stack.env"}, ""),
expectedProject: expectedComposeProfilesProject(map[string]string{
cmdcompose.ComposeEnvFiles: dir + "/stack.env",
}),
},
{
name: "Compose profiles in both env and env file",
filesToCreate: map[string]string{
"docker-compose.yml": testComposeProfilesConfig,
"stack.env": consts.ComposeProfiles + "=web2",
},
configFilepaths: []string{dir + "/docker-compose.yml"},
options: createTestLibstackOptions(dir, projectName, []string{consts.ComposeProfiles + "=web1"}, dir+"/stack.env"),
expectedProject: expectedComposeProfilesProject(nil),
},
{
name: "Compose project name in both options and env",
filesToCreate: map[string]string{
"docker-compose.yml": testSimpleComposeConfig,
},
configFilepaths: []string{dir + "/docker-compose.yml"},
options: createTestLibstackOptions(dir, projectName, []string{consts.ComposeProjectName + "=totally_different_name"}, ""),
expectedProject: expectedSimpleComposeProject("", nil),
},
{
name: "Compose project name in only env",
filesToCreate: map[string]string{
"docker-compose.yml": testSimpleComposeConfig,
},
configFilepaths: []string{dir + "/docker-compose.yml"},
options: createTestLibstackOptions(dir, "", []string{consts.ComposeProjectName + "=totally_different_name"}, ""),
expectedProject: &types.Project{
Name: "totally_different_name",
WorkingDir: dir,
Services: types.Services{
"nginx": {
Name: "nginx",
ContainerName: "nginx",
Environment: types.MappingWithEquals{},
Image: "nginx:latest",
Networks: map[string]*types.ServiceNetworkConfig{"default": nil},
},
},
Networks: types.Networks{"default": {Name: "totally_different_name_default"}},
ComposeFiles: []string{
dir + "/docker-compose.yml",
},
Environment: types.Mapping{consts.ComposeProjectName: "totally_different_name"},
DisabledServices: types.Services{},
Profiles: []string{""},
},
},
{
name: "Compose files in env",
filesToCreate: map[string]string{
"docker-compose.yml": testSimpleComposeConfig,
},
configFilepaths: nil,
options: createTestLibstackOptions(dir, projectName, []string{consts.ComposeFilePath + "=" + dir + "/docker-compose.yml"}, ""),
expectedProject: expectedSimpleComposeProject("", map[string]string{
consts.ComposeFilePath: dir + "/docker-compose.yml",
}),
},
{
name: "Compose files in both options and env",
filesToCreate: map[string]string{
"docker-compose.yml": testSimpleComposeConfig,
"profiles-docker-compose.yml": testComposeProfilesConfig,
},
configFilepaths: []string{dir + "/docker-compose.yml"},
options: createTestLibstackOptions(dir, projectName, []string{consts.ComposeFilePath + "=" + dir + "/profiles-docker-compose.yml"}, ""),
expectedProject: expectedSimpleComposeProject("", map[string]string{
consts.ComposeFilePath: dir + "/profiles-docker-compose.yml",
}),
},
{
name: "Multiple Compose files in options",
filesToCreate: map[string]string{
"docker-compose-0.yml": `services:
nginx:
container_name: nginx
image: nginx:latest`,
"docker-compose-1.yml": `services:
apache:
container_name: apache
image: httpd:latest`,
},
configFilepaths: []string{dir + "/docker-compose-0.yml", dir + "/docker-compose-1.yml"},
options: createTestLibstackOptions(dir, projectName, []string{}, ""),
expectedProject: &types.Project{
Name: projectName,
WorkingDir: dir,
Services: types.Services{
"nginx": {
Name: "nginx",
ContainerName: "nginx",
Environment: types.MappingWithEquals{},
Image: "nginx:latest",
Networks: map[string]*types.ServiceNetworkConfig{"default": nil},
},
"apache": {
Name: "apache",
ContainerName: "apache",
Environment: types.MappingWithEquals{},
Image: "httpd:latest",
Networks: map[string]*types.ServiceNetworkConfig{"default": nil},
},
},
Networks: types.Networks{"default": {Name: "create-project-test_default"}},
ComposeFiles: []string{
dir + "/docker-compose-0.yml",
dir + "/docker-compose-1.yml",
},
Environment: types.Mapping{consts.ComposeProjectName: "create-project-test"},
DisabledServices: types.Services{},
Profiles: []string{""},
},
},
{
name: "Multiple Compose files in env",
filesToCreate: map[string]string{
"docker-compose-0.yml": `services:
nginx:
container_name: nginx
image: nginx:latest`,
"docker-compose-1.yml": `services:
apache:
container_name: apache
image: httpd:latest`,
},
configFilepaths: nil,
options: createTestLibstackOptions(dir, projectName, []string{consts.ComposeFilePath + "=" + dir + "/docker-compose-0.yml:" + dir + "/docker-compose-1.yml"}, ""),
expectedProject: &types.Project{
Name: projectName,
WorkingDir: dir,
Services: types.Services{
"nginx": {
Name: "nginx",
ContainerName: "nginx",
Environment: types.MappingWithEquals{},
Image: "nginx:latest",
Networks: map[string]*types.ServiceNetworkConfig{"default": nil},
},
"apache": {
Name: "apache",
ContainerName: "apache",
Environment: types.MappingWithEquals{},
Image: "httpd:latest",
Networks: map[string]*types.ServiceNetworkConfig{"default": nil},
},
},
Networks: types.Networks{"default": {Name: "create-project-test_default"}},
ComposeFiles: []string{
dir + "/docker-compose-0.yml",
dir + "/docker-compose-1.yml",
},
Environment: types.Mapping{consts.ComposeProjectName: "create-project-test", consts.ComposeFilePath: dir + "/docker-compose-0.yml:" + dir + "/docker-compose-1.yml"},
DisabledServices: types.Services{},
Profiles: []string{""},
},
},
{
name: "Multiple Compose files in env with COMPOSE_PATH_SEPARATOR",
filesToCreate: map[string]string{
"docker-compose-0.yml": `services:
nginx:
container_name: nginx
image: nginx:latest`,
"docker-compose-1.yml": `services:
apache:
container_name: apache
image: httpd:latest`,
},
configFilepaths: nil,
options: createTestLibstackOptions(dir, projectName, []string{consts.ComposePathSeparator + "=|", consts.ComposeFilePath + "=" + dir + "/docker-compose-0.yml|" + dir + "/docker-compose-1.yml"}, ""),
expectedProject: &types.Project{
Name: projectName,
WorkingDir: dir,
Services: types.Services{
"nginx": {
Name: "nginx",
ContainerName: "nginx",
Environment: types.MappingWithEquals{},
Image: "nginx:latest",
Networks: map[string]*types.ServiceNetworkConfig{"default": nil},
},
"apache": {
Name: "apache",
ContainerName: "apache",
Environment: types.MappingWithEquals{},
Image: "httpd:latest",
Networks: map[string]*types.ServiceNetworkConfig{"default": nil},
},
},
Networks: types.Networks{"default": {Name: "create-project-test_default"}},
ComposeFiles: []string{
dir + "/docker-compose-0.yml",
dir + "/docker-compose-1.yml",
},
Environment: types.Mapping{consts.ComposeProjectName: "create-project-test", consts.ComposePathSeparator: "|", consts.ComposeFilePath: dir + "/docker-compose-0.yml|" + dir + "/docker-compose-1.yml"},
DisabledServices: types.Services{},
Profiles: []string{""},
},
},
{
name: "compose ignore orphans",
filesToCreate: map[string]string{
"docker-compose.yml": testSimpleComposeConfig,
},
configFilepaths: []string{dir + "/docker-compose.yml"},
options: createTestLibstackOptions(dir, projectName, []string{cmdcompose.ComposeIgnoreOrphans + "=true"}, ""),
expectedProject: expectedSimpleComposeProject("", map[string]string{
cmdcompose.ComposeIgnoreOrphans: "true",
}),
},
{
name: "compose remove orphans",
filesToCreate: map[string]string{
"docker-compose.yml": testSimpleComposeConfig,
},
configFilepaths: []string{dir + "/docker-compose.yml"},
options: createTestLibstackOptions(dir, projectName, []string{cmdcompose.ComposeRemoveOrphans + "=true"}, ""),
expectedProject: expectedSimpleComposeProject("", map[string]string{
cmdcompose.ComposeRemoveOrphans: "true",
}),
},
{
name: "compose parallel limit",
filesToCreate: map[string]string{
"docker-compose.yml": testSimpleComposeConfig,
},
configFilepaths: []string{dir + "/docker-compose.yml"},
options: createTestLibstackOptions(dir, projectName, []string{cmdcompose.ComposeParallelLimit + "=true"}, ""),
expectedProject: expectedSimpleComposeProject("", map[string]string{
cmdcompose.ComposeParallelLimit: "true",
}),
},
{
name: "Absolute Working Directory",
filesToCreate: map[string]string{
"docker-compose.yml": testSimpleComposeConfig,
},
configFilepaths: []string{dir + "/docker-compose.yml"},
options: libstack.Options{
WorkingDir: "/something-totally-different",
ProjectName: projectName,
},
expectedProject: expectedSimpleComposeProject("/something-totally-different", nil),
},
{
name: "Relative Working Directory",
filesToCreate: map[string]string{
"docker-compose.yml": testSimpleComposeConfig,
},
configFilepaths: []string{dir + "/docker-compose.yml"},
options: libstack.Options{
WorkingDir: "something-totally-different",
ProjectName: projectName,
},
expectedProject: expectedSimpleComposeProject("something-totally-different", nil),
},
{
name: "Absolute Project Directory",
filesToCreate: map[string]string{
"docker-compose.yml": testSimpleComposeConfig,
},
configFilepaths: []string{dir + "/docker-compose.yml"},
options: libstack.Options{
ProjectDir: "/something-totally-different",
ProjectName: projectName,
},
expectedProject: expectedSimpleComposeProject("/something-totally-different", nil),
},
{
name: "Relative Project Directory",
filesToCreate: map[string]string{
"docker-compose.yml": testSimpleComposeConfig,
},
configFilepaths: []string{dir + "/docker-compose.yml"},
options: libstack.Options{
ProjectDir: "something-totally-different",
ProjectName: projectName,
},
expectedProject: expectedSimpleComposeProject("something-totally-different", nil),
},
{
name: "Absolute Project and Working Directory set",
filesToCreate: map[string]string{
"docker-compose.yml": testSimpleComposeConfig,
},
configFilepaths: []string{dir + "/docker-compose.yml"},
options: libstack.Options{
WorkingDir: "/working-dir",
ProjectDir: "/project-dir",
ProjectName: projectName,
},
expectedProject: expectedSimpleComposeProject("/project-dir", nil),
},
}
for _, tc := range testcases {
t.Run(tc.name, func(t *testing.T) {
createdFiles := make([]string, 0, len(tc.filesToCreate))
for f, fc := range tc.filesToCreate {
createdFiles = append(createdFiles, createFile(t, dir, f, fc))
}
defer func() {
var errs []error
for _, f := range createdFiles {
errs = append(errs, os.Remove(f))
}
err := errors.Join(errs...)
if err != nil {
t.Fatalf("Failed to remove config files: %v", err)
}
}()
gotProject, err := createProject(ctx, tc.configFilepaths, tc.options)
if err != nil {
t.Fatalf("Failed to create new project: %v", err)
}
if diff := cmp.Diff(gotProject, tc.expectedProject); diff != "" {
t.Fatalf("Projects are different:\n%s", diff)
}
})
}
}
func createMockComposeService(dockerCli command.Cli) api.Service {
return &mockComposeService{}
}
type mockComposeService struct {
api.Service
maxConcurrency int
}
func (s *mockComposeService) MaxConcurrency(parallel int) {
s.maxConcurrency = parallel
}