From 5863e65f7b8d3ab24a68923716fa7a4719a526ed Mon Sep 17 00:00:00 2001 From: Alexis Couvreur Date: Sun, 2 Mar 2025 11:36:12 -0500 Subject: [PATCH] test(swarm): use testcontainers for tests (#542) * test(swarm): add service_inspect.go * test(swarm): add testcontainers tests --- app/sablier.go | 6 +- go.work.sum | 1 + pkg/provider/dockerswarm/docker_swarm.go | 130 +------- pkg/provider/dockerswarm/docker_swarm_test.go | 290 ------------------ pkg/provider/dockerswarm/events.go | 48 +++ pkg/provider/dockerswarm/events_test.go | 58 ++++ pkg/provider/dockerswarm/service_inspect.go | 59 ++++ .../dockerswarm/service_inspect_test.go | 144 +++++++++ .../dockerswarm/{list.go => service_list.go} | 35 ++- pkg/provider/dockerswarm/service_list_test.go | 113 +++++++ pkg/provider/dockerswarm/service_start.go | 7 + .../dockerswarm/service_start_test.go | 146 +++++++++ pkg/provider/dockerswarm/service_stop.go | 7 + pkg/provider/dockerswarm/service_stop_test.go | 114 +++++++ .../dockerswarm/testcontainers_test.go | 182 +++++++++++ 15 files changed, 915 insertions(+), 425 deletions(-) delete mode 100644 pkg/provider/dockerswarm/docker_swarm_test.go create mode 100644 pkg/provider/dockerswarm/events.go create mode 100644 pkg/provider/dockerswarm/events_test.go create mode 100644 pkg/provider/dockerswarm/service_inspect.go create mode 100644 pkg/provider/dockerswarm/service_inspect_test.go rename pkg/provider/dockerswarm/{list.go => service_list.go} (57%) create mode 100644 pkg/provider/dockerswarm/service_list_test.go create mode 100644 pkg/provider/dockerswarm/service_start.go create mode 100644 pkg/provider/dockerswarm/service_start_test.go create mode 100644 pkg/provider/dockerswarm/service_stop.go create mode 100644 pkg/provider/dockerswarm/service_stop_test.go create mode 100644 pkg/provider/dockerswarm/testcontainers_test.go diff --git a/app/sablier.go b/app/sablier.go index 80229cc..65a3a74 100644 --- a/app/sablier.go +++ b/app/sablier.go @@ -173,7 +173,11 @@ func NewProvider(ctx context.Context, logger *slog.Logger, config config.Provide switch config.Name { case "swarm", "docker_swarm": - return dockerswarm.NewDockerSwarmProvider(ctx, logger) + cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) + if err != nil { + return nil, fmt.Errorf("cannot create docker swarm client: %v", err) + } + return dockerswarm.NewDockerSwarmProvider(ctx, cli, logger) case "docker": cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) if err != nil { diff --git a/go.work.sum b/go.work.sum index f4ec229..a27c866 100644 --- a/go.work.sum +++ b/go.work.sum @@ -412,6 +412,7 @@ github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f h1:JOrtw2xFKzlg+ github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk= +github.com/cpuguy83/go-md2man/v2 v2.0.6 h1:XJtiaUW6dEEqVuZiMTn1ldk455QWwEIsMIJlo5vtkx0= github.com/cyberdelia/go-metrics-graphite v0.0.0-20161219230853-39f87cc3b432 h1:M5QgkYacWj0Xs8MhpIK/5uwU02icXpEoSo9sM2aRCps= github.com/cyberdelia/go-metrics-graphite v0.0.0-20161219230853-39f87cc3b432/go.mod h1:xwIwAxMvYnVrGJPe2FKx5prTrnAjGOD8zvDOnxnrrkM= github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= diff --git a/pkg/provider/dockerswarm/docker_swarm.go b/pkg/provider/dockerswarm/docker_swarm.go index 46b5a35..3c6ae6f 100644 --- a/pkg/provider/dockerswarm/docker_swarm.go +++ b/pkg/provider/dockerswarm/docker_swarm.go @@ -4,17 +4,13 @@ import ( "context" "errors" "fmt" - "github.com/sablierapp/sablier/app/discovery" "github.com/sablierapp/sablier/pkg/provider" - "io" "log/slog" "strings" "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/filters" "github.com/docker/docker/api/types/swarm" "github.com/docker/docker/client" - "github.com/sablierapp/sablier/app/instance" ) // Interface guard @@ -27,12 +23,8 @@ type DockerSwarmProvider struct { l *slog.Logger } -func NewDockerSwarmProvider(ctx context.Context, logger *slog.Logger) (*DockerSwarmProvider, error) { +func NewDockerSwarmProvider(ctx context.Context, cli *client.Client, logger *slog.Logger) (*DockerSwarmProvider, error) { logger = logger.With(slog.String("provider", "swarm")) - cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) - if err != nil { - return nil, fmt.Errorf("cannot create docker client: %w", err) - } serverVersion, err := cli.ServerVersion(ctx) if err != nil { @@ -52,14 +44,6 @@ func NewDockerSwarmProvider(ctx context.Context, logger *slog.Logger) (*DockerSw } -func (p *DockerSwarmProvider) Start(ctx context.Context, name string) error { - return p.scale(ctx, name, uint64(p.desiredReplicas)) -} - -func (p *DockerSwarmProvider) Stop(ctx context.Context, name string) error { - return p.scale(ctx, name, 0) -} - func (p *DockerSwarmProvider) scale(ctx context.Context, name string, replicas uint64) error { service, err := p.getServiceByName(name, ctx) if err != nil { @@ -85,80 +69,6 @@ func (p *DockerSwarmProvider) scale(ctx context.Context, name string, replicas u return nil } -func (p *DockerSwarmProvider) GetGroups(ctx context.Context) (map[string][]string, error) { - f := filters.NewArgs() - f.Add("label", fmt.Sprintf("%s=true", discovery.LabelEnable)) - - services, err := p.Client.ServiceList(ctx, types.ServiceListOptions{ - Filters: f, - }) - - if err != nil { - return nil, err - } - - groups := make(map[string][]string) - for _, service := range services { - groupName := service.Spec.Labels[discovery.LabelGroup] - if len(groupName) == 0 { - groupName = discovery.LabelGroupDefaultValue - } - - group := groups[groupName] - group = append(group, service.Spec.Name) - groups[groupName] = group - } - - return groups, nil -} - -func (p *DockerSwarmProvider) GetState(ctx context.Context, name string) (instance.State, error) { - - service, err := p.getServiceByName(name, ctx) - if err != nil { - return instance.State{}, err - } - - foundName := p.getInstanceName(name, *service) - - if service.Spec.Mode.Replicated == nil { - return instance.State{}, errors.New("swarm service is not in \"replicated\" mode") - } - - if service.ServiceStatus.DesiredTasks != service.ServiceStatus.RunningTasks || service.ServiceStatus.DesiredTasks == 0 { - return instance.NotReadyInstanceState(foundName, 0, p.desiredReplicas), nil - } - - return instance.ReadyInstanceState(foundName, p.desiredReplicas), nil -} - -func (p *DockerSwarmProvider) getServiceByName(name string, ctx context.Context) (*swarm.Service, error) { - opts := types.ServiceListOptions{ - Filters: filters.NewArgs(), - Status: true, - } - opts.Filters.Add("name", name) - - services, err := p.Client.ServiceList(ctx, opts) - - if err != nil { - return nil, err - } - - if len(services) == 0 { - return nil, fmt.Errorf("service with name %s was not found", name) - } - - for _, service := range services { - // Exact match - if service.Spec.Name == name { - return &service, nil - } - } - - return nil, fmt.Errorf("service %s was not found because it did not match exactly or on suffix", name) -} - func (p *DockerSwarmProvider) getInstanceName(name string, service swarm.Service) string { if name == service.Spec.Name { return name @@ -166,41 +76,3 @@ func (p *DockerSwarmProvider) getInstanceName(name string, service swarm.Service return fmt.Sprintf("%s (%s)", name, service.Spec.Name) } - -func (p *DockerSwarmProvider) NotifyInstanceStopped(ctx context.Context, instance chan<- string) { - msgs, errs := p.Client.Events(ctx, types.EventsOptions{ - Filters: filters.NewArgs( - filters.Arg("scope", "swarm"), - filters.Arg("type", "service"), - ), - }) - - go func() { - for { - select { - case msg, ok := <-msgs: - if !ok { - p.l.ErrorContext(ctx, "event stream closed") - return - } - if msg.Actor.Attributes["replicas.new"] == "0" { - instance <- msg.Actor.Attributes["name"] - } else if msg.Action == "remove" { - instance <- msg.Actor.Attributes["name"] - } - case err, ok := <-errs: - if !ok { - p.l.ErrorContext(ctx, "event stream closed") - return - } - if errors.Is(err, io.EOF) { - p.l.ErrorContext(ctx, "event stream closed") - return - } - p.l.ErrorContext(ctx, "event stream error", slog.Any("error", err)) - case <-ctx.Done(): - return - } - } - }() -} diff --git a/pkg/provider/dockerswarm/docker_swarm_test.go b/pkg/provider/dockerswarm/docker_swarm_test.go deleted file mode 100644 index 99c5f0e..0000000 --- a/pkg/provider/dockerswarm/docker_swarm_test.go +++ /dev/null @@ -1,290 +0,0 @@ -package dockerswarm - -import ( - "context" - "github.com/docker/docker/client" - "github.com/neilotoole/slogt" - "github.com/sablierapp/sablier/pkg/provider/mocks" - "reflect" - "testing" - - "github.com/docker/docker/api/types/events" - "github.com/docker/docker/api/types/swarm" - "github.com/sablierapp/sablier/app/instance" - "github.com/stretchr/testify/mock" -) - -func setupProvider(t *testing.T, client client.APIClient) *DockerSwarmProvider { - t.Helper() - return &DockerSwarmProvider{ - Client: client, - desiredReplicas: 1, - l: slogt.New(t), - } -} - -func TestDockerSwarmProvider_Start(t *testing.T) { - type args struct { - name string - } - tests := []struct { - name string - args args - serviceList []swarm.Service - response swarm.ServiceUpdateResponse - wantService swarm.Service - wantErr bool - }{ - { - name: "scale nginx service to 1 replica", - args: args{ - name: "nginx", - }, - serviceList: []swarm.Service{ - mocks.ServiceReplicated("nginx", 0), - }, - response: swarm.ServiceUpdateResponse{ - Warnings: []string{}, - }, - wantService: mocks.ServiceReplicated("nginx", 1), - wantErr: false, - }, - { - name: "exact match service name", - args: args{ - name: "nginx", - }, - serviceList: []swarm.Service{ - mocks.ServiceReplicated("nginx", 0), - mocks.ServiceReplicated("STACK1_nginx", 0), - mocks.ServiceReplicated("STACK2_nginx", 0), - }, - response: swarm.ServiceUpdateResponse{ - Warnings: []string{}, - }, - wantService: mocks.ServiceReplicated("nginx", 1), - wantErr: false, - }, - { - name: "nginx is not a replicated service", - args: args{ - name: "nginx", - }, - serviceList: []swarm.Service{ - mocks.ServiceGlobal("nginx"), - }, - response: swarm.ServiceUpdateResponse{ - Warnings: []string{}, - }, - wantService: mocks.ServiceReplicated("nginx", 1), - wantErr: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - clientMock := mocks.NewDockerAPIClientMock() - provider := setupProvider(t, clientMock) - - clientMock.On("ServiceList", mock.Anything, mock.Anything).Return(tt.serviceList, nil) - clientMock.On("ServiceUpdate", mock.Anything, tt.wantService.ID, tt.wantService.Meta.Version, tt.wantService.Spec, mock.Anything).Return(tt.response, nil) - - err := provider.Start(context.Background(), tt.args.name) - if (err != nil) != tt.wantErr { - t.Errorf("DockerSwarmProvider.Start() error = %v, wantErr %v", err, tt.wantErr) - return - } - }) - } -} - -func TestDockerSwarmProvider_Stop(t *testing.T) { - type args struct { - name string - } - tests := []struct { - name string - args args - serviceList []swarm.Service - response swarm.ServiceUpdateResponse - wantService swarm.Service - wantErr bool - }{ - { - name: "scale nginx service to 0 replica", - args: args{ - name: "nginx", - }, - serviceList: []swarm.Service{ - mocks.ServiceReplicated("nginx", 1), - }, - response: swarm.ServiceUpdateResponse{ - Warnings: []string{}, - }, - wantService: mocks.ServiceReplicated("nginx", 0), - wantErr: false, - }, - { - name: "exact match service name", - args: args{ - name: "nginx", - }, - serviceList: []swarm.Service{ - mocks.ServiceReplicated("nginx", 1), - mocks.ServiceReplicated("STACK1_nginx", 1), - mocks.ServiceReplicated("STACK2_nginx", 1), - }, - response: swarm.ServiceUpdateResponse{ - Warnings: []string{}, - }, - wantService: mocks.ServiceReplicated("nginx", 0), - wantErr: false, - }, - { - name: "nginx is not a replicated service", - args: args{ - name: "nginx", - }, - serviceList: []swarm.Service{ - mocks.ServiceGlobal("nginx"), - }, - response: swarm.ServiceUpdateResponse{ - Warnings: []string{}, - }, - wantService: mocks.ServiceReplicated("nginx", 1), - wantErr: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - clientMock := mocks.NewDockerAPIClientMock() - provider := setupProvider(t, clientMock) - - clientMock.On("ServiceList", mock.Anything, mock.Anything).Return(tt.serviceList, nil) - clientMock.On("ServiceUpdate", mock.Anything, tt.wantService.ID, tt.wantService.Meta.Version, tt.wantService.Spec, mock.Anything).Return(tt.response, nil) - - err := provider.Stop(context.Background(), tt.args.name) - if (err != nil) != tt.wantErr { - t.Errorf("DockerSwarmProvider.Stop() error = %v, wantErr %v", err, tt.wantErr) - return - } - }) - } -} - -func TestDockerSwarmProvider_GetState(t *testing.T) { - type args struct { - name string - } - tests := []struct { - name string - args args - want instance.State - serviceList []swarm.Service - wantErr bool - }{ - { - name: "nginx service is ready", - args: args{ - name: "nginx", - }, - serviceList: []swarm.Service{ - mocks.ServiceReplicated("nginx", 1), - }, - want: instance.State{ - Name: "nginx", - CurrentReplicas: 1, - DesiredReplicas: 1, - Status: instance.Ready, - }, - wantErr: false, - }, - { - name: "nginx service is not ready", - args: args{ - name: "nginx", - }, - serviceList: []swarm.Service{ - mocks.ServiceNotReadyReplicated("nginx", 1, 0), - }, - want: instance.State{ - Name: "nginx", - CurrentReplicas: 0, - DesiredReplicas: 1, - Status: instance.NotReady, - }, - wantErr: false, - }, - { - name: "nginx is not a replicated service", - args: args{ - name: "nginx", - }, - serviceList: []swarm.Service{ - mocks.ServiceGlobal("nginx"), - }, - want: instance.State{}, - wantErr: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - clientMock := mocks.NewDockerAPIClientMock() - provider := setupProvider(t, clientMock) - - clientMock.On("ServiceList", mock.Anything, mock.Anything).Return(tt.serviceList, nil) - - got, err := provider.GetState(context.Background(), tt.args.name) - if (err != nil) != tt.wantErr { - t.Errorf("DockerSwarmProvider.GetState() error = %v, wantErr %v", err, tt.wantErr) - return - } - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("DockerSwarmProvider.GetState() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestDockerSwarmProvider_NotifyInstanceStopped(t *testing.T) { - tests := []struct { - name string - want []string - events []events.Message - errors []error - }{ - { - name: "service nginx is scaled to 0", - want: []string{"nginx"}, - events: []events.Message{ - mocks.ServiceScaledEvent("nginx", "1", "0"), - }, - errors: []error{}, - }, { - name: "service nginx is scaled to 0", - want: []string{"nginx"}, - events: []events.Message{ - mocks.ServiceRemovedEvent("nginx"), - }, - errors: []error{}, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - provider := setupProvider(t, mocks.NewDockerAPIClientMockWithEvents(tt.events, tt.errors)) - - instanceC := make(chan string) - - ctx, cancel := context.WithCancel(context.Background()) - provider.NotifyInstanceStopped(ctx, instanceC) - - var got []string - - got = append(got, <-instanceC) - cancel() - - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("NotifyInstanceStopped() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/pkg/provider/dockerswarm/events.go b/pkg/provider/dockerswarm/events.go new file mode 100644 index 0000000..736fd02 --- /dev/null +++ b/pkg/provider/dockerswarm/events.go @@ -0,0 +1,48 @@ +package dockerswarm + +import ( + "context" + "errors" + "github.com/docker/docker/api/types/events" + "github.com/docker/docker/api/types/filters" + "io" + "log/slog" +) + +func (p *DockerSwarmProvider) NotifyInstanceStopped(ctx context.Context, instance chan<- string) { + msgs, errs := p.Client.Events(ctx, events.ListOptions{ + Filters: filters.NewArgs( + filters.Arg("scope", "swarm"), + filters.Arg("type", "service"), + ), + }) + + go func() { + for { + select { + case msg, ok := <-msgs: + if !ok { + p.l.ErrorContext(ctx, "event stream closed") + return + } + if msg.Actor.Attributes["replicas.new"] == "0" { + instance <- msg.Actor.Attributes["name"] + } else if msg.Action == "remove" { + instance <- msg.Actor.Attributes["name"] + } + case err, ok := <-errs: + if !ok { + p.l.ErrorContext(ctx, "event stream closed") + return + } + if errors.Is(err, io.EOF) { + p.l.ErrorContext(ctx, "event stream closed") + return + } + p.l.ErrorContext(ctx, "event stream error", slog.Any("error", err)) + case <-ctx.Done(): + return + } + } + }() +} diff --git a/pkg/provider/dockerswarm/events_test.go b/pkg/provider/dockerswarm/events_test.go new file mode 100644 index 0000000..7aebf9d --- /dev/null +++ b/pkg/provider/dockerswarm/events_test.go @@ -0,0 +1,58 @@ +package dockerswarm_test + +import ( + "context" + "github.com/docker/docker/api/types" + "github.com/neilotoole/slogt" + "github.com/sablierapp/sablier/pkg/provider/dockerswarm" + "gotest.tools/v3/assert" + "testing" + "time" +) + +func TestDockerSwarmProvider_NotifyInstanceStopped(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode.") + } + + ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) + defer cancel() + dind := setupDinD(t, ctx) + p, err := dockerswarm.NewDockerSwarmProvider(ctx, dind.client, slogt.New(t)) + assert.NilError(t, err) + + c, err := dind.CreateMimic(ctx, MimicOptions{}) + assert.NilError(t, err) + + waitC := make(chan string) + go p.NotifyInstanceStopped(ctx, waitC) + + t.Run("service is scaled to 0 replicas", func(t *testing.T) { + service, _, err := dind.client.ServiceInspectWithRaw(ctx, c.ID, types.ServiceInspectOptions{}) + assert.NilError(t, err) + + replicas := uint64(0) + service.Spec.Mode.Replicated.Replicas = &replicas + + _, err = p.Client.ServiceUpdate(ctx, service.ID, service.Meta.Version, service.Spec, types.ServiceUpdateOptions{}) + assert.NilError(t, err) + + name := <-waitC + + // Docker container name is prefixed with a slash, but we don't use it + assert.Equal(t, name, service.Spec.Name) + }) + + t.Run("service is removed", func(t *testing.T) { + service, _, err := dind.client.ServiceInspectWithRaw(ctx, c.ID, types.ServiceInspectOptions{}) + assert.NilError(t, err) + + err = p.Client.ServiceRemove(ctx, service.ID) + assert.NilError(t, err) + + name := <-waitC + + // Docker container name is prefixed with a slash, but we don't use it + assert.Equal(t, name, service.Spec.Name) + }) +} diff --git a/pkg/provider/dockerswarm/service_inspect.go b/pkg/provider/dockerswarm/service_inspect.go new file mode 100644 index 0000000..ee3fbad --- /dev/null +++ b/pkg/provider/dockerswarm/service_inspect.go @@ -0,0 +1,59 @@ +package dockerswarm + +import ( + "context" + "errors" + "fmt" + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/filters" + "github.com/docker/docker/api/types/swarm" + "github.com/sablierapp/sablier/app/instance" +) + +func (p *DockerSwarmProvider) GetState(ctx context.Context, name string) (instance.State, error) { + service, err := p.getServiceByName(name, ctx) + if err != nil { + return instance.State{}, err + } + + foundName := p.getInstanceName(name, *service) + + if service.Spec.Mode.Replicated == nil { + return instance.State{}, errors.New("swarm service is not in \"replicated\" mode") + } + + if service.ServiceStatus.DesiredTasks != service.ServiceStatus.RunningTasks || service.ServiceStatus.DesiredTasks == 0 { + return instance.NotReadyInstanceState(foundName, 0, p.desiredReplicas), nil + } + + return instance.ReadyInstanceState(foundName, p.desiredReplicas), nil +} + +func (p *DockerSwarmProvider) getServiceByName(name string, ctx context.Context) (*swarm.Service, error) { + opts := types.ServiceListOptions{ + Filters: filters.NewArgs(), + Status: true, + } + opts.Filters.Add("name", name) + + services, err := p.Client.ServiceList(ctx, opts) + if err != nil { + return nil, fmt.Errorf("error listing services: %w", err) + } + + if len(services) == 0 { + return nil, fmt.Errorf("service with name %s was not found", name) + } + + for _, service := range services { + // Exact match + if service.Spec.Name == name { + return &service, nil + } + if service.ID == name { + return &service, nil + } + } + + return nil, fmt.Errorf("service %s was not found because it did not match exactly or on suffix", name) +} diff --git a/pkg/provider/dockerswarm/service_inspect_test.go b/pkg/provider/dockerswarm/service_inspect_test.go new file mode 100644 index 0000000..01ce488 --- /dev/null +++ b/pkg/provider/dockerswarm/service_inspect_test.go @@ -0,0 +1,144 @@ +package dockerswarm_test + +import ( + "context" + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" + "github.com/google/go-cmp/cmp" + "github.com/neilotoole/slogt" + "github.com/sablierapp/sablier/app/instance" + "github.com/sablierapp/sablier/pkg/provider/dockerswarm" + "gotest.tools/v3/assert" + "testing" + "time" +) + +func TestDockerSwarmProvider_GetState(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode.") + } + + ctx := context.Background() + type args struct { + do func(dind *dindContainer) (string, error) + } + tests := []struct { + name string + args args + want instance.State + wantErr error + }{ + { + name: "service with 1/1 replicas", + args: args{ + do: func(dind *dindContainer) (string, error) { + s, err := dind.CreateMimic(ctx, MimicOptions{ + Cmd: []string{"/mimic"}, + Healthcheck: nil, + }) + if err != nil { + return "", err + } + + service, _, err := dind.client.ServiceInspectWithRaw(ctx, s.ID, types.ServiceInspectOptions{}) + if err != nil { + return "", err + } + + <-time.After(5 * time.Second) + + return service.Spec.Name, err + }, + }, + want: instance.State{ + CurrentReplicas: 1, + DesiredReplicas: 1, + Status: instance.Ready, + }, + wantErr: nil, + }, + { + name: "service with 0/1 replicas", + args: args{ + do: func(dind *dindContainer) (string, error) { + s, err := dind.CreateMimic(ctx, MimicOptions{ + Cmd: []string{"/mimic", "-running-after=1ms", "-healthy=false", "-healthy-after=10s"}, + Healthcheck: &container.HealthConfig{ + Test: []string{"CMD", "/mimic", "healthcheck"}, + Interval: time.Second, + Timeout: time.Second, + StartPeriod: time.Second, + StartInterval: time.Second, + Retries: 10, + }, + }) + if err != nil { + return "", err + } + + service, _, err := dind.client.ServiceInspectWithRaw(ctx, s.ID, types.ServiceInspectOptions{}) + if err != nil { + return "", err + } + + return service.Spec.Name, nil + }, + }, + want: instance.State{ + CurrentReplicas: 0, + DesiredReplicas: 1, + Status: instance.NotReady, + }, + wantErr: nil, + }, + { + name: "service with 0/0 replicas", + args: args{ + do: func(dind *dindContainer) (string, error) { + s, err := dind.CreateMimic(ctx, MimicOptions{}) + if err != nil { + return "", err + } + + service, _, err := dind.client.ServiceInspectWithRaw(ctx, s.ID, types.ServiceInspectOptions{}) + if err != nil { + return "", err + } + + replicas := uint64(0) + service.Spec.Mode.Replicated.Replicas = &replicas + _, err = dind.client.ServiceUpdate(ctx, s.ID, service.Version, service.Spec, types.ServiceUpdateOptions{}) + if err != nil { + return "", err + } + + return service.Spec.Name, nil + }, + }, + want: instance.State{ + CurrentReplicas: 0, + DesiredReplicas: 1, + Status: instance.NotReady, + }, + wantErr: nil, + }, + } + c := setupDinD(t, ctx) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + p, err := dockerswarm.NewDockerSwarmProvider(ctx, c.client, slogt.New(t)) + + name, err := tt.args.do(c) + assert.NilError(t, err) + + tt.want.Name = name + got, err := p.GetState(ctx, name) + if !cmp.Equal(err, tt.wantErr) { + t.Errorf("DockerSwarmProvider.GetState() error = %v, wantErr %v", err, tt.wantErr) + return + } + assert.DeepEqual(t, got, tt.want) + }) + } +} diff --git a/pkg/provider/dockerswarm/list.go b/pkg/provider/dockerswarm/service_list.go similarity index 57% rename from pkg/provider/dockerswarm/list.go rename to pkg/provider/dockerswarm/service_list.go index c8e4ba5..7f98e05 100644 --- a/pkg/provider/dockerswarm/list.go +++ b/pkg/provider/dockerswarm/service_list.go @@ -11,12 +11,10 @@ import ( "github.com/sablierapp/sablier/pkg/provider" ) -func (p *DockerSwarmProvider) InstanceList(ctx context.Context, options provider.InstanceListOptions) ([]types.Instance, error) { +func (p *DockerSwarmProvider) InstanceList(ctx context.Context, _ provider.InstanceListOptions) ([]types.Instance, error) { args := filters.NewArgs() - for _, label := range options.Labels { - args.Add("label", label) - args.Add("label", fmt.Sprintf("%s=true", label)) - } + args.Add("label", fmt.Sprintf("%s=true", discovery.LabelEnable)) + args.Add("mode", "replicated") services, err := p.Client.ServiceList(ctx, dockertypes.ServiceListOptions{ Filters: args, @@ -51,3 +49,30 @@ func (p *DockerSwarmProvider) serviceToInstance(s swarm.Service) (i types.Instan Group: group, } } + +func (p *DockerSwarmProvider) GetGroups(ctx context.Context) (map[string][]string, error) { + f := filters.NewArgs() + f.Add("label", fmt.Sprintf("%s=true", discovery.LabelEnable)) + + services, err := p.Client.ServiceList(ctx, dockertypes.ServiceListOptions{ + Filters: f, + }) + + if err != nil { + return nil, err + } + + groups := make(map[string][]string) + for _, service := range services { + groupName := service.Spec.Labels[discovery.LabelGroup] + if len(groupName) == 0 { + groupName = discovery.LabelGroupDefaultValue + } + + group := groups[groupName] + group = append(group, service.Spec.Name) + groups[groupName] = group + } + + return groups, nil +} diff --git a/pkg/provider/dockerswarm/service_list_test.go b/pkg/provider/dockerswarm/service_list_test.go new file mode 100644 index 0000000..3cb3619 --- /dev/null +++ b/pkg/provider/dockerswarm/service_list_test.go @@ -0,0 +1,113 @@ +package dockerswarm_test + +import ( + dockertypes "github.com/docker/docker/api/types" + + "github.com/neilotoole/slogt" + "github.com/sablierapp/sablier/app/types" + "github.com/sablierapp/sablier/pkg/provider" + "github.com/sablierapp/sablier/pkg/provider/dockerswarm" + "gotest.tools/v3/assert" + "sort" + "strings" + "testing" +) + +func TestDockerClassicProvider_InstanceList(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode.") + } + + ctx := t.Context() + dind := setupDinD(t, ctx) + p, err := dockerswarm.NewDockerSwarmProvider(ctx, dind.client, slogt.New(t)) + assert.NilError(t, err) + + s1, err := dind.CreateMimic(ctx, MimicOptions{ + Labels: map[string]string{ + "sablier.enable": "true", + }, + }) + assert.NilError(t, err) + + i1, _, err := dind.client.ServiceInspectWithRaw(ctx, s1.ID, dockertypes.ServiceInspectOptions{}) + assert.NilError(t, err) + + s2, err := dind.CreateMimic(ctx, MimicOptions{ + Labels: map[string]string{ + "sablier.enable": "true", + "sablier.group": "my-group", + }, + }) + assert.NilError(t, err) + + i2, _, err := dind.client.ServiceInspectWithRaw(ctx, s2.ID, dockertypes.ServiceInspectOptions{}) + assert.NilError(t, err) + + got, err := p.InstanceList(ctx, provider.InstanceListOptions{ + All: true, + }) + assert.NilError(t, err) + + want := []types.Instance{ + { + Name: i1.Spec.Name, + Group: "default", + }, + { + Name: i2.Spec.Name, + Group: "my-group", + }, + } + // Assert go is equal to want + // Sort both array to ensure they are equal + sort.Slice(got, func(i, j int) bool { + return strings.Compare(got[i].Name, got[j].Name) < 0 + }) + sort.Slice(want, func(i, j int) bool { + return strings.Compare(want[i].Name, want[j].Name) < 0 + }) + assert.DeepEqual(t, got, want) +} + +func TestDockerClassicProvider_GetGroups(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode.") + } + + ctx := t.Context() + dind := setupDinD(t, ctx) + p, err := dockerswarm.NewDockerSwarmProvider(ctx, dind.client, slogt.New(t)) + assert.NilError(t, err) + + s1, err := dind.CreateMimic(ctx, MimicOptions{ + Labels: map[string]string{ + "sablier.enable": "true", + }, + }) + assert.NilError(t, err) + + i1, _, err := dind.client.ServiceInspectWithRaw(ctx, s1.ID, dockertypes.ServiceInspectOptions{}) + assert.NilError(t, err) + + s2, err := dind.CreateMimic(ctx, MimicOptions{ + Labels: map[string]string{ + "sablier.enable": "true", + "sablier.group": "my-group", + }, + }) + assert.NilError(t, err) + + i2, _, err := dind.client.ServiceInspectWithRaw(ctx, s2.ID, dockertypes.ServiceInspectOptions{}) + assert.NilError(t, err) + + got, err := p.GetGroups(ctx) + assert.NilError(t, err) + + want := map[string][]string{ + "default": {i1.Spec.Name}, + "my-group": {i2.Spec.Name}, + } + + assert.DeepEqual(t, got, want) +} diff --git a/pkg/provider/dockerswarm/service_start.go b/pkg/provider/dockerswarm/service_start.go new file mode 100644 index 0000000..bb97030 --- /dev/null +++ b/pkg/provider/dockerswarm/service_start.go @@ -0,0 +1,7 @@ +package dockerswarm + +import "context" + +func (p *DockerSwarmProvider) Start(ctx context.Context, name string) error { + return p.scale(ctx, name, uint64(p.desiredReplicas)) +} diff --git a/pkg/provider/dockerswarm/service_start_test.go b/pkg/provider/dockerswarm/service_start_test.go new file mode 100644 index 0000000..e586098 --- /dev/null +++ b/pkg/provider/dockerswarm/service_start_test.go @@ -0,0 +1,146 @@ +package dockerswarm_test + +import ( + "context" + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" + "github.com/google/go-cmp/cmp" + "github.com/neilotoole/slogt" + "github.com/sablierapp/sablier/app/instance" + "github.com/sablierapp/sablier/pkg/provider/dockerswarm" + "gotest.tools/v3/assert" + "testing" + "time" +) + +func TestDockerSwarmProvider_Start(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode.") + } + + ctx := context.Background() + type args struct { + do func(dind *dindContainer) (string, error) + } + tests := []struct { + name string + args args + want instance.State + wantErr error + }{ + { + name: "service with 1/1 replicas", + args: args{ + do: func(dind *dindContainer) (string, error) { + s, err := dind.CreateMimic(ctx, MimicOptions{ + Cmd: []string{"/mimic"}, + Healthcheck: nil, + }) + if err != nil { + return "", err + } + + service, _, err := dind.client.ServiceInspectWithRaw(ctx, s.ID, types.ServiceInspectOptions{}) + if err != nil { + return "", err + } + + return service.Spec.Name, err + }, + }, + want: instance.State{ + CurrentReplicas: 1, + DesiredReplicas: 1, + Status: instance.Ready, + }, + wantErr: nil, + }, + { + name: "service with 0/1 replicas", + args: args{ + do: func(dind *dindContainer) (string, error) { + s, err := dind.CreateMimic(ctx, MimicOptions{ + Cmd: []string{"/mimic", "-running-after=1ms", "-healthy=false", "-healthy-after=10s"}, + Healthcheck: &container.HealthConfig{ + Test: []string{"CMD", "/mimic", "healthcheck"}, + Interval: time.Second, + Timeout: time.Second, + StartPeriod: time.Second, + StartInterval: time.Second, + Retries: 10, + }, + }) + if err != nil { + return "", err + } + + service, _, err := dind.client.ServiceInspectWithRaw(ctx, s.ID, types.ServiceInspectOptions{}) + if err != nil { + return "", err + } + + return service.Spec.Name, nil + }, + }, + want: instance.State{ + CurrentReplicas: 0, + DesiredReplicas: 1, + Status: instance.NotReady, + }, + wantErr: nil, + }, + + { + name: "service with 0/0 replicas", + args: args{ + do: func(dind *dindContainer) (string, error) { + s, err := dind.CreateMimic(ctx, MimicOptions{}) + if err != nil { + return "", err + } + + service, _, err := dind.client.ServiceInspectWithRaw(ctx, s.ID, types.ServiceInspectOptions{}) + if err != nil { + return "", err + } + + replicas := uint64(0) + service.Spec.Mode.Replicated.Replicas = &replicas + _, err = dind.client.ServiceUpdate(ctx, s.ID, service.Version, service.Spec, types.ServiceUpdateOptions{}) + if err != nil { + return "", err + } + + return service.Spec.Name, nil + }, + }, + want: instance.State{ + CurrentReplicas: 0, + DesiredReplicas: 1, + Status: instance.NotReady, + }, + wantErr: nil, + }, + } + c := setupDinD(t, ctx) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + p, err := dockerswarm.NewDockerSwarmProvider(ctx, c.client, slogt.New(t)) + + name, err := tt.args.do(c) + assert.NilError(t, err) + + tt.want.Name = name + err = p.Start(ctx, name) + if !cmp.Equal(err, tt.wantErr) { + t.Errorf("DockerSwarmProvider.Stop() error = %v, wantErr %v", err, tt.wantErr) + return + } + + service, _, err := c.client.ServiceInspectWithRaw(ctx, name, types.ServiceInspectOptions{}) + assert.NilError(t, err) + assert.Equal(t, *service.Spec.Mode.Replicated.Replicas, uint64(1)) + }) + } +} diff --git a/pkg/provider/dockerswarm/service_stop.go b/pkg/provider/dockerswarm/service_stop.go new file mode 100644 index 0000000..a05a474 --- /dev/null +++ b/pkg/provider/dockerswarm/service_stop.go @@ -0,0 +1,7 @@ +package dockerswarm + +import "context" + +func (p *DockerSwarmProvider) Stop(ctx context.Context, name string) error { + return p.scale(ctx, name, 0) +} diff --git a/pkg/provider/dockerswarm/service_stop_test.go b/pkg/provider/dockerswarm/service_stop_test.go new file mode 100644 index 0000000..4f79034 --- /dev/null +++ b/pkg/provider/dockerswarm/service_stop_test.go @@ -0,0 +1,114 @@ +package dockerswarm_test + +import ( + "context" + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" + "github.com/google/go-cmp/cmp" + "github.com/neilotoole/slogt" + "github.com/sablierapp/sablier/app/instance" + "github.com/sablierapp/sablier/pkg/provider/dockerswarm" + "gotest.tools/v3/assert" + "testing" + "time" +) + +func TestDockerSwarmProvider_Stop(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode.") + } + + ctx := context.Background() + type args struct { + do func(dind *dindContainer) (string, error) + } + tests := []struct { + name string + args args + want instance.State + wantErr error + }{ + { + name: "service with 1/1 replicas", + args: args{ + do: func(dind *dindContainer) (string, error) { + s, err := dind.CreateMimic(ctx, MimicOptions{ + Cmd: []string{"/mimic"}, + Healthcheck: nil, + }) + if err != nil { + return "", err + } + + service, _, err := dind.client.ServiceInspectWithRaw(ctx, s.ID, types.ServiceInspectOptions{}) + if err != nil { + return "", err + } + + return service.Spec.Name, err + }, + }, + want: instance.State{ + CurrentReplicas: 1, + DesiredReplicas: 1, + Status: instance.Ready, + }, + wantErr: nil, + }, + { + name: "service with 0/1 replicas", + args: args{ + do: func(dind *dindContainer) (string, error) { + s, err := dind.CreateMimic(ctx, MimicOptions{ + Cmd: []string{"/mimic", "-running-after=1ms", "-healthy=false", "-healthy-after=10s"}, + Healthcheck: &container.HealthConfig{ + Test: []string{"CMD", "/mimic", "healthcheck"}, + Interval: time.Second, + Timeout: time.Second, + StartPeriod: time.Second, + StartInterval: time.Second, + Retries: 10, + }, + }) + if err != nil { + return "", err + } + + service, _, err := dind.client.ServiceInspectWithRaw(ctx, s.ID, types.ServiceInspectOptions{}) + if err != nil { + return "", err + } + + return service.Spec.Name, nil + }, + }, + want: instance.State{ + CurrentReplicas: 0, + DesiredReplicas: 1, + Status: instance.NotReady, + }, + wantErr: nil, + }, + } + c := setupDinD(t, ctx) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + p, err := dockerswarm.NewDockerSwarmProvider(ctx, c.client, slogt.New(t)) + + name, err := tt.args.do(c) + assert.NilError(t, err) + + tt.want.Name = name + err = p.Stop(ctx, name) + if !cmp.Equal(err, tt.wantErr) { + t.Errorf("DockerSwarmProvider.Stop() error = %v, wantErr %v", err, tt.wantErr) + return + } + + service, _, err := c.client.ServiceInspectWithRaw(ctx, name, types.ServiceInspectOptions{}) + assert.NilError(t, err) + assert.Equal(t, *service.Spec.Mode.Replicated.Replicas, uint64(0)) + }) + } +} diff --git a/pkg/provider/dockerswarm/testcontainers_test.go b/pkg/provider/dockerswarm/testcontainers_test.go new file mode 100644 index 0000000..464f8f7 --- /dev/null +++ b/pkg/provider/dockerswarm/testcontainers_test.go @@ -0,0 +1,182 @@ +package dockerswarm_test + +import ( + "bytes" + "context" + "fmt" + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/image" + "github.com/docker/docker/api/types/swarm" + "github.com/docker/docker/client" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" + "gotest.tools/v3/assert" + "slices" + "testing" +) + +type dindContainer struct { + testcontainers.Container + client *client.Client + t *testing.T +} + +type MimicOptions struct { + Cmd []string + Healthcheck *container.HealthConfig + RestartPolicy *swarm.RestartPolicy + Labels map[string]string +} + +func (d *dindContainer) CreateMimic(ctx context.Context, opts MimicOptions) (swarm.ServiceCreateResponse, error) { + if len(opts.Cmd) == 0 { + opts.Cmd = []string{"/mimic", "-running", "-running-after=1s", "-healthy=false"} + } + + d.t.Log("Creating mimic service with options", opts) + var replicas uint64 = 1 + return d.client.ServiceCreate(ctx, swarm.ServiceSpec{ + Mode: swarm.ServiceMode{ + Replicated: &swarm.ReplicatedService{Replicas: &replicas}, + }, + TaskTemplate: swarm.TaskSpec{ + RestartPolicy: opts.RestartPolicy, + ContainerSpec: &swarm.ContainerSpec{ + Image: "sablierapp/mimic:v0.3.1", + Healthcheck: opts.Healthcheck, + Command: opts.Cmd, + }, + }, + Annotations: swarm.Annotations{ + Labels: opts.Labels, + }, + }, types.ServiceCreateOptions{}) +} + +func setupDinD(t *testing.T, ctx context.Context) *dindContainer { + t.Helper() + cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) + assert.NilError(t, err) + + req := testcontainers.ContainerRequest{ + Image: "docker:dind", + ExposedPorts: []string{"2375/tcp"}, + WaitingFor: wait.ForLog("API listen on [::]:2375"), + Cmd: []string{ + "dockerd", "-H", "tcp://0.0.0.0:2375", "--tls=false", + }, + Privileged: true, + } + c, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: req, + Started: true, + Logger: testcontainers.TestLogger(t), + }) + assert.NilError(t, err) + t.Cleanup(func() { + testcontainers.CleanupContainer(t, c) + }) + + ip, err := c.Host(ctx) + assert.NilError(t, err) + + mappedPort, err := c.MappedPort(ctx, "2375") + assert.NilError(t, err) + + host := fmt.Sprintf("http://%s:%s", ip, mappedPort.Port()) + dindCli, err := client.NewClientWithOpts(client.WithHost(host), client.WithAPIVersionNegotiation()) + assert.NilError(t, err) + + err = addMimicToDind(ctx, cli, dindCli) + assert.NilError(t, err) + + // Initialize the swarm + _, err = dindCli.SwarmInit(ctx, swarm.InitRequest{ + ListenAddr: "0.0.0.0", + }) + assert.NilError(t, err) + + return &dindContainer{ + Container: c, + client: dindCli, + t: t, + } +} + +func searchMimicImage(ctx context.Context, cli *client.Client) (string, error) { + images, err := cli.ImageList(ctx, image.ListOptions{}) + if err != nil { + return "", fmt.Errorf("failed to list images: %w", err) + } + + for _, summary := range images { + if slices.Contains(summary.RepoTags, "sablierapp/mimic:v0.3.1") { + return summary.ID, nil + } + } + + return "", nil +} + +func pullMimicImage(ctx context.Context, cli *client.Client) error { + reader, err := cli.ImagePull(ctx, "sablierapp/mimic:v0.3.1", image.PullOptions{}) + if err != nil { + return fmt.Errorf("failed to pull image: %w", err) + } + defer reader.Close() + resp, err := cli.ImageLoad(ctx, reader, true) + if err != nil { + return fmt.Errorf("failed to load image: %w", err) + } + defer resp.Body.Close() + return nil +} + +func addMimicToDind(ctx context.Context, cli *client.Client, dindCli *client.Client) error { + ID, err := searchMimicImage(ctx, cli) + if err != nil { + return fmt.Errorf("failed to search for mimic image: %w", err) + } + + if ID == "" { + err = pullMimicImage(ctx, cli) + if err != nil { + return err + } + + ID, err = searchMimicImage(ctx, cli) + if err != nil { + return fmt.Errorf("failed to search for mimic image even though it's just been pulled without errors: %w", err) + } + } + + reader, err := cli.ImageSave(ctx, []string{ID}) + if err != nil { + return fmt.Errorf("failed to save image: %w", err) + } + defer reader.Close() + + resp, err := dindCli.ImageLoad(ctx, reader, true) + if err != nil { + return fmt.Errorf("failed to load image in docker in docker container: %w", err) + } + defer resp.Body.Close() + buf := new(bytes.Buffer) + _, err = buf.ReadFrom(resp.Body) + if err != nil { + return fmt.Errorf("failed to read from response body: %w", err) + } + + list, err := dindCli.ImageList(ctx, image.ListOptions{}) + if err != nil { + return err + } + + err = dindCli.ImageTag(ctx, list[0].ID, "sablierapp/mimic:v0.3.1") + if err != nil { + return err + } + + return nil +}