mirrors/dozzle
1
0
Fork 0
mirror of https://github.com/amir20/dozzle.git synced 2025-12-21 21:33:18 +01:00
Code Issues Projects Releases Wiki Activity

perf: fixes an issue with too many containers (#3144)

Browse Source
This commit is contained in:
Amir Raminfar
2024-07-24 06:58:21 -07:00
committed by GitHub
parent 4aa315a471
commit 082bed2382
16 changed files with 136 additions and 173 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
assets/components/FuzzySearchModal.spec.ts
View File

@@ -31,9 +31,9 @@ function createFuzzySearchModal() {
initialState: {
container: {
containers: [
new Container("123", new Date(), "image", "test", "command", "host", {}, "status", "running", []),
new Container("345", new Date(), "image", "foo bar", "command", "host", {}, "status", "running", []),
new Container("567", new Date(), "image", "baz", "command", "host", {}, "status", "running", []),
new Container("123", new Date(), "image", "test", "command", "host", {}, "running", []),
new Container("345", new Date(), "image", "foo bar", "command", "host", {}, "running", []),
new Container("567", new Date(), "image", "baz", "command", "host", {}, "running", []),
],
},
},

2
assets/components/LogViewer/EventSource.spec.ts
View File

@@ -88,7 +88,7 @@ describe("<ContainerEventSource />", () => {
},
props: {
streamSource: useContainerStream,
entity: new Container("abc", new Date(), "image", "name", "command", "localhost", {}, "status", "created", []),
entity: new Container("abc", new Date(), "image", "name", "command", "localhost", {}, "created", []),
},
});
}

1
assets/models/Container.ts
View File

@@ -35,7 +35,6 @@ export class Container {
public readonly command: string,
public readonly host: string,
public readonly labels = {} as Record<string, string>,
public status: string,
public state: ContainerState,
stats: Stat[],
public readonly group?: string,

2
assets/stores/container.ts
View File

@@ -121,7 +121,6 @@ export const useContainerStore = defineStore("container", () => {
existingContainers.forEach((c) => {
const existing = allContainersById.value[c.id];
existing.status = c.status;
existing.state = c.state;
existing.health = c.health;
});
@@ -137,7 +136,6 @@ export const useContainerStore = defineStore("container", () => {
c.command,
c.host,
c.labels,
c.status,
c.state,
c.stats,
c.group,

3
internal/agent/client.go
View File

@@ -263,7 +263,6 @@ func (c *Client) StreamNewContainers(ctx context.Context, containers chan<- dock
ImageID: resp.Container.ImageId,
Created: resp.Container.Created.AsTime(),
State: resp.Container.State,
Status: resp.Container.Status,
Health: resp.Container.Health,
Host: resp.Container.Host,
Tty: resp.Container.Tty,
@@ -305,7 +304,6 @@ func (c *Client) FindContainer(containerID string) (docker.Container, error) {
ImageID: response.Container.ImageId,
Created: response.Container.Created.AsTime(),
State: response.Container.State,
Status: response.Container.Status,
Health: response.Container.Health,
Host: response.Container.Host,
Tty: response.Container.Tty,
@@ -348,7 +346,6 @@ func (c *Client) ListContainers() ([]docker.Container, error) {
ImageID: container.ImageId,
Created: container.Created.AsTime(),
State: container.State,
Status: container.Status,
Health: container.Health,
Host: container.Host,
Tty: container.Tty,

12
internal/agent/client_test.go
View File

@@ -94,16 +94,19 @@ func init() {
client = &MockedClient{}
client.On("ListContainers").Return([]docker.Container{
{
ID: "123456",
Name: "test",
Host: "localhost",
ID: "123456",
Name: "test",
Host: "localhost",
State: "running",
},
}, nil)
client.On("Host").Return(docker.Host{
ID: "localhost",
Endpoint: "local",
Name: "local",
})
client.On("ContainerEvents", mock.Anything, mock.AnythingOfType("chan<- docker.ContainerEvent")).Return(nil).Run(func(args mock.Arguments) {
time.Sleep(5 * time.Second)
})
@@ -116,7 +119,6 @@ func init() {
ImageID: "test",
StartedAt: &time.Time{},
State: "running",
Status: "running",
Health: "healthy",
Group: "test",
Command: "test",
@@ -151,7 +153,6 @@ func TestFindContainer(t *testing.T) {
ImageID: "test",
StartedAt: &time.Time{},
State: "running",
Status: "running",
Health: "healthy",
Group: "test",
Command: "test",
@@ -181,7 +182,6 @@ func TestListContainers(t *testing.T) {
ImageID: "test",
StartedAt: &time.Time{},
State: "running",
Status: "running",
Health: "healthy",
Group: "test",
Command: "test",

3
internal/agent/server.go
View File

@@ -182,7 +182,6 @@ func (s *server) FindContainer(ctx context.Context, in *pb.FindContainerRequest)
Command: container.Command,
Created: timestamppb.New(container.Created),
State: container.State,
Status: container.Status,
Health: container.Health,
Host: container.Host,
Tty: container.Tty,
@@ -224,7 +223,6 @@ func (s *server) ListContainers(ctx context.Context, in *pb.ListContainersReques
ImageId: container.ImageID,
Created: timestamppb.New(container.Created),
State: container.State,
Status: container.Status,
Health: container.Health,
Host: container.Host,
Tty: container.Tty,
@@ -269,7 +267,6 @@ func (s *server) StreamContainerStarted(in *pb.StreamContainerStartedRequest, ou
ImageId: container.ImageID,
Created: timestamppb.New(container.Created),
State: container.State,
Status: container.Status,
Health: container.Health,
Host: container.Host,
Tty: container.Tty,

130
internal/docker/client.go
View File

@@ -5,7 +5,6 @@ import (
"fmt"
"io"
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
@@ -176,37 +175,15 @@ func NewRemoteClient(f map[string][]string, host Host) (Client, error) {
return NewClient(cli, filterArgs, host), nil
}
// Finds a container by id, skipping the filters
func (d *httpClient) FindContainer(id string) (Container, error) {
log.Debugf("finding container with id: %s", id)
var container Container
containers, err := d.ListContainers()
if err != nil {
return container, err
}
found := false
for _, c := range containers {
if c.ID == id {
container = c
found = true
break
}
}
if !found {
return container, fmt.Errorf("unable to find container with id: %s", id)
}
if json, err := d.cli.ContainerInspect(context.Background(), container.ID); err == nil {
container.Tty = json.Config.Tty
if startedAt, err := time.Parse(time.RFC3339Nano, json.State.StartedAt); err == nil {
utc := startedAt.UTC()
container.StartedAt = &utc
}
if json, err := d.cli.ContainerInspect(context.Background(), id); err == nil {
return newContainerFromJSON(json, d.host.ID), nil
} else {
return container, err
return Container{}, err
}
return container, nil
}
func (d *httpClient) ContainerActions(action ContainerAction, containerID string) error {
@@ -223,6 +200,7 @@ func (d *httpClient) ContainerActions(action ContainerAction, containerID string
}
func (d *httpClient) ListContainers() ([]Container, error) {
log.Debugf("listing containers with filters: %v", d.filters)
containerListOptions := container.ListOptions{
Filters: d.filters,
All: true,
@@ -234,33 +212,7 @@ func (d *httpClient) ListContainers() ([]Container, error) {
var containers = make([]Container, 0, len(list))
for _, c := range list {
name := "no name"
if len(c.Names) > 0 {
name = strings.TrimPrefix(c.Names[0], "/")
}
group := ""
if c.Labels["dev.dozzle.group"] != "" {
group = c.Labels["dev.dozzle.group"]
}
container := Container{
ID: c.ID[:12],
Names: c.Names,
Name: name,
Image: c.Image,
ImageID: c.ImageID,
Command: c.Command,
Created: time.Unix(c.Created, 0),
State: c.State,
Status: c.Status,
Host: d.host.ID,
Health: findBetweenParentheses(c.Status),
Labels: c.Labels,
Stats: utils.NewRingBuffer[ContainerStat](300), // 300 seconds of stats
Group: group,
}
containers = append(containers, container)
containers = append(containers, newContainer(c, d.host.ID))
}
sort.Slice(containers, func(i, j int) bool {
@@ -398,11 +350,69 @@ func (d *httpClient) SystemInfo() system.Info {
return d.info
}
var PARENTHESIS_RE = regexp.MustCompile(`\(([a-zA-Z]+)\)`)
func findBetweenParentheses(s string) string {
if results := PARENTHESIS_RE.FindStringSubmatch(s); results != nil {
return results[1]
func newContainer(c types.Container, host string) Container {
name := "no name"
if len(c.Names) > 0 {
name = strings.TrimPrefix(c.Names[0], "/")
}
group := ""
if c.Labels["dev.dozzle.group"] != "" {
group = c.Labels["dev.dozzle.group"]
}
return Container{
ID: c.ID[:12],
Name: name,
Image: c.Image,
ImageID: c.ImageID,
Command: c.Command,
Created: time.Unix(c.Created, 0),
State: c.State,
Host: host,
Labels: c.Labels,
Stats: utils.NewRingBuffer[ContainerStat](300), // 300 seconds of stats
Group: group,
}
return ""
}
func newContainerFromJSON(c types.ContainerJSON, host string) Container {
name := "no name"
if len(c.Name) > 0 {
name = strings.TrimPrefix(c.Name, "/")
}
group := ""
if c.Config.Labels["dev.dozzle.group"] != "" {
group = c.Config.Labels["dev.dozzle.group"]
}
container := Container{
ID: c.ID[:12],
Name: name,
Image: c.Image,
ImageID: c.Image,
Command: strings.Join(c.Config.Entrypoint, " ") + " " + strings.Join(c.Config.Cmd, " "),
State: c.State.Status,
Host: host,
Labels: c.Config.Labels,
Stats: utils.NewRingBuffer[ContainerStat](300), // 300 seconds of stats
Group: group,
Tty: c.Config.Tty,
}
if startedAt, err := time.Parse(time.RFC3339Nano, c.State.StartedAt); err == nil {
utc := startedAt.UTC()
container.StartedAt = &utc
}
if createdAt, err := time.Parse(time.RFC3339Nano, c.Created); err == nil {
utc := createdAt.UTC()
container.Created = utc
}
if c.State.Health != nil {
container.Health = strings.ToLower(c.State.Health.Status)
}
return container
}

55
internal/docker/client_test.go
View File

@@ -183,22 +183,10 @@ func Test_dockerClient_ContainerLogs_error(t *testing.T) {
}
func Test_dockerClient_FindContainer_happy(t *testing.T) {
containers := []types.Container{
{
ID: "abcdefghijklmnopqrst",
Names: []string{"/z_test_container"},
},
{
ID: "1234567890_abcxyzdef",
Names: []string{"/a_test_container"},
},
}
proxy := new(mockedProxy)
proxy.On("ContainerList", mock.Anything, mock.Anything).Return(containers, nil)
state := &types.ContainerState{Status: "running", StartedAt: time.Now().Format(time.RFC3339Nano)}
json := types.ContainerJSON{ContainerJSONBase: &types.ContainerJSONBase{State: state}, Config: &container.Config{Tty: false}}
json := types.ContainerJSON{ContainerJSONBase: &types.ContainerJSONBase{ID: "abcdefghijklmnopqrst", State: state}, Config: &container.Config{Tty: false}}
proxy.On("ContainerInspect", mock.Anything, "abcdefghijkl").Return(json, nil)
client := &httpClient{proxy, filters.NewArgs(), Host{ID: "localhost"}, system.Info{}}
@@ -210,20 +198,10 @@ func Test_dockerClient_FindContainer_happy(t *testing.T) {
proxy.AssertExpectations(t)
}
func Test_dockerClient_FindContainer_error(t *testing.T) {
containers := []types.Container{
{
ID: "abcdefghijklmnopqrst",
Names: []string{"/z_test_container"},
},
{
ID: "1234567890_abcxyzdef",
Names: []string{"/a_test_container"},
},
}
func Test_dockerClient_FindContainer_error(t *testing.T) {
proxy := new(mockedProxy)
proxy.On("ContainerList", mock.Anything, mock.Anything).Return(containers, nil)
proxy.On("ContainerInspect", mock.Anything, "not_valid").Return(types.ContainerJSON{}, errors.New("not found"))
client := &httpClient{proxy, filters.NewArgs(), Host{ID: "localhost"}, system.Info{}}
_, err := client.FindContainer("not_valid")
@@ -233,24 +211,12 @@ func Test_dockerClient_FindContainer_error(t *testing.T) {
}
func Test_dockerClient_ContainerActions_happy(t *testing.T) {
containers := []types.Container{
{
ID: "abcdefghijklmnopqrst",
Names: []string{"/z_test_container"},
},
{
ID: "1234567890_abcxyzdef",
Names: []string{"/a_test_container"},
},
}
proxy := new(mockedProxy)
client := &httpClient{proxy, filters.NewArgs(), Host{ID: "localhost"}, system.Info{}}
state := &types.ContainerState{Status: "running", StartedAt: time.Now().Format(time.RFC3339Nano)}
json := types.ContainerJSON{ContainerJSONBase: &types.ContainerJSONBase{State: state}, Config: &container.Config{Tty: false}}
json := types.ContainerJSON{ContainerJSONBase: &types.ContainerJSONBase{ID: "abcdefghijkl", State: state}, Config: &container.Config{Tty: false}}
proxy.On("ContainerList", mock.Anything, mock.Anything).Return(containers, nil)
proxy.On("ContainerInspect", mock.Anything, "abcdefghijkl").Return(json, nil)
proxy.On("ContainerStart", mock.Anything, "abcdefghijkl", mock.Anything).Return(nil)
proxy.On("ContainerStop", mock.Anything, "abcdefghijkl", mock.Anything).Return(nil)
@@ -272,21 +238,10 @@ func Test_dockerClient_ContainerActions_happy(t *testing.T) {
}
func Test_dockerClient_ContainerActions_error(t *testing.T) {
containers := []types.Container{
{
ID: "abcdefghijklmnopqrst",
Names: []string{"/z_test_container"},
},
{
ID: "1234567890_abcxyzdef",
Names: []string{"/a_test_container"},
},
}
proxy := new(mockedProxy)
client := &httpClient{proxy, filters.NewArgs(), Host{ID: "localhost"}, system.Info{}}
proxy.On("ContainerList", mock.Anything, mock.Anything).Return(containers, nil)
proxy.On("ContainerInspect", mock.Anything, "random-id").Return(types.ContainerJSON{}, errors.New("not found"))
proxy.On("ContainerStart", mock.Anything, mock.Anything, mock.Anything).Return(errors.New("test"))
proxy.On("ContainerStop", mock.Anything, mock.Anything, mock.Anything).Return(errors.New("test"))
proxy.On("ContainerRestart", mock.Anything, mock.Anything, mock.Anything).Return(errors.New("test"))

62
internal/docker/container_store.go
View File

@@ -7,6 +7,7 @@ import (
"sync/atomic"
"github.com/puzpuzpuz/xsync/v3"
"github.com/samber/lo"
lop "github.com/samber/lo/parallel"
log "github.com/sirupsen/logrus"
)
@@ -59,10 +60,23 @@ func (s *ContainerStore) checkConnectivity() error {
return err
} else {
s.containers.Clear()
lop.ForEach(containers, func(c Container, _ int) {
container, _ := s.client.FindContainer(c.ID)
s.containers.Store(c.ID, &container)
for _, c := range containers {
s.containers.Store(c.ID, &c)
}
running := lo.Filter(containers, func(item Container, index int) bool {
return item.State == "running"
})
chunks := lo.Chunk(running, 100)
for _, chunk := range chunks {
lop.ForEach(chunk, func(c Container, _ int) {
container, _ := s.client.FindContainer(c.ID)
s.containers.Store(c.ID, &container)
})
}
}
}
@@ -85,19 +99,14 @@ func (s *ContainerStore) ListContainers() ([]Container, error) {
}
func (s *ContainerStore) FindContainer(id string) (Container, error) {
list, err := s.ListContainers()
if err != nil {
return Container{}, err
}
container, ok := s.containers.Load(id)
for _, c := range list {
if c.ID == id {
return c, nil
}
if ok {
return *container, nil
} else {
log.Warnf("container %s not found in store", id)
return Container{}, ErrContainerNotFound
}
log.Warnf("container %s not found in store", id)
return Container{}, ErrContainerNotFound
}
func (s *ContainerStore) Client() Client {
@@ -150,15 +159,24 @@ func (s *ContainerStore) init() {
switch event.Name {
case "start":
if container, err := s.client.FindContainer(event.ActorID); err == nil {
log.Debugf("container %s started", container.ID)
s.containers.Store(container.ID, &container)
s.newContainerSubscribers.Range(func(c context.Context, containers chan<- Container) bool {
select {
case containers <- container:
case <-c.Done():
}
return true
list, _ := s.client.ListContainers()
// make sure the container is in the list of containers when using filter
valid := lo.ContainsBy(list, func(item Container) bool {
return item.ID == container.ID
})
if valid {
log.Debugf("container %s started", container.ID)
s.containers.Store(container.ID, &container)
s.newContainerSubscribers.Range(func(c context.Context, containers chan<- Container) bool {
select {
case containers <- container:
case <-c.Done():
}
return true
})
}
}
case "destroy":
log.Debugf("container %s destroyed", event.ActorID)

2
internal/docker/types.go
View File

@@ -11,7 +11,6 @@ import (
// Container represents an internal representation of docker containers
type Container struct {
ID string `json:"id"`
Names []string `json:"names"`
Name string `json:"name"`
Image string `json:"image"`
ImageID string `json:"imageId"`
@@ -19,7 +18,6 @@ type Container struct {
Created time.Time `json:"created"`
StartedAt *time.Time `json:"startedAt,omitempty"`
State string `json:"state"`
Status string `json:"status"`
Health string `json:"health,omitempty"`
Host string `json:"host,omitempty"`
Tty bool `json:"-"`

11
internal/support/docker/client_service.go
View File

@@ -71,16 +71,7 @@ func (d *dockerClientService) StreamLogs(ctx context.Context, container docker.C
}
func (d *dockerClientService) FindContainer(id string) (docker.Container, error) {
container, err := d.store.FindContainer(id)
if err != nil {
if err == docker.ErrContainerNotFound {
return d.client.FindContainer(id)
} else {
return docker.Container{}, err
}
}
return container, nil
return d.store.FindContainer(id)
}
func (d *dockerClientService) ContainerAction(container docker.Container, action docker.ContainerAction) error {

2
internal/web/__snapshots__/web.snapshot
View File

@@ -132,7 +132,7 @@ data: []
event: containers-changed
data: [{"id":"1234","names":null,"name":"test","image":"test","imageId":"","command":"","created":"0001-01-01T00:00:00Z","state":"","status":"","stats":[]}]
data: []
event: container-event

4
internal/web/download_test.go
View File

@@ -25,7 +25,7 @@ func Test_handler_download_logs(t *testing.T) {
data := makeMessage("INFO Testing logs...", docker.STDOUT)
mockedClient.On("FindContainer", id).Return(docker.Container{ID: id, Tty: false}, nil).Once()
mockedClient.On("FindContainer", id).Return(docker.Container{ID: id, Tty: false}, nil)
mockedClient.On("ContainerLogsBetweenDates", mock.Anything, id, mock.Anything, mock.Anything, docker.STDOUT).Return(io.NopCloser(bytes.NewReader(data)), nil)
mockedClient.On("Host").Return(docker.Host{
ID: "localhost",
@@ -34,7 +34,7 @@ func Test_handler_download_logs(t *testing.T) {
time.Sleep(1 * time.Second)
})
mockedClient.On("ListContainers").Return([]docker.Container{
{ID: id, Name: "test"},
{ID: id, Name: "test", State: "running"},
}, nil)
handler := createDefaultHandler(mockedClient)

12
internal/web/logs_test.go
View File

@@ -50,7 +50,7 @@ func Test_handler_streamLogs_happy(t *testing.T) {
ID: "localhost",
})
mockedClient.On("ListContainers").Return([]docker.Container{
{ID: id, Name: "test", Host: "localhost"},
{ID: id, Name: "test", Host: "localhost", State: "running"},
}, nil)
mockedClient.On("ContainerEvents", mock.Anything, mock.AnythingOfType("chan<- docker.ContainerEvent")).Return(nil).Run(func(args mock.Arguments) {
time.Sleep(50 * time.Millisecond)
@@ -93,7 +93,7 @@ func Test_handler_streamLogs_happy_with_id(t *testing.T) {
})
mockedClient.On("ListContainers").Return([]docker.Container{
{ID: id, Name: "test", Host: "localhost"},
{ID: id, Name: "test", Host: "localhost", State: "running"},
}, nil)
mockedClient.On("ContainerEvents", mock.Anything, mock.AnythingOfType("chan<- docker.ContainerEvent")).Return(nil).Run(func(args mock.Arguments) {
@@ -132,7 +132,7 @@ func Test_handler_streamLogs_happy_container_stopped(t *testing.T) {
ID: "localhost",
})
mockedClient.On("ListContainers").Return([]docker.Container{
{ID: id, Name: "test", Host: "localhost"},
{ID: id, Name: "test", Host: "localhost", State: "running"},
}, nil)
mockedClient.On("ContainerEvents", mock.Anything, mock.AnythingOfType("chan<- docker.ContainerEvent")).Return(nil)
@@ -201,7 +201,7 @@ func Test_handler_streamLogs_error_reading(t *testing.T) {
ID: "localhost",
})
mockedClient.On("ListContainers").Return([]docker.Container{
{ID: id, Name: "test", Host: "localhost"},
{ID: id, Name: "test", Host: "localhost", State: "running"},
}, nil)
mockedClient.On("ContainerEvents", mock.Anything, mock.AnythingOfType("chan<- docker.ContainerEvent")).Return(nil)
@@ -224,7 +224,7 @@ func Test_handler_streamLogs_error_std(t *testing.T) {
ID: "localhost",
})
mockedClient.On("ListContainers").Return([]docker.Container{
{ID: id, Name: "test", Host: "localhost"},
{ID: id, Name: "test", Host: "localhost", State: "running"},
}, nil)
mockedClient.On("ContainerEvents", mock.Anything, mock.AnythingOfType("chan<- docker.ContainerEvent")).Return(nil).
Run(func(args mock.Arguments) {
@@ -265,7 +265,7 @@ func Test_handler_between_dates(t *testing.T) {
ID: "localhost",
})
mockedClient.On("ListContainers").Return([]docker.Container{
{ID: id, Name: "test", Host: "localhost"},
{ID: id, Name: "test", Host: "localhost", State: "running"},
}, nil)
mockedClient.On("ContainerEvents", mock.Anything, mock.AnythingOfType("chan<- docker.ContainerEvent")).Return(nil)

2
protos/types.proto
View File

@@ -10,7 +10,7 @@ message Container {
string id = 1;
string name = 2;
string image = 3;
string status = 4;
string status = 4; // deprecated
string state = 5;
string ImageId = 6;
google.protobuf.Timestamp created = 7;