From 3a988b8fdcf3339e599d2c14d4faf94d46b297e7 Mon Sep 17 00:00:00 2001
From: Amir Raminfar <findamir@gmail.com>
Date: Thu, 25 Jul 2024 08:01:33 -0700
Subject: [PATCH] feat: updates agents to be more resilient by reconnecting. 
 also adds big performance issues in swarm mode with little updates to the UI.
  (#3145)

---
 .vscode/settings.json                         |   2 +-
 Makefile                                      |   4 +-
 assets/components.d.ts                        |   5 +
 assets/components/HostList.vue                |  37 ++--
 assets/components/HostMenu.vue                |   3 +-
 assets/components/common/HostIcon.vue         |  10 ++
 assets/main.css                               |   1 +
 assets/stores/config.ts                       |   4 +-
 assets/stores/container.ts                    |   9 +-
 assets/stores/hosts.ts                        |  34 ++--
 examples/docker.swarm.yml                     |   2 +-
 go.mod                                        |   2 +-
 go.sum                                        |  16 --
 internal/agent/client.go                      |  13 +-
 internal/agent/client_test.go                 |   3 +-
 internal/agent/pb/types.pb.go                 |  39 +++-
 internal/agent/server.go                      |  36 ++--
 internal/docker/client.go                     |   4 +
 internal/docker/host.go                       |  24 +--
 internal/docker/stats_collector.go            |   4 +-
 internal/support/cli/clients.go               |  23 +--
 internal/support/docker/multi_host_service.go | 141 +++------------
 .../docker/retriable_client_manager.go        | 149 +++++++++++++++
 .../support/docker/swarm_client_manager.go    | 170 ++++++++++++++++++
 internal/web/events.go                        |  19 +-
 internal/web/events_test.go                   |   7 +-
 internal/web/routes_test.go                   |   4 +-
 main.go                                       |  34 +++-
 protos/types.proto                            |   2 +
 29 files changed, 560 insertions(+), 241 deletions(-)
 create mode 100644 assets/components/common/HostIcon.vue
 create mode 100644 internal/support/docker/retriable_client_manager.go
 create mode 100644 internal/support/docker/swarm_client_manager.go

diff --git a/.vscode/settings.json b/.vscode/settings.json
index bac65365..fcee7c4e 100644
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@@ -1,7 +1,7 @@
 {
   "i18n-ally.localesPaths": ["locales"],
   "i18n-ally.keystyle": "nested",
-  "cSpell.words": ["healthcheck", "orderedmap", "stdcopy", "Warnf"],
+  "cSpell.words": ["healthcheck", "orderedmap", "Retriable", "stdcopy", "Warnf"],
   "editor.formatOnSave": true,
   "i18n-ally.extract.autoDetect": true
 }
diff --git a/Makefile b/Makefile
index 5d73e563..4fbd89b4 100644
--- a/Makefile
+++ b/Makefile
@@ -55,5 +55,5 @@ $(GEN_DIR)/%.pb.go: $(PROTO_DIR)/%.proto
 
 .PHONY: push
 push: docker
-	@docker tag amir20/dozzle:latest amir20/dozzle:agent
-	@docker push amir20/dozzle:agent
+	@docker tag amir20/dozzle:latest amir20/dozzle:local-test
+	@docker push amir20/dozzle:local-test
diff --git a/assets/components.d.ts b/assets/components.d.ts
index f577a6a5..be09a019 100644
--- a/assets/components.d.ts
+++ b/assets/components.d.ts
@@ -40,6 +40,7 @@ declare module 'vue' {
     FieldList: typeof import('./components/LogViewer/FieldList.vue')['default']
     FuzzySearchModal: typeof import('./components/FuzzySearchModal.vue')['default']
     GroupedLog: typeof import('./components/GroupedViewer/GroupedLog.vue')['default']
+    HostIcon: typeof import('./components/common/HostIcon.vue')['default']
     HostList: typeof import('./components/HostList.vue')['default']
     HostMenu: typeof import('./components/HostMenu.vue')['default']
     'Ic:sharpKeyboardReturn': typeof import('~icons/ic/sharp-keyboard-return')['default']
@@ -63,9 +64,12 @@ declare module 'vue' {
     'Mdi:chevronRight': typeof import('~icons/mdi/chevron-right')['default']
     'Mdi:close': typeof import('~icons/mdi/close')['default']
     'Mdi:cog': typeof import('~icons/mdi/cog')['default']
+    'Mdi:docker': typeof import('~icons/mdi/docker')['default']
     'Mdi:hamburgerMenu': typeof import('~icons/mdi/hamburger-menu')['default']
+    'Mdi:hexagonMultiple': typeof import('~icons/mdi/hexagon-multiple')['default']
     'Mdi:keyboardEsc': typeof import('~icons/mdi/keyboard-esc')['default']
     'Mdi:magnify': typeof import('~icons/mdi/magnify')['default']
+    'Mdi:satelliteVariant': typeof import('~icons/mdi/satellite-variant')['default']
     MobileMenu: typeof import('./components/common/MobileMenu.vue')['default']
     MultiContainerLog: typeof import('./components/MultiContainerViewer/MultiContainerLog.vue')['default']
     MultiContainerStat: typeof import('./components/LogViewer/MultiContainerStat.vue')['default']
@@ -80,6 +84,7 @@ declare module 'vue' {
     'Ph:computerTower': typeof import('~icons/ph/computer-tower')['default']
     'Ph:controlBold': typeof import('~icons/ph/control-bold')['default']
     'Ph:cpu': typeof import('~icons/ph/cpu')['default']
+    'Ph:globeSimple': typeof import('~icons/ph/globe-simple')['default']
     'Ph:memory': typeof import('~icons/ph/memory')['default']
     'Ph:stack': typeof import('~icons/ph/stack')['default']
     'Ph:stackSimple': typeof import('~icons/ph/stack-simple')['default']
diff --git a/assets/components/HostList.vue b/assets/components/HostList.vue
index 01a5f598..f6616b19 100644
--- a/assets/components/HostList.vue
+++ b/assets/components/HostList.vue
@@ -2,20 +2,37 @@
   <ul class="grid gap-4 md:grid-cols-[repeat(auto-fill,minmax(480px,1fr))]">
     <li v-for="host in hosts" class="card bg-base-lighter">
       <div class="card-body grid auto-cols-auto grid-flow-col justify-between gap-4">
-        <div class="overflow-hidden">
-          <div class="truncate text-xl font-semibold">
-            {{ host.name }} <span class="badge badge-error badge-xs p-1.5" v-if="!host.available">offline</span>
+        <div class="flex flex-col gap-2 overflow-hidden">
+          <div class="flex items-center gap-1 truncate text-xl font-semibold">
+            <HostIcon :type="host.type" />
+            {{ host.name }}
+
+            <span class="badge badge-error badge-xs gap-2 p-2" v-if="!host.available">
+              <carbon:warning />
+              offline
+            </span>
+            <span
+              class="badge badge-success badge-xs gap-2 p-2"
+              :class="{ 'badge-warning': config.version != host.agentVersion }"
+              v-else-if="host.type == 'agent'"
+              title="Dozzle Agent"
+            >
+              {{ host.agentVersion }}
+            </span>
           </div>
-          <ul class="flex flex-row gap-2 text-sm md:gap-4">
-            <li><ph:cpu class="inline-block" /> {{ host.nCPU }} <span class="mobile-hidden">CPUs</span></li>
-            <li>
-              <ph:memory class="inline-block" /> {{ formatBytes(host.memTotal) }}
+          <ul class="flex flex-row gap-2 text-sm md:gap-3">
+            <li class="flex items-center gap-1"><ph:cpu /> {{ host.nCPU }} <span class="mobile-hidden">CPUs</span></li>
+            <li class="flex items-center gap-1">
+              <ph:memory /> {{ formatBytes(host.memTotal) }}
               <span class="mobile-hidden">total</span>
             </li>
           </ul>
-          <div class="text-sm">
-            <octicon:container-24 class="inline-block" /> {{ $t("label.container", hostContainers[host.id]?.length) }}
-          </div>
+          <ul class="flex flex-row gap-2 text-sm md:gap-3">
+            <li class="flex items-center gap-1">
+              <octicon:container-24 class="inline-block" /> {{ $t("label.container", hostContainers[host.id]?.length) }}
+            </li>
+            <li class="flex items-center gap-1"><mdi:docker class="inline-block" /> {{ host.dockerVersion }}</li>
+          </ul>
         </div>
 
         <div class="flex flex-row gap-4 md:gap-8" v-if="weightedStats[host.id]">
diff --git a/assets/components/HostMenu.vue b/assets/components/HostMenu.vue
index df5889a6..2e88fb1c 100644
--- a/assets/components/HostMenu.vue
+++ b/assets/components/HostMenu.vue
@@ -15,7 +15,7 @@
       <ul class="menu p-0">
         <li v-for="host in hosts" :key="host.id">
           <a @click.prevent="setHost(host.id)" :class="{ 'pointer-events-none text-base-content/50': !host.available }">
-            <ph:computer-tower />
+            <HostIcon :type="host.type" />
             {{ host.name }}
             <span class="badge badge-error badge-xs p-1.5" v-if="!host.available">offline</span>
           </a>
@@ -25,7 +25,6 @@
     <template #right>
       <ul class="containers menu p-0 [&_li.menu-title]:px-0">
         <li v-for="{ label, containers, icon } in menuItems" :key="label">
-          <!-- @vue-ignore -->
           <details :open="!collapsedGroups.has(label)" @toggle="updateCollapsedGroups($event, label)">
             <summary class="font-light text-base-content/80">
               <component :is="icon" />
diff --git a/assets/components/common/HostIcon.vue b/assets/components/common/HostIcon.vue
new file mode 100644
index 00000000..0838bb7a
--- /dev/null
+++ b/assets/components/common/HostIcon.vue
@@ -0,0 +1,10 @@
+<template>
+  <mdi:satellite-variant v-if="type == 'agent'" />
+  <ph:globe-simple v-else-if="type == 'remote'" />
+  <mdi:hexagon-multiple v-else-if="type == 'swarm'" />
+  <ph:computer-tower v-else />
+</template>
+<script setup lang="ts">
+import { Host } from "@/stores/hosts";
+const { type } = defineProps<{ type: Host["type"] }>();
+</script>
diff --git a/assets/main.css b/assets/main.css
index c9b0a420..c2b23003 100644
--- a/assets/main.css
+++ b/assets/main.css
@@ -17,6 +17,7 @@
     --in: 65% 0.171 249.5;
     --inc: 100% 0 0;
     --er: 64% 0.218 28.85;
+    --su: 56% 0.119722 164.12;
     --erc: 100% 0 0;
     --wa: 70% 0.186 48.13;
     --wac: 100% 0 0;
diff --git a/assets/stores/config.ts b/assets/stores/config.ts
index 194c9f70..3ccfa9a4 100644
--- a/assets/stores/config.ts
+++ b/assets/stores/config.ts
@@ -3,14 +3,12 @@ import { Host } from "@/stores/hosts";
 
 const text = document.querySelector("script#config__json")?.textContent || "{}";
 
-type HostWithoutAvailable = Omit<Host, "available">;
-
 export interface Config {
   version: string;
   base: string;
   maxLogs: number;
   hostname: string;
-  hosts: HostWithoutAvailable[];
+  hosts: Host[];
   authProvider: "simple" | "none" | "forward-proxy";
   enableActions: boolean;
   user?: {
diff --git a/assets/stores/container.ts b/assets/stores/container.ts
index 4491d215..535d26bf 100644
--- a/assets/stores/container.ts
+++ b/assets/stores/container.ts
@@ -3,9 +3,10 @@ import { Ref, UnwrapNestedRefs } from "vue";
 import type { ContainerHealth, ContainerJson, ContainerStat } from "@/types/Container";
 import { Container } from "@/models/Container";
 import i18n from "@/modules/i18n";
+import { Host } from "./hosts";
 
 const { showToast, removeToast } = useToast();
-const { markHostAvailable } = useHosts();
+const { updateHost } = useHosts();
 // @ts-ignore
 const { t } = i18n.global;
 
@@ -74,9 +75,9 @@ export const useContainerStore = defineStore("container", () => {
       }
     });
 
-    es.addEventListener("host-unavailable", (e) => {
-      const hostId = (e as MessageEvent).data;
-      markHostAvailable(hostId, false);
+    es.addEventListener("update-host", (e) => {
+      const host = JSON.parse((e as MessageEvent).data) as Host;
+      updateHost(host);
     });
 
     es.addEventListener("container-health", (e) => {
diff --git a/assets/stores/hosts.ts b/assets/stores/hosts.ts
index acd23a23..58d105c3 100644
--- a/assets/stores/hosts.ts
+++ b/assets/stores/hosts.ts
@@ -1,27 +1,35 @@
 export type Host = {
-  name: string;
   id: string;
+  name: string;
   nCPU: number;
   memTotal: number;
+  type: "agent" | "local" | "remote" | "swarm";
+  endpoint: string;
   available: boolean;
+  dockerVersion: string;
+  agentVersion: string;
 };
-const hosts = computed(() =>
-  config.hosts.reduce(
-    (acc, item) => {
-      acc[item.id] = { ...item, available: true };
-      return acc;
-    },
-    {} as Record<string, Host>,
-  ),
-);
 
-const markHostAvailable = (id: string, available: boolean) => {
-  hosts.value[id].available = available;
+const hosts = ref(
+  config.hosts
+    .sort((a, b) => a.name.localeCompare(b.name))
+    .reduce(
+      (acc, item) => {
+        acc[item.id] = item;
+        return acc;
+      },
+      {} as Record<string, Host>,
+    ),
+);
+const updateHost = (host: Host) => {
+  delete hosts.value[host.endpoint];
+  hosts.value[host.id] = host;
+  return host;
 };
 
 export function useHosts() {
   return {
     hosts,
-    markHostAvailable,
+    updateHost,
   };
 }
diff --git a/examples/docker.swarm.yml b/examples/docker.swarm.yml
index 96907905..813c1d8d 100644
--- a/examples/docker.swarm.yml
+++ b/examples/docker.swarm.yml
@@ -1,6 +1,6 @@
 services:
   dozzle:
-    image: amir20/dozzle:pr-3097
+    image: amir20/dozzle:local-test
     environment:
       - DOZZLE_LEVEL=debug
       - DOZZLE_MODE=swarm
diff --git a/go.mod b/go.mod
index d975f16a..53c1b769 100644
--- a/go.mod
+++ b/go.mod
@@ -26,7 +26,6 @@ require (
 
 require (
 	github.com/PuerkitoBio/goquery v1.9.2
-	github.com/cenkalti/backoff/v4 v4.3.0
 	github.com/go-chi/chi/v5 v5.1.0
 	github.com/go-chi/jwtauth/v5 v5.3.1
 	github.com/goccy/go-json v0.10.3
@@ -43,6 +42,7 @@ require (
 	github.com/andybalholm/cascadia v1.3.2 // indirect
 	github.com/bahlo/generic-list-go v0.2.0 // indirect
 	github.com/buger/jsonparser v1.1.1 // indirect
+	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
 	github.com/containerd/log v0.1.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect
diff --git a/go.sum b/go.sum
index 4b2c36cb..b54c7527 100644
--- a/go.sum
+++ b/go.sum
@@ -28,10 +28,6 @@ github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 h1:rpfIENRNNilwHwZeG5+P150SMrnN
 github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0=
 github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
 github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
-github.com/docker/docker v27.0.3+incompatible h1:aBGI9TeQ4MPlhquTQKq9XbK79rKFVwXNUAYz9aXyEBE=
-github.com/docker/docker v27.0.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
-github.com/docker/docker v27.1.0+incompatible h1:rEHVQc4GZ0MIQKifQPHSFGV/dVgaZafgRf8fCPtDYBs=
-github.com/docker/docker v27.1.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/docker v27.1.1+incompatible h1:hO/M4MtV36kzKldqnA37IWhebRA+LnqqcqDja6kVaKY=
 github.com/docker/docker v27.1.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c=
@@ -102,20 +98,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/puzpuzpuz/xsync/v3 v3.2.0 h1:9AzuUeF88YC5bK8u2vEG1Fpvu4wgpM1wfPIExfaaDxQ=
-github.com/puzpuzpuz/xsync/v3 v3.2.0/go.mod h1:VjzYrABPabuM4KyBh1Ftq6u8nhwY5tBPKP9jpmh0nnA=
-github.com/puzpuzpuz/xsync/v3 v3.3.0 h1:vX9b3zg+gIivLYwoHav6CoI9PylvXqdfhr/nFyu8O5o=
-github.com/puzpuzpuz/xsync/v3 v3.3.0/go.mod h1:VjzYrABPabuM4KyBh1Ftq6u8nhwY5tBPKP9jpmh0nnA=
-github.com/puzpuzpuz/xsync/v3 v3.3.1 h1:vZPJk3OOfoaSjy3cdTX3BZxhDCUVp9SqdHnd+ilGlbQ=
-github.com/puzpuzpuz/xsync/v3 v3.3.1/go.mod h1:VjzYrABPabuM4KyBh1Ftq6u8nhwY5tBPKP9jpmh0nnA=
 github.com/puzpuzpuz/xsync/v3 v3.4.0 h1:DuVBAdXuGFHv8adVXjWWZ63pJq+NRXOWVXlKDBZ+mJ4=
 github.com/puzpuzpuz/xsync/v3 v3.4.0/go.mod h1:VjzYrABPabuM4KyBh1Ftq6u8nhwY5tBPKP9jpmh0nnA=
-github.com/samber/lo v1.43.0 h1:ts0VhPi8+ZQZFVLv/2Vkgt2Cds05FM2v3Enmv+YMBtg=
-github.com/samber/lo v1.43.0/go.mod h1:w7R6fO7h2lrnx/s0bWcZ55vXJI89p5UPM6+kyDL373E=
-github.com/samber/lo v1.44.0 h1:5il56KxRE+GHsm1IR+sZ/6J42NODigFiqCWpSc2dybA=
-github.com/samber/lo v1.44.0/go.mod h1:RmDH9Ct32Qy3gduHQuKJ3gW1fMHAnE/fAzQuf6He5cU=
-github.com/samber/lo v1.45.0 h1:TPK85Y30Lv9Jh8s3TrJeA94u1hwcbFA9JObx/vT6lYU=
-github.com/samber/lo v1.45.0/go.mod h1:RmDH9Ct32Qy3gduHQuKJ3gW1fMHAnE/fAzQuf6He5cU=
 github.com/samber/lo v1.46.0 h1:w8G+oaCPgz1PoCJztqymCFaKwXt+5cCXn51uPxExFfQ=
 github.com/samber/lo v1.46.0/go.mod h1:RmDH9Ct32Qy3gduHQuKJ3gW1fMHAnE/fAzQuf6He5cU=
 github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys=
diff --git a/internal/agent/client.go b/internal/agent/client.go
index 44a0e9a2..404f9d45 100644
--- a/internal/agent/client.go
+++ b/internal/agent/client.go
@@ -62,11 +62,14 @@ func NewClient(endpoint string, certificates tls.Certificate, opts ...grpc.DialO
 		conn:   conn,
 
 		host: docker.Host{
-			ID:       info.Host.Id,
-			Name:     info.Host.Name,
-			NCPU:     int(info.Host.CpuCores),
-			MemTotal: int64(info.Host.Memory),
-			Endpoint: endpoint,
+			ID:            info.Host.Id,
+			Name:          info.Host.Name,
+			NCPU:          int(info.Host.CpuCores),
+			MemTotal:      int64(info.Host.Memory),
+			Endpoint:      endpoint,
+			Type:          "agent",
+			DockerVersion: info.Host.DockerVersion,
+			AgentVersion:  info.Host.AgentVersion,
 		},
 	}, nil
 }
diff --git a/internal/agent/client_test.go b/internal/agent/client_test.go
index ccdc2ea4..16155781 100644
--- a/internal/agent/client_test.go
+++ b/internal/agent/client_test.go
@@ -130,7 +130,8 @@ func init() {
 		Stats: utils.NewRingBuffer[docker.ContainerStat](300),
 	}, nil)
 
-	go RunServer(client, certs, lis)
+	server := NewServer(client, certs, "test")
+	go server.Serve(lis)
 }
 
 func bufDialer(ctx context.Context, address string) (net.Conn, error) {
diff --git a/internal/agent/pb/types.pb.go b/internal/agent/pb/types.pb.go
index 1f9abc8b..155f4698 100644
--- a/internal/agent/pb/types.pb.go
+++ b/internal/agent/pb/types.pb.go
@@ -30,7 +30,7 @@ type Container struct {
 	Id      string                 `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
 	Name    string                 `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"`
 	Image   string                 `protobuf:"bytes,3,opt,name=image,proto3" json:"image,omitempty"`
-	Status  string                 `protobuf:"bytes,4,opt,name=status,proto3" json:"status,omitempty"`
+	Status  string                 `protobuf:"bytes,4,opt,name=status,proto3" json:"status,omitempty"` // deprecated
 	State   string                 `protobuf:"bytes,5,opt,name=state,proto3" json:"state,omitempty"`
 	ImageId string                 `protobuf:"bytes,6,opt,name=ImageId,proto3" json:"ImageId,omitempty"`
 	Created *timestamppb.Timestamp `protobuf:"bytes,7,opt,name=created,proto3" json:"created,omitempty"`
@@ -519,6 +519,8 @@ type Host struct {
 	OsType          string            `protobuf:"bytes,8,opt,name=osType,proto3" json:"osType,omitempty"`
 	CpuCores        uint32            `protobuf:"varint,9,opt,name=cpuCores,proto3" json:"cpuCores,omitempty"`
 	Memory          uint64            `protobuf:"varint,10,opt,name=memory,proto3" json:"memory,omitempty"`
+	AgentVersion    string            `protobuf:"bytes,11,opt,name=agentVersion,proto3" json:"agentVersion,omitempty"`
+	DockerVersion   string            `protobuf:"bytes,12,opt,name=dockerVersion,proto3" json:"dockerVersion,omitempty"`
 }
 
 func (x *Host) Reset() {
@@ -623,6 +625,20 @@ func (x *Host) GetMemory() uint64 {
 	return 0
 }
 
+func (x *Host) GetAgentVersion() string {
+	if x != nil {
+		return x.AgentVersion
+	}
+	return ""
+}
+
+func (x *Host) GetDockerVersion() string {
+	if x != nil {
+		return x.DockerVersion
+	}
+	return ""
+}
+
 var File_types_proto protoreflect.FileDescriptor
 
 var file_types_proto_rawDesc = []byte{
@@ -698,7 +714,7 @@ var file_types_proto_rawDesc = []byte{
 	0x09, 0x52, 0x07, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x49, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61,
 	0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12,
 	0x0a, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x6f,
-	0x73, 0x74, 0x22, 0xe5, 0x02, 0x0a, 0x04, 0x48, 0x6f, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69,
+	0x73, 0x74, 0x22, 0xaf, 0x03, 0x0a, 0x04, 0x48, 0x6f, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69,
 	0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e,
 	0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12,
 	0x20, 0x0a, 0x0b, 0x6e, 0x6f, 0x64, 0x65, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x03,
@@ -716,13 +732,18 @@ var file_types_proto_rawDesc = []byte{
 	0x01, 0x28, 0x09, 0x52, 0x06, 0x6f, 0x73, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x63,
 	0x70, 0x75, 0x43, 0x6f, 0x72, 0x65, 0x73, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x63,
 	0x70, 0x75, 0x43, 0x6f, 0x72, 0x65, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x6d, 0x65, 0x6d, 0x6f, 0x72,
-	0x79, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x1a,
-	0x39, 0x0a, 0x0b, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10,
-	0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79,
-	0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52,
-	0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x42, 0x13, 0x5a, 0x11, 0x69, 0x6e,
-	0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2f, 0x70, 0x62, 0x62,
-	0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x79, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x12,
+	0x22, 0x0a, 0x0c, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18,
+	0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x56, 0x65, 0x72, 0x73,
+	0x69, 0x6f, 0x6e, 0x12, 0x24, 0x0a, 0x0d, 0x64, 0x6f, 0x63, 0x6b, 0x65, 0x72, 0x56, 0x65, 0x72,
+	0x73, 0x69, 0x6f, 0x6e, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x64, 0x6f, 0x63, 0x6b,
+	0x65, 0x72, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x1a, 0x39, 0x0a, 0x0b, 0x4c, 0x61, 0x62,
+	0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18,
+	0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61,
+	0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65,
+	0x3a, 0x02, 0x38, 0x01, 0x42, 0x13, 0x5a, 0x11, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c,
+	0x2f, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f,
+	0x33,
 }
 
 var (
diff --git a/internal/agent/server.go b/internal/agent/server.go
index 140c96b0..f5f02d00 100644
--- a/internal/agent/server.go
+++ b/internal/agent/server.go
@@ -6,7 +6,6 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/json"
-	"net"
 	"time"
 
 	"github.com/amir20/dozzle/internal/agent/pb"
@@ -24,16 +23,19 @@ import (
 )
 
 type server struct {
-	client docker.Client
-	store  *docker.ContainerStore
+	client  docker.Client
+	store   *docker.ContainerStore
+	version string
 
 	pb.UnimplementedAgentServiceServer
 }
 
-func NewServer(client docker.Client) pb.AgentServiceServer {
+func newServer(client docker.Client, dozzleVersion string) pb.AgentServiceServer {
 	return &server{
-		client: client,
-		store:  docker.NewContainerStore(context.Background(), client),
+		client:  client,
+		version: dozzleVersion,
+
+		store: docker.NewContainerStore(context.Background(), client),
 	}
 }
 
@@ -243,10 +245,12 @@ func (s *server) HostInfo(ctx context.Context, in *pb.HostInfoRequest) (*pb.Host
 	host := s.client.Host()
 	return &pb.HostInfoResponse{
 		Host: &pb.Host{
-			Id:       host.ID,
-			Name:     host.Name,
-			CpuCores: uint32(host.NCPU),
-			Memory:   uint64(host.MemTotal),
+			Id:            host.ID,
+			Name:          host.Name,
+			CpuCores:      uint32(host.NCPU),
+			Memory:        uint64(host.MemTotal),
+			DockerVersion: host.DockerVersion,
+			AgentVersion:  s.version,
 		},
 	}, nil
 }
@@ -281,7 +285,7 @@ func (s *server) StreamContainerStarted(in *pb.StreamContainerStartedRequest, ou
 	}
 }
 
-func RunServer(client docker.Client, certificates tls.Certificate, listener net.Listener) {
+func NewServer(client docker.Client, certificates tls.Certificate, dozzleVersion string) *grpc.Server {
 	caCertPool := x509.NewCertPool()
 	c, err := x509.ParseCertificate(certificates.Certificate[0])
 	if err != nil {
@@ -300,15 +304,9 @@ func RunServer(client docker.Client, certificates tls.Certificate, listener net.
 	creds := credentials.NewTLS(tlsConfig)
 
 	grpcServer := grpc.NewServer(grpc.Creds(creds))
-	pb.RegisterAgentServiceServer(grpcServer, NewServer(client))
+	pb.RegisterAgentServiceServer(grpcServer, newServer(client, dozzleVersion))
 
-	if err != nil {
-		log.Fatalf("failed to listen: %v", err)
-	}
-	log.Infof("gRPC server listening on %s", listener.Addr().String())
-	if err := grpcServer.Serve(listener); err != nil {
-		log.Fatalf("failed to serve: %v", err)
-	}
+	return grpcServer
 }
 
 func logEventToPb(event *docker.LogEvent) *pb.LogEvent {
diff --git a/internal/docker/client.go b/internal/docker/client.go
index cfaaf551..b835ba9c 100644
--- a/internal/docker/client.go
+++ b/internal/docker/client.go
@@ -89,6 +89,7 @@ func NewClient(cli DockerCLI, filters filters.Args, host Host) Client {
 
 	host.NCPU = info.NCPU
 	host.MemTotal = info.MemTotal
+	host.DockerVersion = info.ServerVersion
 
 	return &httpClient{
 		cli:     cli,
@@ -130,6 +131,7 @@ func NewLocalClient(f map[string][]string, hostname string) (Client, error) {
 		MemTotal: info.MemTotal,
 		NCPU:     info.NCPU,
 		Endpoint: "local",
+		Type:     "local",
 	}
 
 	if hostname != "" {
@@ -172,6 +174,8 @@ func NewRemoteClient(f map[string][]string, host Host) (Client, error) {
 		return nil, err
 	}
 
+	host.Type = "remote"
+
 	return NewClient(cli, filterArgs, host), nil
 }
 
diff --git a/internal/docker/host.go b/internal/docker/host.go
index bdbd9acb..8fe0153a 100644
--- a/internal/docker/host.go
+++ b/internal/docker/host.go
@@ -11,16 +11,20 @@ import (
 )
 
 type Host struct {
-	Name       string   `json:"name"`
-	ID         string   `json:"id"`
-	URL        *url.URL `json:"-"`
-	CertPath   string   `json:"-"`
-	CACertPath string   `json:"-"`
-	KeyPath    string   `json:"-"`
-	ValidCerts bool     `json:"-"`
-	NCPU       int      `json:"nCPU"`
-	MemTotal   int64    `json:"memTotal"`
-	Endpoint   string   `json:"endpoint"`
+	Name          string   `json:"name"`
+	ID            string   `json:"id"`
+	URL           *url.URL `json:"-"`
+	CertPath      string   `json:"-"`
+	CACertPath    string   `json:"-"`
+	KeyPath       string   `json:"-"`
+	ValidCerts    bool     `json:"-"`
+	NCPU          int      `json:"nCPU"`
+	MemTotal      int64    `json:"memTotal"`
+	Endpoint      string   `json:"endpoint"`
+	DockerVersion string   `json:"dockerVersion"`
+	AgentVersion  string   `json:"agentVersion,omitempty"`
+	Type          string   `json:"type"`
+	Available     bool     `json:"available"`
 }
 
 func (h Host) String() string {
diff --git a/internal/docker/stats_collector.go b/internal/docker/stats_collector.go
index 26fdf306..9516097c 100644
--- a/internal/docker/stats_collector.go
+++ b/internal/docker/stats_collector.go
@@ -56,7 +56,7 @@ func (c *StatsCollector) Stop() {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	if c.totalStarted.Add(-1) == 0 {
-		log.Debugf("scheduled to stop container stats collector %s", c.client.Host())
+		log.Tracef("scheduled to stop container stats collector %s", c.client.Host())
 		c.timer = time.AfterFunc(timeToStop, func() {
 			c.forceStop()
 		})
@@ -66,7 +66,7 @@ func (c *StatsCollector) Stop() {
 func (c *StatsCollector) reset() {
 	c.mu.Lock()
 	defer c.mu.Unlock()
-	log.Debugf("resetting timer for container stats collector %s", c.client.Host())
+	log.Tracef("resetting timer for container stats collector %s", c.client.Host())
 	if c.timer != nil {
 		c.timer.Stop()
 	}
diff --git a/internal/support/cli/clients.go b/internal/support/cli/clients.go
index f8ef960f..3baff59d 100644
--- a/internal/support/cli/clients.go
+++ b/internal/support/cli/clients.go
@@ -3,13 +3,12 @@ package cli
 import (
 	"embed"
 
-	"github.com/amir20/dozzle/internal/agent"
 	"github.com/amir20/dozzle/internal/docker"
 	docker_support "github.com/amir20/dozzle/internal/support/docker"
 	log "github.com/sirupsen/logrus"
 )
 
-func CreateMultiHostService(embededCerts embed.FS, args Args) *docker_support.MultiHostService {
+func CreateMultiHostService(embeddedCerts embed.FS, args Args) *docker_support.MultiHostService {
 	var clients []docker_support.ClientService
 	if len(args.RemoteHost) > 0 {
 		log.Warnf(`Remote host flag is deprecated and will be removed in future versions. Agents will replace remote hosts as a safer and performant option. See https://github.com/amir20/dozzle/issues/3066 for discussion.`)
@@ -33,18 +32,6 @@ func CreateMultiHostService(embededCerts embed.FS, args Args) *docker_support.Mu
 			log.Warnf("Could not create client for %s: %s", host.ID, err)
 		}
 	}
-	certs, err := ReadCertificates(embededCerts)
-	if err != nil {
-		log.Fatalf("Could not read certificates: %v", err)
-	}
-	for _, remoteAgent := range args.RemoteAgent {
-		client, err := agent.NewClient(remoteAgent, certs)
-		if err != nil {
-			log.Warnf("Could not connect to remote agent %s: %s", remoteAgent, err)
-			continue
-		}
-		clients = append(clients, docker_support.NewAgentService(client))
-	}
 
 	localClient, err := docker.NewLocalClient(args.Filter, args.Hostname)
 	if err == nil {
@@ -59,5 +46,11 @@ func CreateMultiHostService(embededCerts embed.FS, args Args) *docker_support.Mu
 		}
 	}
 
-	return docker_support.NewMultiHostService(clients)
+	certs, err := ReadCertificates(embeddedCerts)
+	if err != nil {
+		log.Fatalf("Could not read certificates: %v", err)
+	}
+
+	clientManager := docker_support.NewRetriableClientManager(args.RemoteAgent, certs, clients...)
+	return docker_support.NewMultiHostService(clientManager)
 }
diff --git a/internal/support/docker/multi_host_service.go b/internal/support/docker/multi_host_service.go
index 04a26e36..0878d482 100644
--- a/internal/support/docker/multi_host_service.go
+++ b/internal/support/docker/multi_host_service.go
@@ -2,15 +2,10 @@ package docker_support
 
 import (
 	"context"
-	"crypto/tls"
 	"fmt"
-	"net"
 
-	"github.com/amir20/dozzle/internal/agent"
 	"github.com/amir20/dozzle/internal/docker"
 	log "github.com/sirupsen/logrus"
-
-	"github.com/cenkalti/backoff/v4"
 )
 
 type ContainerFilter = func(*docker.Container) bool
@@ -24,106 +19,31 @@ func (h *HostUnavailableError) Error() string {
 	return fmt.Sprintf("host %s unavailable: %v", h.Host.ID, h.Err)
 }
 
+type ClientManager interface {
+	Find(id string) (ClientService, bool)
+	List() []ClientService
+	RetryAndList() ([]ClientService, []error)
+	Subscribe(ctx context.Context, channel chan<- docker.Host)
+	Hosts() []docker.Host
+}
+
 type MultiHostService struct {
-	clients   map[string]ClientService
+	manager   ClientManager
 	SwarmMode bool
 }
 
-func NewMultiHostService(clients []ClientService) *MultiHostService {
+func NewMultiHostService(manager ClientManager) *MultiHostService {
 	m := &MultiHostService{
-		clients: make(map[string]ClientService),
+		manager: manager,
 	}
 
-	for _, client := range clients {
-		if _, ok := m.clients[client.Host().ID]; ok {
-			log.Warnf("duplicate host %s found, skipping", client.Host())
-			continue
-		} else {
-			log.Debugf("found a new host %s", client.Host())
-		}
-		m.clients[client.Host().ID] = client
-	}
+	log.Debugf("created multi host service manager %s", manager)
 
 	return m
 }
 
-func NewSwarmService(localClient docker.Client, certificates tls.Certificate) *MultiHostService {
-	m := &MultiHostService{
-		clients:   make(map[string]ClientService),
-		SwarmMode: true,
-	}
-
-	localService := NewDockerClientService(localClient)
-	m.clients[localClient.Host().ID] = localService
-
-	discover := func() {
-		ips, err := net.LookupIP("tasks.dozzle")
-		if err != nil {
-			log.Fatalf("error looking up swarm services: %v", err)
-		}
-
-		found := 0
-		replaced := 0
-		for _, ip := range ips {
-			clientAgent, err := agent.NewClient(ip.String()+":7007", certificates)
-			if err != nil {
-				log.Warnf("error creating client for %s: %v", ip, err)
-				continue
-			}
-
-			if clientAgent.Host().ID == localClient.Host().ID {
-				closeAgent(clientAgent)
-				continue
-			}
-
-			service := NewAgentService(clientAgent)
-			if existing, ok := m.clients[service.Host().ID]; !ok {
-				log.Debugf("adding swarm service %s", service.Host().ID)
-				m.clients[service.Host().ID] = service
-				found++
-			} else if existing.Host().Endpoint != service.Host().Endpoint {
-				log.Debugf("swarm service %s already exists with different endpoint %s and old value %s", service.Host().ID, service.Host().Endpoint, existing.Host().Endpoint)
-				delete(m.clients, existing.Host().ID)
-				m.clients[service.Host().ID] = service
-				replaced++
-				if existingAgent, ok := existing.(*agentService); ok {
-					closeAgent(existingAgent.client)
-				}
-			} else {
-				closeAgent(clientAgent)
-			}
-		}
-
-		if found > 0 {
-			log.Infof("found %d new dozzle replicas", found)
-		}
-		if replaced > 0 {
-			log.Infof("replaced %d dozzle replicas", replaced)
-		}
-	}
-
-	go func() {
-		ticker := backoff.NewTicker(backoff.NewExponentialBackOff(
-			backoff.WithMaxElapsedTime(0)),
-		)
-		for range ticker.C {
-			log.Tracef("discovering swarm services")
-			discover()
-		}
-	}()
-
-	return m
-}
-
-func closeAgent(agent *agent.Client) {
-	log.Tracef("closing agent %s", agent.Host())
-	if err := agent.Close(); err != nil {
-		log.Warnf("error closing agent: %v", err)
-	}
-}
-
 func (m *MultiHostService) FindContainer(host string, id string) (*containerService, error) {
-	client, ok := m.clients[host]
+	client, ok := m.manager.Find(host)
 	if !ok {
 		return nil, fmt.Errorf("host %s not found", host)
 	}
@@ -140,7 +60,7 @@ func (m *MultiHostService) FindContainer(host string, id string) (*containerServ
 }
 
 func (m *MultiHostService) ListContainersForHost(host string) ([]docker.Container, error) {
-	client, ok := m.clients[host]
+	client, ok := m.manager.Find(host)
 	if !ok {
 		return nil, fmt.Errorf("host %s not found", host)
 	}
@@ -150,13 +70,15 @@ func (m *MultiHostService) ListContainersForHost(host string) ([]docker.Containe
 
 func (m *MultiHostService) ListAllContainers() ([]docker.Container, []error) {
 	containers := make([]docker.Container, 0)
-	var errors []error
+	clients, errors := m.manager.RetryAndList()
 
-	for _, client := range m.clients {
+	for _, client := range clients {
 		list, err := client.ListContainers()
 		if err != nil {
 			log.Debugf("error listing containers for host %s: %v", client.Host().ID, err)
-			errors = append(errors, &HostUnavailableError{Host: client.Host(), Err: err})
+			host := client.Host()
+			host.Available = false
+			errors = append(errors, &HostUnavailableError{Host: host, Err: err})
 			continue
 		}
 
@@ -178,7 +100,7 @@ func (m *MultiHostService) ListAllContainersFiltered(filter ContainerFilter) ([]
 }
 
 func (m *MultiHostService) SubscribeEventsAndStats(ctx context.Context, events chan<- docker.ContainerEvent, stats chan<- docker.ContainerStat) {
-	for _, client := range m.clients {
+	for _, client := range m.manager.List() {
 		client.SubscribeEvents(ctx, events)
 		client.SubscribeStats(ctx, stats)
 	}
@@ -186,7 +108,7 @@ func (m *MultiHostService) SubscribeEventsAndStats(ctx context.Context, events c
 
 func (m *MultiHostService) SubscribeContainersStarted(ctx context.Context, containers chan<- docker.Container, filter ContainerFilter) {
 	newContainers := make(chan docker.Container)
-	for _, client := range m.clients {
+	for _, client := range m.manager.List() {
 		client.SubscribeContainersStarted(ctx, newContainers)
 	}
 	go func() {
@@ -208,27 +130,22 @@ func (m *MultiHostService) SubscribeContainersStarted(ctx context.Context, conta
 }
 
 func (m *MultiHostService) TotalClients() int {
-	return len(m.clients)
+	return len(m.manager.List())
 }
 
 func (m *MultiHostService) Hosts() []docker.Host {
-	hosts := make([]docker.Host, 0, len(m.clients))
-	for _, client := range m.clients {
-		hosts = append(hosts, client.Host())
-	}
-
-	return hosts
+	return m.manager.Hosts()
 }
 
 func (m *MultiHostService) LocalHost() (docker.Host, error) {
-	host := docker.Host{}
-
 	for _, host := range m.Hosts() {
-		if host.Endpoint == "local" {
-
+		if host.Type == "local" {
 			return host, nil
 		}
 	}
-
-	return host, fmt.Errorf("local host not found")
+	return docker.Host{}, fmt.Errorf("local host not found")
+}
+
+func (m *MultiHostService) SubscribeAvailableHosts(ctx context.Context, hosts chan<- docker.Host) {
+	m.manager.Subscribe(ctx, hosts)
 }
diff --git a/internal/support/docker/retriable_client_manager.go b/internal/support/docker/retriable_client_manager.go
new file mode 100644
index 00000000..474b05dd
--- /dev/null
+++ b/internal/support/docker/retriable_client_manager.go
@@ -0,0 +1,149 @@
+package docker_support
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"sync"
+
+	"github.com/amir20/dozzle/internal/agent"
+	"github.com/amir20/dozzle/internal/docker"
+	"github.com/puzpuzpuz/xsync/v3"
+
+	log "github.com/sirupsen/logrus"
+)
+
+type RetriableClientManager struct {
+	clients      map[string]ClientService
+	failedAgents []string
+	certs        tls.Certificate
+	mu           sync.RWMutex
+	subscribers  *xsync.MapOf[context.Context, chan<- docker.Host]
+}
+
+func NewRetriableClientManager(agents []string, certs tls.Certificate, clients ...ClientService) *RetriableClientManager {
+	log.Debugf("creating retriable client manager with %d clients and %d agents", len(clients), len(agents))
+
+	clientMap := make(map[string]ClientService)
+	for _, client := range clients {
+		if _, ok := clientMap[client.Host().ID]; ok {
+			log.Warnf("duplicate client found for host %s", client.Host().ID)
+		} else {
+			clientMap[client.Host().ID] = client
+		}
+	}
+
+	failed := make([]string, 0)
+	for _, endpoint := range agents {
+		if agent, err := agent.NewClient(endpoint, certs); err == nil {
+			if _, ok := clientMap[agent.Host().ID]; ok {
+				log.Warnf("duplicate client found for host %s", agent.Host().ID)
+			} else {
+				clientMap[agent.Host().ID] = NewAgentService(agent)
+			}
+		} else {
+			log.Warnf("error creating agent client for %s: %v", endpoint, err)
+			failed = append(failed, endpoint)
+		}
+	}
+
+	return &RetriableClientManager{
+		clients:      clientMap,
+		failedAgents: failed,
+		certs:        certs,
+		subscribers:  xsync.NewMapOf[context.Context, chan<- docker.Host](),
+	}
+}
+
+func (m *RetriableClientManager) Subscribe(ctx context.Context, channel chan<- docker.Host) {
+	m.subscribers.Store(ctx, channel)
+
+	go func() {
+		<-ctx.Done()
+		m.subscribers.Delete(ctx)
+	}()
+}
+
+func (m *RetriableClientManager) RetryAndList() ([]ClientService, []error) {
+	m.mu.Lock()
+	errors := make([]error, 0)
+	if len(m.failedAgents) > 0 {
+		newFailed := make([]string, 0)
+		for _, endpoint := range m.failedAgents {
+			if agent, err := agent.NewClient(endpoint, m.certs); err == nil {
+				m.clients[agent.Host().ID] = NewAgentService(agent)
+
+				m.subscribers.Range(func(ctx context.Context, channel chan<- docker.Host) bool {
+					host := agent.Host()
+					host.Available = true
+
+					// We don't want to block the subscribers in event.go
+					go func() {
+						select {
+						case channel <- host:
+						case <-ctx.Done():
+						}
+					}()
+
+					return true
+				})
+
+			} else {
+				log.Warnf("error creating agent client for %s: %v", endpoint, err)
+				errors = append(errors, err)
+				newFailed = append(newFailed, endpoint)
+			}
+		}
+		m.failedAgents = newFailed
+	}
+
+	m.mu.Unlock()
+
+	return m.List(), errors
+}
+
+func (m *RetriableClientManager) List() []ClientService {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+
+	clients := make([]ClientService, 0, len(m.clients))
+	for _, client := range m.clients {
+		clients = append(clients, client)
+	}
+	return clients
+}
+
+func (m *RetriableClientManager) Find(id string) (ClientService, bool) {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+
+	client, ok := m.clients[id]
+	return client, ok
+}
+
+func (m *RetriableClientManager) String() string {
+	return fmt.Sprintf("RetriableClientManager{clients: %d, failedAgents: %d}", len(m.clients), len(m.failedAgents))
+}
+
+func (m *RetriableClientManager) Hosts() []docker.Host {
+	clients := m.List()
+
+	hosts := make([]docker.Host, 0, len(clients))
+	for _, client := range clients {
+		host := client.Host()
+		host.Available = true
+		hosts = append(hosts, host)
+	}
+
+	for _, endpoint := range m.failedAgents {
+		hosts = append(hosts, docker.Host{
+			ID:        endpoint,
+			Name:      endpoint,
+			Endpoint:  endpoint,
+			Available: false,
+			Type:      "agent",
+		})
+	}
+
+	return hosts
+}
diff --git a/internal/support/docker/swarm_client_manager.go b/internal/support/docker/swarm_client_manager.go
new file mode 100644
index 00000000..40a3718b
--- /dev/null
+++ b/internal/support/docker/swarm_client_manager.go
@@ -0,0 +1,170 @@
+package docker_support
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"net"
+	"sync"
+
+	"github.com/amir20/dozzle/internal/agent"
+	"github.com/amir20/dozzle/internal/docker"
+	"github.com/puzpuzpuz/xsync/v3"
+	"github.com/samber/lo"
+
+	log "github.com/sirupsen/logrus"
+)
+
+type SwarmClientManager struct {
+	clients     map[string]ClientService
+	certs       tls.Certificate
+	mu          sync.RWMutex
+	subscribers *xsync.MapOf[context.Context, chan<- docker.Host]
+	localClient docker.Client
+	localIPs    []string
+}
+
+func localIPs() []string {
+	addrs, err := net.InterfaceAddrs()
+	if err != nil {
+		return []string{}
+	}
+
+	ips := make([]string, 0)
+	for _, address := range addrs {
+		if ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
+			if ipnet.IP.To4() != nil {
+				ips = append(ips, ipnet.IP.String())
+			}
+		}
+	}
+	return ips
+}
+
+func NewSwarmClientManager(localClient docker.Client, certs tls.Certificate) *SwarmClientManager {
+	clientMap := make(map[string]ClientService)
+	localService := NewDockerClientService(localClient)
+	clientMap[localClient.Host().ID] = localService
+
+	return &SwarmClientManager{
+		localClient: localClient,
+		clients:     clientMap,
+		certs:       certs,
+		subscribers: xsync.NewMapOf[context.Context, chan<- docker.Host](),
+		localIPs:    localIPs(),
+	}
+}
+
+func (m *SwarmClientManager) Subscribe(ctx context.Context, channel chan<- docker.Host) {
+	m.subscribers.Store(ctx, channel)
+
+	go func() {
+		<-ctx.Done()
+		m.subscribers.Delete(ctx)
+	}()
+}
+
+func (m *SwarmClientManager) RetryAndList() ([]ClientService, []error) {
+	m.mu.Lock()
+	errors := make([]error, 0)
+
+	ips, err := net.LookupIP("tasks.dozzle")
+
+	if err != nil {
+		log.Fatalf("error looking up swarm services: %v", err)
+		errors = append(errors, err)
+		return m.List(), errors
+	}
+
+	clients := lo.Values(m.clients)
+	endpoints := lo.KeyBy(clients, func(client ClientService) string {
+		return client.Host().Endpoint
+	})
+
+	log.Debugf("tasks.dozzle = %v, localIP = %v, clients.endpoints = %v", ips, m.localIPs, lo.Keys(endpoints))
+
+	for _, ip := range ips {
+		if lo.Contains(m.localIPs, ip.String()) {
+			log.Debugf("skipping local ip %s", ip.String())
+			continue
+		}
+
+		if _, ok := endpoints[ip.String()+":7007"]; ok {
+			log.Debugf("skipping existing client for %s", ip.String())
+			continue
+		}
+
+		agent, err := agent.NewClient(ip.String()+":7007", m.certs)
+		if err != nil {
+			log.Warnf("error creating client for %s: %v", ip, err)
+			errors = append(errors, err)
+			continue
+		}
+
+		if agent.Host().ID == m.localClient.Host().ID {
+			log.Debugf("skipping local client with ID %s", agent.Host().ID)
+			if err := agent.Close(); err != nil {
+				log.Warnf("error closing local client: %v", err)
+			}
+			continue
+		}
+
+		client := NewAgentService(agent)
+		m.clients[agent.Host().ID] = client
+		log.Infof("added client for %s", agent.Host().ID)
+
+		m.subscribers.Range(func(ctx context.Context, channel chan<- docker.Host) bool {
+			host := agent.Host()
+			host.Available = true
+			host.Type = "swarm"
+
+			// We don't want to block the subscribers in event.go
+			go func() {
+				select {
+				case channel <- host:
+				case <-ctx.Done():
+				}
+			}()
+
+			return true
+		})
+	}
+
+	m.mu.Unlock()
+
+	return m.List(), errors
+}
+
+func (m *SwarmClientManager) List() []ClientService {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+
+	return lo.Values(m.clients)
+}
+
+func (m *SwarmClientManager) Find(id string) (ClientService, bool) {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+
+	client, ok := m.clients[id]
+	return client, ok
+}
+
+func (m *SwarmClientManager) Hosts() []docker.Host {
+	clients := m.List()
+
+	hosts := make([]docker.Host, 0, len(clients))
+
+	for _, client := range clients {
+		host := client.Host()
+		host.Available = true
+		host.Type = "swarm"
+		hosts = append(hosts, host)
+	}
+
+	return hosts
+}
+
+func (m *SwarmClientManager) String() string {
+	return fmt.Sprintf("SwarmClientManager{clients: %d}", len(m.clients))
+}
diff --git a/internal/web/events.go b/internal/web/events.go
index 1b9dbc32..a166bd52 100644
--- a/internal/web/events.go
+++ b/internal/web/events.go
@@ -27,21 +27,24 @@ func (h *handler) streamEvents(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("X-Accel-Buffering", "no")
 
 	ctx := r.Context()
+	events := make(chan docker.ContainerEvent)
+	stats := make(chan docker.ContainerStat)
+	availableHosts := make(chan docker.Host)
+
+	h.multiHostService.SubscribeEventsAndStats(ctx, events, stats)
+	h.multiHostService.SubscribeAvailableHosts(ctx, availableHosts)
 
 	allContainers, errors := h.multiHostService.ListAllContainers()
 
 	for _, err := range errors {
 		log.Warnf("error listing containers: %v", err)
 		if hostNotAvailableError, ok := err.(*docker_support.HostUnavailableError); ok {
-			if _, err := fmt.Fprintf(w, "event: host-unavailable\ndata: %s\n\n", hostNotAvailableError.Host.ID); err != nil {
+			bytes, _ := json.Marshal(hostNotAvailableError.Host)
+			if _, err := fmt.Fprintf(w, "event: update-host\ndata: %s\n\n", string(bytes)); err != nil {
 				log.Errorf("error writing event to event stream: %v", err)
 			}
 		}
 	}
-	events := make(chan docker.ContainerEvent)
-	stats := make(chan docker.ContainerStat)
-
-	h.multiHostService.SubscribeEventsAndStats(ctx, events, stats)
 
 	if err := sendContainersJSON(allContainers, w); err != nil {
 		log.Errorf("error writing containers to event stream: %v", err)
@@ -53,6 +56,12 @@ func (h *handler) streamEvents(w http.ResponseWriter, r *http.Request) {
 
 	for {
 		select {
+		case host := <-availableHosts:
+			bytes, _ := json.Marshal(host)
+			if _, err := fmt.Fprintf(w, "event: update-host\ndata: %s\n\n", string(bytes)); err != nil {
+				log.Errorf("error writing event to event stream: %v", err)
+			}
+			f.Flush()
 		case stat := <-stats:
 			bytes, _ := json.Marshal(stat)
 			if _, err := fmt.Fprintf(w, "event: container-stat\ndata: %s\n\n", string(bytes)); err != nil {
diff --git a/internal/web/events_test.go b/internal/web/events_test.go
index 64bd73a1..cefd85bf 100644
--- a/internal/web/events_test.go
+++ b/internal/web/events_test.go
@@ -2,6 +2,7 @@ package web
 
 import (
 	"context"
+	"crypto/tls"
 	"time"
 
 	"net/http"
@@ -53,9 +54,9 @@ func Test_handler_streamEvents_happy(t *testing.T) {
 	})
 
 	// This is needed so that the server is initialized for store
-	multiHostService := docker_support.NewMultiHostService(
-		[]docker_support.ClientService{docker_support.NewDockerClientService(mockedClient)},
-	)
+	manager := docker_support.NewRetriableClientManager(nil, tls.Certificate{}, docker_support.NewDockerClientService(mockedClient))
+	multiHostService := docker_support.NewMultiHostService(manager)
+
 	server := CreateServer(multiHostService, nil, Config{Base: "/", Authorization: Authorization{Provider: NONE}})
 
 	handler := server.Handler
diff --git a/internal/web/routes_test.go b/internal/web/routes_test.go
index b01cc910..39c1b949 100644
--- a/internal/web/routes_test.go
+++ b/internal/web/routes_test.go
@@ -2,6 +2,7 @@ package web
 
 import (
 	"context"
+	"crypto/tls"
 	"time"
 
 	"io"
@@ -85,7 +86,8 @@ func createHandler(client docker.Client, content fs.FS, config Config) *chi.Mux
 		content = afero.NewIOFS(fs)
 	}
 
-	multiHostService := docker_support.NewMultiHostService([]docker_support.ClientService{docker_support.NewDockerClientService(client)})
+	manager := docker_support.NewRetriableClientManager(nil, tls.Certificate{}, docker_support.NewDockerClientService(client))
+	multiHostService := docker_support.NewMultiHostService(manager)
 	return createRouter(&handler{
 		multiHostService: multiHostService,
 		content:          content,
diff --git a/main.go b/main.go
index 51d4a91b..c4f3f958 100644
--- a/main.go
+++ b/main.go
@@ -51,14 +51,29 @@ func main() {
 			if err != nil {
 				log.Fatalf("failed to listen: %v", err)
 			}
-			tempFile, err := os.CreateTemp("/", "agent-*.addr")
+			tempFile, err := os.CreateTemp("./", "agent-*.addr")
 			if err != nil {
 				log.Fatalf("failed to create temp file: %v", err)
 			}
-			defer os.Remove(tempFile.Name())
 			io.WriteString(tempFile, listener.Addr().String())
 			go cli.StartEvent(args, "", client, "agent")
-			agent.RunServer(client, certs, listener)
+			server := agent.NewServer(client, certs, args.Version())
+			ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+			defer stop()
+			go func() {
+				log.Infof("Dozzle agent version %s", args.Version())
+				log.Infof("Agent listening on %s", listener.Addr().String())
+				if err := server.Serve(listener); err != nil {
+					log.Fatalf("failed to serve: %v", err)
+				}
+			}()
+			<-ctx.Done()
+			stop()
+			log.Info("Shutting down agent")
+			server.Stop()
+			log.Debugf("deleting %s", tempFile.Name())
+			os.Remove(tempFile.Name())
+
 		case *cli.HealthcheckCmd:
 			go cli.StartEvent(args, "", nil, "healthcheck")
 			files, err := os.ReadDir(".")
@@ -135,13 +150,20 @@ func main() {
 		if err != nil {
 			log.Fatalf("Could not read certificates: %v", err)
 		}
-		multiHostService = docker_support.NewSwarmService(localClient, certs)
+		manager := docker_support.NewSwarmClientManager(localClient, certs)
+		multiHostService = docker_support.NewMultiHostService(manager)
 		log.Infof("Starting in Swarm mode")
 		listener, err := net.Listen("tcp", ":7007")
 		if err != nil {
 			log.Fatalf("failed to listen: %v", err)
 		}
-		go agent.RunServer(localClient, certs, listener)
+		server := agent.NewServer(localClient, certs, args.Version())
+		go func() {
+			log.Infof("Agent listening on %s", listener.Addr().String())
+			if err := server.Serve(listener); err != nil {
+				log.Fatalf("failed to serve: %v", err)
+			}
+		}()
 	} else {
 		log.Fatalf("Invalid mode %s", args.Mode)
 	}
@@ -159,7 +181,7 @@ func main() {
 	<-ctx.Done()
 	stop()
 	log.Info("shutting down gracefully, press Ctrl+C again to force")
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
 	defer cancel()
 	if err := srv.Shutdown(ctx); err != nil {
 		log.Fatal(err)
diff --git a/protos/types.proto b/protos/types.proto
index e97211b5..2c1a953a 100644
--- a/protos/types.proto
+++ b/protos/types.proto
@@ -62,4 +62,6 @@ message Host {
   string osType = 8;
   uint32 cpuCores = 9;
   uint64 memory = 10;
+  string agentVersion = 11;
+  string dockerVersion = 12;
 }