From 1cf178e37db3392bf0c57403223e8e0e7cc0e3a9 Mon Sep 17 00:00:00 2001
From: Amir Raminfar <findamir@gmail.com>
Date: Thu, 17 Apr 2025 14:09:06 -0700
Subject: [PATCH] feat: more improvements to analytic and gzips logs for better
 performance (#3814)

---
 .air.toml                                    |  2 +-
 assets/components/LogViewer/LogAnalytics.vue | 49 ++++++++++++--------
 internal/web/logs.go                         | 11 ++++-
 locales/da.yml                               |  1 +
 locales/de.yml                               |  1 +
 locales/en.yml                               |  1 +
 locales/es.yml                               |  1 +
 locales/fr.yml                               |  1 +
 locales/it.yml                               |  1 +
 locales/pl.yml                               |  1 +
 locales/pr.yml                               |  1 +
 locales/pt.yml                               |  1 +
 locales/ru.yml                               |  1 +
 locales/tr.yml                               |  1 +
 locales/zh-tw.yml                            |  1 +
 locales/zh.yml                               |  1 +
 16 files changed, 53 insertions(+), 22 deletions(-)

diff --git a/.air.toml b/.air.toml
index f7f3a5d8..1f6385f3 100644
--- a/.air.toml
+++ b/.air.toml
@@ -35,7 +35,7 @@ pre_cmd = []
 rerun = false
 rerun_delay = 500
 send_interrupt = true
-stop_on_error = false
+stop_on_error = true
 
 [color]
 app = ""
diff --git a/assets/components/LogViewer/LogAnalytics.vue b/assets/components/LogViewer/LogAnalytics.vue
index 5bd70680..5dc19879 100644
--- a/assets/components/LogViewer/LogAnalytics.vue
+++ b/assets/components/LogViewer/LogAnalytics.vue
@@ -13,11 +13,12 @@
             class="textarea textarea-primary w-full font-mono text-lg"
             :class="{ 'textarea-error': error }"
           ></textarea>
-          <div class="label mt-2 overflow-y-auto">
-            <span v-if="!isReady">{{ $t("analytics.creating_table") }}</span>
+          <div class="mt-2">
+            <span class="text-error" v-if="state === 'error'">{{ error }}</span>
+            <span v-else-if="state === 'initializing'">{{ $t("analytics.creating_table") }}</span>
+            <span v-else-if="state === 'downloading'">{{ $t("analytics.downloading") }}</span>
             <span v-else-if="evaluating">{{ $t("analytics.evaluating_query") }}</span>
-            <span class="label-text-alt text-error" v-else-if="error">{{ error }}</span>
-            <span class="label-text-alt" v-else>
+            <span v-else>
               {{ $t("analytics.total_records", { count: results.numRows.toLocaleString() }) }}
               <template v-if="results.numRows > pageLimit">{{
                 $t("analytics.showing_first", { count: page.numRows.toLocaleString() })
@@ -26,7 +27,7 @@
           </div>
         </label>
       </section>
-      <SQLTable :table="page" :loading="evaluating || !isReady" />
+      <SQLTable :table="page" :loading="evaluating || state !== 'ready'" />
     </div>
   </aside>
 </template>
@@ -34,12 +35,14 @@
 <script setup lang="ts">
 import { Container } from "@/models/Container";
 import { type Table } from "@apache-arrow/esnext-esm";
+
 const { container } = defineProps<{ container: Container }>();
 const query = ref("SELECT * FROM logs LIMIT 100");
 const error = ref<string | null>(null);
 const debouncedQuery = debouncedRef(query, 500);
 const evaluating = ref(false);
 const pageLimit = 1000;
+const state = ref<"downloading" | "error" | "ready" | "initializing">("downloading");
 
 const url = withBase(
   `/api/hosts/${container.host}/containers/${container.id}/logs?stdout=1&stderr=1&everything&jsonOnly`,
@@ -55,27 +58,29 @@ if (!response.ok) {
 const { db, conn } = await useDuckDB();
 const empty = await conn.query<Record<string, any>>(`SELECT 1 LIMIT 0`);
 
-const { isReady } = useAsyncState(
-  async () => {
+onMounted(async () => {
+  try {
+    state.value = "downloading";
     await db.registerFileBuffer("logs.json", new Uint8Array(await response.arrayBuffer()));
+
+    state.value = "initializing";
     await conn.query(
       `CREATE TABLE logs AS SELECT unnest(m) FROM read_json('logs.json', ignore_errors = true, format = 'newline_delimited')`,
     );
-  },
-  undefined,
-  {
-    onError: (e) => {
-      console.error(e);
-      if (e instanceof Error) {
-        error.value = e.message;
-      }
-    },
-  },
-);
+
+    state.value = "ready";
+  } catch (e) {
+    console.error(e);
+    state.value = "error";
+    if (e instanceof Error) {
+      error.value = e.message;
+    }
+  }
+});
 
 const results = computedAsync(
   async () => {
-    if (isReady.value) {
+    if (state.value === "ready") {
       return await conn.query<Record<string, any>>(debouncedQuery.value);
     } else {
       return empty;
@@ -85,6 +90,7 @@ const results = computedAsync(
   {
     onError: (e) => {
       console.error(e);
+      state.value = "error";
       if (e instanceof Error) {
         error.value = e.message;
       }
@@ -93,7 +99,10 @@ const results = computedAsync(
   },
 );
 
-whenever(evaluating, () => (error.value = null));
+whenever(evaluating, () => {
+  error.value = null;
+  state.value = "ready";
+});
 const page = computed(() =>
   results.value.numRows > pageLimit ? results.value.slice(0, pageLimit) : results.value,
 ) as unknown as ComputedRef<Table<Record<string, any>>>;
diff --git a/internal/web/logs.go b/internal/web/logs.go
index 0032e711..796e9c15 100644
--- a/internal/web/logs.go
+++ b/internal/web/logs.go
@@ -1,6 +1,7 @@
 package web
 
 import (
+	"compress/gzip"
 	"context"
 	"errors"
 	"regexp"
@@ -111,7 +112,15 @@ func (h *handler) fetchLogsBetweenDates(w http.ResponseWriter, r *http.Request)
 		lastSeenId = uint32(num)
 	}
 
-	encoder := json.NewEncoder(w)
+	var encoder *json.Encoder
+	if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
+		w.Header().Set("Content-Encoding", "gzip")
+		writer := gzip.NewWriter(w)
+		defer writer.Close()
+		encoder = json.NewEncoder(writer)
+	} else {
+		encoder = json.NewEncoder(w)
+	}
 
 	for {
 		if buffer.Len() > minimum {
diff --git a/locales/da.yml b/locales/da.yml
index 3b63f500..c9113317 100644
--- a/locales/da.yml
+++ b/locales/da.yml
@@ -116,6 +116,7 @@ toasts:
     message: Log kopieret til clipboard
 analytics:
   creating_table: Opretter midlertidig tabel...
+  downloading: Henter containerlogfiler...
   evaluating_query: Evaluerer forespørgsel...
   total_records: I alt {count} poster.
   showing_first: Viser de første {count}.
diff --git a/locales/de.yml b/locales/de.yml
index 32198ed9..162d20d0 100644
--- a/locales/de.yml
+++ b/locales/de.yml
@@ -116,6 +116,7 @@ toasts:
     message: Log in Zwischenablage kopiert
 analytics:
   creating_table: Temporäre Tabelle wird erstellt...
+  downloading: Container-Logs werden abgerufen...
   evaluating_query: Abfrage wird ausgewertet...
   total_records: Insgesamt {count} Datensätze.
   showing_first: Zeige die ersten {count}.
diff --git a/locales/en.yml b/locales/en.yml
index e58109aa..a411150c 100644
--- a/locales/en.yml
+++ b/locales/en.yml
@@ -121,6 +121,7 @@ toasts:
     message: Log copied to clipboard
 analytics:
   creating_table: Creating temporary table...
+  downloading: Fetching containers logs...
   evaluating_query: Evaluating query...
   total_records: Total {count} records.
   showing_first: Showing first {count}.
diff --git a/locales/es.yml b/locales/es.yml
index 9a02000d..ee78ebae 100644
--- a/locales/es.yml
+++ b/locales/es.yml
@@ -116,6 +116,7 @@ toasts:
     message: Registro copiado al portapapeles
 analytics:
   creating_table: Creando tabla temporal...
+  downloading: Obteniendo registros de contenedores...
   evaluating_query: Evaluando consulta...
   total_records: Total {count} registros.
   showing_first: Mostrando los primeros {count}.
diff --git a/locales/fr.yml b/locales/fr.yml
index c0c953ec..617dac1c 100644
--- a/locales/fr.yml
+++ b/locales/fr.yml
@@ -116,6 +116,7 @@ toasts:
     message: Journal copié dans le presse-papiers
 analytics:
   creating_table: Création d'une table temporaire...
+  downloading: Récupération des journaux des conteneurs...
   evaluating_query: Évaluation de la requête...
   total_records: Total de {count} enregistrements.
   showing_first: Affichage des {count} premiers.
diff --git a/locales/it.yml b/locales/it.yml
index 755fc9b1..8361af6c 100644
--- a/locales/it.yml
+++ b/locales/it.yml
@@ -116,6 +116,7 @@ toasts:
     message: Log copiato nella clipboard
 analytics:
   creating_table: Creazione tabella temporanea...
+  downloading: Recupero dei log dei container...
   evaluating_query: Valutazione della query...
   total_records: Totale {count} record.
   showing_first: Mostrati i primi {count}.
diff --git a/locales/pl.yml b/locales/pl.yml
index c06eab60..7c66708c 100644
--- a/locales/pl.yml
+++ b/locales/pl.yml
@@ -116,6 +116,7 @@ toasts:
     message: Log skopiowany do schowka
 analytics:
   creating_table: Tworzenie tymczasowej tabeli...
+  downloading: Pobieranie logów kontenerów...
   evaluating_query: Przetwarzanie zapytania...
   total_records: Razem {count} rekordów.
   showing_first: Pokazywanie pierwszych {count}.
diff --git a/locales/pr.yml b/locales/pr.yml
index e1d2723c..af0c0900 100644
--- a/locales/pr.yml
+++ b/locales/pr.yml
@@ -112,6 +112,7 @@ settings:
 
 analytics:
   creating_table: A criar tabela temporária...
+  downloading: A obter registos dos contentores...
   evaluating_query: A avaliar consulta...
   total_records: Total de {count} registos.
   showing_first: A mostrar os primeiros {count}.
diff --git a/locales/pt.yml b/locales/pt.yml
index 866438df..390b76c1 100644
--- a/locales/pt.yml
+++ b/locales/pt.yml
@@ -116,6 +116,7 @@ toasts:
     message: Log copiado para a área de transferência
 analytics:
   creating_table: Criando tabela temporária...
+  downloading: Buscando logs dos containers...
   evaluating_query: Avaliando consulta...
   total_records: Total de {count} registros.
   showing_first: Mostrando os primeiros {count}.
diff --git a/locales/ru.yml b/locales/ru.yml
index bc8b2eaf..358efa4e 100644
--- a/locales/ru.yml
+++ b/locales/ru.yml
@@ -116,6 +116,7 @@ toasts:
     message: Log copied to clipboard
 analytics:
   creating_table: Создание временной таблицы...
+  downloading: Получение логов контейнеров...
   evaluating_query: Выполнение запроса...
   total_records: Всего {count} записей.
   showing_first: Показаны первые {count}.
diff --git a/locales/tr.yml b/locales/tr.yml
index 19e80eb6..1998e0ca 100644
--- a/locales/tr.yml
+++ b/locales/tr.yml
@@ -109,6 +109,7 @@ toasts:
     message: Günlük panoya kopyalandı
 analytics:
   creating_table: Geçici tablo oluşturuluyor...
+  downloading: Konteyner günlükleri alınıyor...
   evaluating_query: Sorgu değerlendiriliyor...
   total_records: Toplam {count} kayıt.
   showing_first: İlk {count} gösteriliyor.
diff --git a/locales/zh-tw.yml b/locales/zh-tw.yml
index fe831308..2b0cae17 100644
--- a/locales/zh-tw.yml
+++ b/locales/zh-tw.yml
@@ -119,6 +119,7 @@ toasts:
     message: 日誌已複製到剪貼簿
 analytics:
   creating_table: 正在建立臨時資料表...
+  downloading: 正在取得容器日誌...
   evaluating_query: 正在評估查詢...
   total_records: 共計 {count} 筆記錄。
   showing_first: 顯示前 {count} 筆。
diff --git a/locales/zh.yml b/locales/zh.yml
index b5902325..f77938c8 100644
--- a/locales/zh.yml
+++ b/locales/zh.yml
@@ -116,6 +116,7 @@ toasts:
     message: 日志已复制到剪贴板
 analytics:
   creating_table: 正在创建临时表...
+  downloading: 正在获取容器日志...
   evaluating_query: 正在评估查询...
   total_records: 总共 {count} 条记录。
   showing_first: 显示前 {count} 条。