mirrors/dozzle
1
0
Fork 0
mirror of https://github.com/amir20/dozzle.git synced 2025-12-28 07:56:37 +01:00
Code Issues Projects Releases Wiki Activity

feat: duckdb wasm capibility (#3272)

Browse Source
This commit is contained in:
Amir Raminfar
2024-09-18 17:27:46 -07:00
committed by GitHub
parent f1553e33e1
commit 90b9d97685
26 changed files with 851 additions and 260 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
assets/auto-imports.d.ts vendored
View File

@@ -31,6 +31,7 @@ declare global {
const controlledComputed: typeof import('@vueuse/core')['controlledComputed']
const controlledRef: typeof import('@vueuse/core')['controlledRef']
const createApp: typeof import('vue')['createApp']
const createDrawer: typeof import('./composable/drawer')['createDrawer']
const createEventHook: typeof import('@vueuse/core')['createEventHook']
const createGlobalState: typeof import('@vueuse/core')['createGlobalState']
const createInjectionState: typeof import('@vueuse/core')['createInjectionState']
@@ -47,6 +48,8 @@ declare global {
const defineAsyncComponent: typeof import('vue')['defineAsyncComponent']
const defineComponent: typeof import('vue')['defineComponent']
const defineStore: typeof import('pinia')['defineStore']
const drawerContext: typeof import('./composable/drawer')['drawerContext']
const drwaer: typeof import('./composable/showLogDetails')['drwaer']
const eagerComputed: typeof import('@vueuse/core')['eagerComputed']
const effectScope: typeof import('vue')['effectScope']
const extendRef: typeof import('@vueuse/core')['extendRef']
@@ -62,6 +65,7 @@ declare global {
const hashCode: typeof import('./utils/index')['hashCode']
const hourStyle: typeof import('./stores/settings')['hourStyle']
const ignorableWatch: typeof import('@vueuse/core')['ignorableWatch']
const initDuckDB: typeof import('./composable/duckdb')['initDuckDB']
const inject: typeof import('vue')['inject']
const injectLocal: typeof import('@vueuse/core')['injectLocal']
const isDefined: typeof import('@vueuse/core')['isDefined']
@@ -217,7 +221,9 @@ declare global {
const useDisplayMedia: typeof import('@vueuse/core')['useDisplayMedia']
const useDocumentVisibility: typeof import('@vueuse/core')['useDocumentVisibility']
const useDraggable: typeof import('@vueuse/core')['useDraggable']
const useDrawer: typeof import('./composable/drawer')['useDrawer']
const useDropZone: typeof import('@vueuse/core')['useDropZone']
const useDuckDB: typeof import('./composable/duckdb')['useDuckDB']
const useElementBounding: typeof import('@vueuse/core')['useElementBounding']
const useElementByPoint: typeof import('@vueuse/core')['useElementByPoint']
const useElementHover: typeof import('@vueuse/core')['useElementHover']
@@ -401,6 +407,7 @@ declare module 'vue' {
readonly controlledComputed: UnwrapRef<typeof import('@vueuse/core')['controlledComputed']>
readonly controlledRef: UnwrapRef<typeof import('@vueuse/core')['controlledRef']>
readonly createApp: UnwrapRef<typeof import('vue')['createApp']>
readonly createDrawer: UnwrapRef<typeof import('./composable/drawer')['createDrawer']>
readonly createEventHook: UnwrapRef<typeof import('@vueuse/core')['createEventHook']>
readonly createGlobalState: UnwrapRef<typeof import('@vueuse/core')['createGlobalState']>
readonly createInjectionState: UnwrapRef<typeof import('@vueuse/core')['createInjectionState']>
@@ -417,6 +424,7 @@ declare module 'vue' {
readonly defineAsyncComponent: UnwrapRef<typeof import('vue')['defineAsyncComponent']>
readonly defineComponent: UnwrapRef<typeof import('vue')['defineComponent']>
readonly defineStore: UnwrapRef<typeof import('pinia')['defineStore']>
readonly drawerContext: UnwrapRef<typeof import('./composable/drawer')['drawerContext']>
readonly eagerComputed: UnwrapRef<typeof import('@vueuse/core')['eagerComputed']>
readonly effectScope: UnwrapRef<typeof import('vue')['effectScope']>
readonly extendRef: UnwrapRef<typeof import('@vueuse/core')['extendRef']>
@@ -478,7 +486,6 @@ declare module 'vue' {
readonly pinnedContainers: UnwrapRef<typeof import('./composable/storage')['pinnedContainers']>
readonly provide: UnwrapRef<typeof import('vue')['provide']>
readonly provideLocal: UnwrapRef<typeof import('@vueuse/core')['provideLocal']>
readonly provideLogDetails: UnwrapRef<typeof import('./composable/showLogDetails')['provideLogDetails']>
readonly provideLoggingContext: UnwrapRef<typeof import('./composable/logContext')['provideLoggingContext']>
readonly provideScrollContext: UnwrapRef<typeof import('./composable/scrollContext')['provideScrollContext']>
readonly reactify: UnwrapRef<typeof import('@vueuse/core')['reactify']>
@@ -508,7 +515,6 @@ declare module 'vue' {
readonly shallowReadonly: UnwrapRef<typeof import('vue')['shallowReadonly']>
readonly shallowRef: UnwrapRef<typeof import('vue')['shallowRef']>
readonly showAllContainers: UnwrapRef<typeof import('./stores/settings')['showAllContainers']>
readonly showLogDetails: UnwrapRef<typeof import('./composable/showLogDetails')['showLogDetails']>
readonly showStd: UnwrapRef<typeof import('./stores/settings')['showStd']>
readonly showTimestamp: UnwrapRef<typeof import('./stores/settings')['showTimestamp']>
readonly size: UnwrapRef<typeof import('./stores/settings')['size']>
@@ -585,7 +591,9 @@ declare module 'vue' {
readonly useDisplayMedia: UnwrapRef<typeof import('@vueuse/core')['useDisplayMedia']>
readonly useDocumentVisibility: UnwrapRef<typeof import('@vueuse/core')['useDocumentVisibility']>
readonly useDraggable: UnwrapRef<typeof import('@vueuse/core')['useDraggable']>
readonly useDrawer: UnwrapRef<typeof import('./composable/drawer')['useDrawer']>
readonly useDropZone: UnwrapRef<typeof import('@vueuse/core')['useDropZone']>
readonly useDuckDB: UnwrapRef<typeof import('./composable/duckdb')['useDuckDB']>
readonly useElementBounding: UnwrapRef<typeof import('@vueuse/core')['useElementBounding']>
readonly useElementByPoint: UnwrapRef<typeof import('@vueuse/core')['useElementByPoint']>
readonly useElementHover: UnwrapRef<typeof import('@vueuse/core')['useElementHover']>
@@ -620,7 +628,6 @@ declare module 'vue' {
readonly useKeyModifier: UnwrapRef<typeof import('@vueuse/core')['useKeyModifier']>
readonly useLastChanged: UnwrapRef<typeof import('@vueuse/core')['useLastChanged']>
readonly useLocalStorage: UnwrapRef<typeof import('@vueuse/core')['useLocalStorage']>
readonly useLogDetails: UnwrapRef<typeof import('./composable/showLogDetails')['useLogDetails']>
readonly useLoggingContext: UnwrapRef<typeof import('./composable/logContext')['useLoggingContext']>
readonly useMagicKeys: UnwrapRef<typeof import('@vueuse/core')['useMagicKeys']>
readonly useManualRefHistory: UnwrapRef<typeof import('@vueuse/core')['useManualRefHistory']>

2
assets/components.d.ts vendored
View File

@@ -46,6 +46,7 @@ declare module 'vue' {
KeyShortcut: typeof import('./components/common/KeyShortcut.vue')['default']
LabeledInput: typeof import('./components/common/LabeledInput.vue')['default']
Links: typeof import('./components/Links.vue')['default']
LogAnalytics: typeof import('./components/LogViewer/LogAnalytics.vue')['default']
LogDate: typeof import('./components/LogViewer/LogDate.vue')['default']
LogDetails: typeof import('./components/LogViewer/LogDetails.vue')['default']
LogLevel: typeof import('./components/LogViewer/LogLevel.vue')['default']
@@ -85,6 +86,7 @@ declare module 'vue' {
'Ph:computerTower': typeof import('~icons/ph/computer-tower')['default']
'Ph:controlBold': typeof import('~icons/ph/control-bold')['default']
'Ph:cpu': typeof import('~icons/ph/cpu')['default']
'Ph:fileSql': typeof import('~icons/ph/file-sql')['default']
'Ph:globeSimple': typeof import('~icons/ph/globe-simple')['default']
'Ph:memory': typeof import('~icons/ph/memory')['default']
'Ph:stack': typeof import('~icons/ph/stack')['default']

28
assets/components/ContainerViewer/ContainerActionsToolbar.vue
View File

@@ -20,6 +20,12 @@
<KeyShortcut char="f" />
</a>
</li>
<li v-if="hasComplexLogs">
<a @click.prevent="showDrawer(LogAnalytics, { container }, 'lg')">
<ph:file-sql /> SQL Analytics
<KeyShortcut char="f" :modifiers="['shift', 'meta']" />
</a>
</li>
<li class="line"></li>
<li>
<a
@@ -102,18 +108,26 @@
<script lang="ts" setup>
import { Container } from "@/models/Container";
import LogAnalytics from "../LogViewer/LogAnalytics.vue";
const { showSearch } = useSearchFilter();
const { enableActions } = config;
const clear = defineEmit();
const { streamConfig } = useLoggingContext();
const { streamConfig, hasComplexLogs } = useLoggingContext();
const showDrawer = useDrawer();
const { container } = defineProps<{ container: Container }>();
const clear = defineEmit();
const { actionStates, start, stop, restart } = useContainerActions(toRef(() => container));
onKeyStroke("f", (e) => {
if (hasComplexLogs.value) {
if ((e.ctrlKey || e.metaKey) && e.shiftKey) {
showDrawer(LogAnalytics, { container }, "lg");
e.preventDefault();
}
}
});
const downloadParams = computed(() =>
Object.entries(toValue(streamConfig))
.filter(([, value]) => value)
@@ -126,9 +140,7 @@ const downloadUrl = computed(() =>
),
);
const disableRestart = computed(() => {
return actionStates.stop || actionStates.start || actionStates.restart;
});
const disableRestart = computed(() => actionStates.stop || actionStates.start || actionStates.restart);
</script>
<style scoped lang="postcss">

5
assets/components/LogViewer/ComplexLogItem.vue
View File

@@ -1,5 +1,5 @@
<template>
<div class="group/item clickable relative flex w-full gap-x-2" @click="showLogDetails(logEntry)">
<div class="group/item clickable relative flex w-full gap-x-2" @click="showDrawer(LogDetails, { entry: logEntry })">
<div v-if="showContainerName">
<ContainerName :id="logEntry.containerID" />
</div>
@@ -28,6 +28,7 @@
</template>
<script lang="ts" setup>
import { type ComplexLogEntry } from "@/models/LogEntry";
import LogDetails from "./LogDetails.vue";
const { logEntry, showContainerName = false } = defineProps<{
logEntry: ComplexLogEntry;
@@ -38,7 +39,7 @@ const validValues = computed(() => {
return Object.fromEntries(Object.entries(logEntry.message).filter(([_, value]) => value !== undefined));
});
const showLogDetails = useLogDetails();
const showDrawer = useDrawer();
</script>
<style lang="postcss" scoped>

1
assets/components/LogViewer/EventSource.spec.ts
View File

@@ -81,6 +81,7 @@ describe("<ContainerEventSource />", () => {
[loggingContextKey as symbol]: {
containers: computed(() => [{ id: "abc", image: "test:v123", host: "localhost" }]),
streamConfig: reactive({ stdout: true, stderr: true }),
hasComplexLogs: ref(false),
},
},
},

2
assets/components/LogViewer/EventSource.vue
View File

@@ -21,7 +21,7 @@ defineExpose({
});
const fetchMore = async () => {
if (!isLoadingMore()) {
if (!isLoadingMore.value) {
loadingMore.value = true;
enabled.value = false;
await loadOlderLogs();

99
assets/components/LogViewer/LogAnalytics.vue Normal file
View File

@@ -0,0 +1,99 @@
<template>
<aside>
<header class="flex items-center gap-4">
<h1 class="mobile-hidden text-2xl">{{ container.name }}</h1>
<h2 class="text-sm"><DistanceTime :date="container.created" /></h2>
</header>
<div class="mt-8 flex flex-col gap-2">
<section>
<label class="form-control">
<textarea
v-model="query"
class="textarea textarea-primary w-full font-mono text-lg"
:class="{ 'textarea-error': error }"
></textarea>
<div class="label">
<span class="label-text-alt text-error" v-if="error">{{ error }}</span>
<span class="label-text-alt" v-else>Total {{ results.numRows }} records</span>
</div>
</label>
</section>
<table class="table table-zebra table-pin-rows table-md" v-if="!evaluating">
<thead>
<tr>
<th v-for="column in columns" :key="column">{{ column }}</th>
</tr>
</thead>
<tbody>
<tr v-for="row in page" :key="row">
<td v-for="column in columns" :key="column">{{ row[column] }}</td>
</tr>
</tbody>
</table>
<table class="table table-md animate-pulse" v-else>
<thead>
<tr>
<th v-for="_ in 3">
<div class="h-4 w-20 animate-pulse bg-base-content/50 opacity-50"></div>
</th>
</tr>
</thead>
<tbody>
<tr v-for="_ in 9">
<td v-for="_ in 3">
<div class="h-4 w-20 bg-base-content/50 opacity-20"></div>
</td>
</tr>
</tbody>
</table>
</div>
</aside>
</template>
<script setup lang="ts">
import { Container } from "@/models/Container";
const { container } = defineProps<{ container: Container }>();
const query = ref("SELECT * FROM logs");
const error = ref<string | null>(null);
const debouncedQuery = debouncedRef(query, 500);
const evaluating = ref(false);
const url = withBase(
`/api/hosts/${container.host}/containers/${container.id}/logs?stdout=1&stderr=1&everything&jsonOnly`,
);
const [{ useDuckDB }, response] = await Promise.all([import(`@/composable/duckdb`), fetch(url)]);
if (!response.ok) {
console.log("error fetching logs from", url);
throw new Error(`Failed to fetch logs: ${response.statusText}`);
}
const { db, conn } = await useDuckDB();
await db.registerFileBuffer("logs.json", new Uint8Array(await response.arrayBuffer()));
await conn.query(`CREATE TABLE logs AS SELECT unnest(m) FROM 'logs.json'`);
const empty = await conn.query<Record<string, any>>(`SELECT * FROM logs LIMIT 0`);
const results = computedAsync(async () => await conn.query<Record<string, any>>(debouncedQuery.value), empty, {
onError: (e) => {
if (e instanceof Error) {
error.value = e.message;
}
},
evaluating,
});
whenever(evaluating, () => {
error.value = null;
});
const columns = computed(() =>
results.value.numRows > 0 ? Object.keys(results.value.get(0) as Record<string, any>) : [],
);
const page = computed(() => (results.value.numRows > 0 ? results.value.slice(0, 20) : []));
</script>
<style lang="postcss" scoped></style>

11
assets/components/LogViewer/LogViewer.vue
View File

@@ -1,13 +1,9 @@
<template>
<SideDrawer ref="drawer">
<LogDetails :entry="entry" v-if="entry && entry instanceof ComplexLogEntry" />
</SideDrawer>
<LogList :messages="visibleMessages" :show-container-name="showContainerName" />
</template>
<script lang="ts" setup>
import SideDrawer from "@/components/common/SideDrawer.vue";
import { ComplexLogEntry, type JSONObject, LogEntry } from "@/models/LogEntry";
import { type JSONObject, LogEntry } from "@/models/LogEntry";
const props = defineProps<{
messages: LogEntry<string | JSONObject>[];
@@ -21,12 +17,7 @@ const { filteredPayload } = useVisibleFilter(visibleKeys);
const { debouncedSearchFilter } = useSearchFilter();
const { streamConfig } = useLoggingContext();
const drawer = ref<InstanceType<typeof SideDrawer>>() as Ref<InstanceType<typeof SideDrawer>>;
const { entry } = provideLogDetails(drawer);
const visibleMessages = filteredPayload(messages);
const router = useRouter();
watchEffect(() => {

9
assets/components/LogViewer/ViewerWithSource.vue
View File

@@ -5,8 +5,9 @@
</template>
<script lang="ts" setup generic="T">
import LogEventSource from "@/components/ContainerViewer/LogEventSource.vue";
import EventSource from "@/components/LogViewer/EventSource.vue";
import { LogStreamSource } from "@/composable/eventStreams";
import { ComponentExposed } from "vue-component-type-helpers";
const { streamSource, visibleKeys, showContainerName, entity } = defineProps<{
streamSource: (t: Ref<T>) => LogStreamSource;
@@ -15,15 +16,15 @@ const { streamSource, visibleKeys, showContainerName, entity } = defineProps<{
entity: T;
}>();
const source = $ref<InstanceType<typeof LogEventSource>>();
const source = useTemplateRef<ComponentExposed<typeof EventSource>>("source");
defineExpose({
clear: () => source?.clear(),
clear: () => source.value?.clear(),
});
onKeyStroke("k", (e) => {
if ((e.ctrlKey || e.metaKey) && e.shiftKey) {
source?.clear();
source.value?.clear();
e.preventDefault();
}
});

10
assets/components/LogViewer/__snapshots__/EventSource.spec.ts.snap
View File

@@ -59,16 +59,6 @@ exports[`<ContainerEventSource /> > render html correctly > should render messag
exports[`<ContainerEventSource /> > renders loading correctly 1`] = `
"<div class="flex min-h-[1px] justify-center"><span class="loading loading-bars loading-md mt-4 text-primary" style="display: none;"></span></div>
<dialog data-v-43493931="" class="modal-right modal items-start outline-none backdrop:bg-none">
<div data-v-43493931="" class="modal-box">
<form data-v-43493931="" method="dialog"><button data-v-43493931="" class="swap swap-rotate absolute right-4 top-4 outline-none hover:swap-active"><svg data-v-43493931="" viewBox="0 0 24 24" width="1.2em" height="1.2em" class="swap-off">
<path fill="currentColor" d="M1 7h6v2H3v2h4v2H3v2h4v2H1zm10 0h4v2h-4v2h2a2 2 0 0 1 2 2v2c0 1.11-.89 2-2 2H9v-2h4v-2h-2a2 2 0 0 1-2-2V9c0-1.1.9-2 2-2m8 0h2a2 2 0 0 1 2 2v1h-2V9h-2v6h2v-1h2v1c0 1.11-.89 2-2 2h-2a2 2 0 0 1-2-2V9c0-1.1.9-2 2-2"></path>
</svg><svg data-v-43493931="" viewBox="0 0 24 24" width="1.2em" height="1.2em" class="swap-on">
<path fill="currentColor" d="M19 6.41L17.59 5L12 10.59L6.41 5L5 6.41L10.59 12L5 17.59L6.41 19L12 13.41L17.59 19L19 17.59L13.41 12z"></path>
</svg></button></form>
</div>
<form data-v-43493931="" method="dialog" class="modal-backdrop"><button data-v-43493931="">close</button></form>
</dialog>
<!--v-if-->"
`;

2
assets/components/Search.vue
View File

@@ -34,7 +34,7 @@ const { searchQueryFilter, showSearch, resetSearch, isValidQuery } = useSearchFi
const { style } = useDraggable(container);
onKeyStroke("f", (e) => {
if (e.ctrlKey || e.metaKey) {
if ((e.ctrlKey || e.metaKey) && !e.shiftKey) {
showSearch.value = true;
nextTick(() => input.value?.focus() || input.value?.select());
e.preventDefault();

25
assets/components/common/SideDrawer.vue
View File

@@ -1,13 +1,13 @@
<template>
<dialog ref="panel" class="modal-right modal items-start outline-none backdrop:bg-none">
<div class="modal-box">
<div class="modal-box" :width="width">
<form method="dialog">
<button class="swap swap-rotate absolute right-4 top-4 outline-none hover:swap-active">
<mdi:keyboard-esc class="swap-off" />
<mdi:close class="swap-on" />
</button>
</form>
<slot></slot>
<slot v-if="open"></slot>
</div>
<form method="dialog" class="modal-backdrop">
<button>close</button>
@@ -15,17 +15,34 @@
</dialog>
</template>
<script setup lang="ts">
const panel = ref<HTMLDialogElement>();
import { type DrawerWidth } from "@/composable/drawer";
const panel = useTemplateRef<HTMLDialogElement>("panel");
const open = ref(false);
const { width } = defineProps<{
width: DrawerWidth;
}>();
defineExpose({
open: () => {
open.value = true;
panel.value?.showModal();
},
});
useEventListener(panel, "close", () => (open.value = false));
</script>
<style scoped lang="postcss">
.modal-right :where(.modal-box) {
@apply fixed right-0 h-lvh max-h-screen max-w-3xl translate-x-24 scale-100 rounded-none bg-base-lighter shadow-none;
@apply fixed right-0 h-lvh max-h-screen translate-x-24 scale-100 rounded-none bg-base-lighter shadow-none;
&[width="md"] {
@apply max-w-3xl;
}
&[width="lg"] {
@apply max-w-5xl;
}
}
.modal-right[open] .modal-box {

29
assets/composable/drawer.ts Normal file
View File

@@ -0,0 +1,29 @@
import SideDrawer from "@/components/common/SideDrawer.vue";
import { Component } from "vue";
export type DrawerWidth = "md" | "xl" | "lg";
export const drawerContext = Symbol("drawer") as InjectionKey<
(c: Component, p: Record<string, any>, s?: DrawerWidth) => void
>;
export const createDrawer = (drawer: Ref<InstanceType<typeof SideDrawer>>) => {
const component = shallowRef<Component | null>(null);
const properties = shallowRef<Record<string, any>>({});
const width = ref<DrawerWidth>("md");
const showDrawer = (c: Component, p: Record<string, any>, w: DrawerWidth = "md") => {
component.value = c;
properties.value = p;
width.value = w;
drawer.value?.open();
};
provide(drawerContext, showDrawer);
return { component, properties, showDrawer, width };
};
export const useDrawer = () =>
inject(drawerContext, () => {
console.error("No drawer context provided");
});

30
assets/composable/duckdb.ts Normal file
View File

@@ -0,0 +1,30 @@
import * as duckdb from "@duckdb/duckdb-wasm";
const JSDELIVR_BUNDLES = duckdb.getJsDelivrBundles();
export async function useDuckDB() {
let cleanup: (() => void) | undefined;
onUnmounted(() => cleanup?.());
const bundle = await duckdb.selectBundle(JSDELIVR_BUNDLES);
const worker_url = URL.createObjectURL(
new Blob([`importScripts("${bundle.mainWorker!}");`], { type: "text/javascript" }),
);
// Instantiate the asynchronus version of DuckDB-Wasm
const worker = new Worker(worker_url);
const logger = new duckdb.ConsoleLogger();
const db = new duckdb.AsyncDuckDB(logger, worker);
await db.instantiate(bundle.mainModule, bundle.pthreadWorker);
URL.revokeObjectURL(worker_url);
const conn = await db.connect();
cleanup = async () => {
console.log("Cleaning up DuckDB");
await conn.close();
await db.terminate();
worker.terminate();
};
return { db, conn };
}

15
assets/composable/eventStreams.ts
View File

@@ -7,6 +7,7 @@ import {
LogEntry,
asLogEntry,
ContainerEventLogEntry,
ComplexLogEntry,
SkippedLogsEntry,
} from "@/models/LogEntry";
import { Service, Stack } from "@/models/Stack";
@@ -103,7 +104,7 @@ function useLogStream(url: Ref<string>, loadMoreUrl?: Ref<string>) {
buffer.value = [];
}
const { streamConfig } = useLoggingContext();
const { streamConfig, hasComplexLogs } = useLoggingContext();
const params = computed(() => {
const params = Object.entries(toValue(streamConfig))
@@ -154,11 +155,11 @@ function useLogStream(url: Ref<string>, loadMoreUrl?: Ref<string>) {
watch(urlWithParams, () => connect(), { immediate: true });
let fetchingInProgress = false;
const isLoadingMore = ref(false);
async function loadOlderLogs() {
if (!loadMoreUrl) return;
if (fetchingInProgress) return;
if (isLoadingMore.value) return;
const to = messages.value[0].date;
const last = messages.value[Math.min(messages.value.length - 1, 300)].date;
@@ -167,7 +168,7 @@ function useLogStream(url: Ref<string>, loadMoreUrl?: Ref<string>) {
const abortController = new AbortController();
const signal = abortController.signal;
fetchingInProgress = true;
isLoadingMore.value = true;
try {
const moreParams = { ...params.value, from: from.toISOString(), to: to.toISOString(), minimum: "100" };
const urlWithMoreParams = computed(() =>
@@ -187,13 +188,13 @@ function useLogStream(url: Ref<string>, loadMoreUrl?: Ref<string>) {
} catch (e) {
console.error("Error loading older logs", e);
} finally {
fetchingInProgress = false;
isLoadingMore.value = false;
}
}
onScopeDispose(() => close());
const isLoadingMore = () => fetchingInProgress;
watch(messages, () => (hasComplexLogs.value = messages.value.some((m) => m instanceof ComplexLogEntry)));
return { messages, loadOlderLogs, isLoadingMore };
return { messages, loadOlderLogs, isLoadingMore, hasComplexLogs };
}

3
assets/composable/logContext.ts
View File

@@ -4,6 +4,7 @@ type LogContext = {
streamConfig: { stdout: boolean; stderr: boolean };
containers: Container[];
loadingMore: boolean;
hasComplexLogs: boolean;
};
// export for testing
@@ -19,6 +20,7 @@ export const provideLoggingContext = (containers: Ref<Container[]>) => {
streamConfig: { stdout, stderr },
containers,
loadingMore: false,
hasComplexLogs: false,
}),
);
};
@@ -30,6 +32,7 @@ export const useLoggingContext = () => {
streamConfig: { stdout: true, stderr: true },
containers: [],
loadingMore: false,
hasComplexLogs: false,
}),
);

22
assets/composable/showLogDetails.ts
View File

@@ -1,22 +0,0 @@
import { JSONObject, LogEntry } from "@/models/LogEntry";
import SideDrawer from "@/components/common/SideDrawer.vue";
export const showLogDetails = Symbol("showLogDetails") as InjectionKey<
(logEntry: LogEntry<string | JSONObject>) => void
>;
export const provideLogDetails = (drawer: Ref<InstanceType<typeof SideDrawer>>) => {
const entry = ref<LogEntry<string | JSONObject>>();
provide(showLogDetails, (logEntry: LogEntry<string | JSONObject>) => {
entry.value = logEntry;
drawer.value?.open();
});
return { entry };
};
export const useLogDetails = () => {
const showDetails = inject(showLogDetails, () => {});
return showDetails;
};

9
assets/layouts/default.vue
View File

@@ -42,6 +42,12 @@
<button>close</button>
</form>
</dialog>
<SideDrawer ref="drawer" :width="drawerWidth">
<Suspense>
<component :is="drawerComponent" v-bind="drawerProperties" />
<template #fallback> Loading... </template>
</Suspense>
</SideDrawer>
<div class="toast toast-end whitespace-normal">
<div
class="alert max-w-xl"
@@ -71,11 +77,14 @@
// @ts-ignore - splitpanes types are not available
import { Splitpanes, Pane } from "splitpanes";
import { collapseNav } from "@/stores/settings";
import SideDrawer from "@/components/common/SideDrawer.vue";
const pinnedLogsStore = usePinnedLogsStore();
const { pinnedLogs } = storeToRefs(pinnedLogsStore);
const { toasts, removeToast } = useToast();
const drawer = useTemplateRef<InstanceType<typeof SideDrawer>>("drawer") as Ref<InstanceType<typeof SideDrawer>>;
const { component: drawerComponent, properties: drawerProperties, width: drawerWidth } = createDrawer(drawer);
const modal = ref<HTMLDialogElement>();
const open = ref(false);

BIN
data.parquet Normal file
View File

Binary file not shown.

1
docs/.vitepress/config.ts
View File

@@ -75,6 +75,7 @@ export default defineConfig({
{ text: "Swarm Mode", link: "/guide/swarm-mode" },
{ text: "Supported Env Vars", link: "/guide/supported-env-vars" },
{ text: "Logging Files on Disk", link: "/guide/log-files-on-disk" },
{ text: "SQL Engine", link: "/guide/sql-engine" },
],
},
{

57
docs/guide/sql-engine.md Normal file
View File

@@ -0,0 +1,57 @@
---
title: SQL Engine
---
# SQL Engine <Badge type="warning" text="beta" /> <Badge type="tip" text="v8.5x" />
The SQL Engine is a powerful tool that allows you to run SQL queries against your data. It is designed to provide a seamless experience for users who are familiar with SQL and want to interact with their data using a familiar language.
This feature is currently in beta and is available to all users. If you have any feedback or suggestions, please let us know!
## Getting Started
To get started with the SQL Engine, you will need to have a dataset that you can query. Only JSON logs can be queried using SQL. Dozzle leveages the power of WebAssembly to run SQL queries in the browser, which means that your data never leaves your machine.
To start using the SQL Engine, make sure you have JSON logs and naviage to the drop down and choose `SQL Analytics`. There is also a keyboard shortcut `^+⇧+f` or `⌘+⇧+f` to quickly open the SQL Engine.
## How does it work?
The SQL Engine uses WebAssembly to run SQL queries in the browser with DuckDB. When the SQL Engine is first opened, DuckDB WASM is downloaded and initialized in the browser. This could take a while if you are on a slow connection. The SQL Engine then reads _only_ the JSON logs and creates a virtual table in DuckDB. This allows you to run SQL queries against your data in real-time.
The query that Dozzle runs initially is similar to:
```sql
CREATE TABLE logs AS SELECT unnest(m) FROM 'logs.json'
```
This query creates a table called `logs` and unnests the JSON logs into rows. You can then run SQL queries against this table to analyze your data.
## Example Queries
Here are some example queries that you can run using the SQL Engine:
### Count the number of logs
```sql
SELECT COUNT(*) FROM logs
```
### Filter logs by a specific field
```sql
SELECT * FROM logs WHERE level = 'error'
```
### Group logs by a specific field
```sql
SELECT level, COUNT(*) FROM logs GROUP BY level
```
## Limitations
WebAssembly has some limitations that you should be aware of when using the SQL Engine:
- The SQL Engine only supports structured data such as JSON.
- The SQL Engine is limited to running queries in the browser. This means that you cannot run queries that require access to external resources or databases.
- There is a maximum of 4GB of memory that can be used by the SQL Engine. If you run out of memory, you will need to refresh the page to clear the memory.

26
internal/web/__snapshots__/web.snapshot
View File

@@ -1,7 +1,7 @@
/* snapshot: Test_createRoutes_foobar */
HTTP/1.1 200 OK
Connection: close
Content-Security-Policy: default-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data:;
Content-Security-Policy: default-src 'self' 'wasm-unsafe-eval' blob: https://cdn.jsdelivr.net https://*.duckdb.org; style-src 'self' 'unsafe-inline'; img-src 'self' data:;
Content-Type: text/plain; charset=utf-8
foo page
@@ -9,7 +9,7 @@ foo page
/* snapshot: Test_createRoutes_index */
HTTP/1.1 200 OK
Connection: close
Content-Security-Policy: default-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data:;
Content-Security-Policy: default-src 'self' 'wasm-unsafe-eval' blob: https://cdn.jsdelivr.net https://*.duckdb.org; style-src 'self' 'unsafe-inline'; img-src 'self' data:;
Content-Type: text/plain; charset=utf-8
index page
@@ -17,7 +17,7 @@ index page
/* snapshot: Test_createRoutes_redirect */
HTTP/1.1 301 Moved Permanently
Connection: close
Content-Security-Policy: default-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data:;
Content-Security-Policy: default-src 'self' 'wasm-unsafe-eval' blob: https://cdn.jsdelivr.net https://*.duckdb.org; style-src 'self' 'unsafe-inline'; img-src 'self' data:;
Content-Type: text/html; charset=utf-8
Location: /foobar/
@@ -35,7 +35,7 @@ Location: /foobar/login
/* snapshot: Test_createRoutes_simple_redirect */
HTTP/1.1 307 Temporary Redirect
Connection: close
Content-Security-Policy: default-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data:;
Content-Security-Policy: default-src 'self' 'wasm-unsafe-eval' blob: https://cdn.jsdelivr.net https://*.duckdb.org; style-src 'self' 'unsafe-inline'; img-src 'self' data:;
Content-Type: text/html; charset=utf-8
Location: /login?redirectUrl=/
@@ -75,7 +75,7 @@ data: end of stream
/* snapshot: Test_createRoutes_version */
HTTP/1.1 200 OK
Connection: close
Content-Security-Policy: default-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data:;
Content-Security-Policy: default-src 'self' 'wasm-unsafe-eval' blob: https://cdn.jsdelivr.net https://*.duckdb.org; style-src 'self' 'unsafe-inline'; img-src 'self' data:;
Content-Type: text/html
<pre>dev</pre>
@@ -83,7 +83,7 @@ Content-Type: text/html
/* snapshot: Test_handler_between_dates */
HTTP/1.1 200 OK
Connection: close
Content-Security-Policy: default-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data:;
Content-Security-Policy: default-src 'self' 'wasm-unsafe-eval' blob: https://cdn.jsdelivr.net https://*.duckdb.org; style-src 'self' 'unsafe-inline'; img-src 'self' data:;
Content-Type: application/x-jsonl; charset=UTF-8
{"m":"INFO Testing stdout logs...","ts":1589396137772,"id":466600245,"l":"info","s":"stdout","c":"123456"}
@@ -92,7 +92,7 @@ Content-Type: application/x-jsonl; charset=UTF-8
/* snapshot: Test_handler_between_dates_with_fill */
HTTP/1.1 200 OK
Connection: close
Content-Security-Policy: default-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data:;
Content-Security-Policy: default-src 'self' 'wasm-unsafe-eval' blob: https://cdn.jsdelivr.net https://*.duckdb.org; style-src 'self' 'unsafe-inline'; img-src 'self' data:;
Content-Type: application/x-jsonl; charset=UTF-8
{"m":"INFO Testing stdout logs...","ts":1589396137772,"id":466600245,"l":"info","s":"stdout","c":"123456"}
@@ -132,7 +132,7 @@ Connection: close
Cache-Control: no-transform
Cache-Control: no-cache
Connection: keep-alive
Content-Security-Policy: default-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data:;
Content-Security-Policy: default-src 'self' 'wasm-unsafe-eval' blob: https://cdn.jsdelivr.net https://*.duckdb.org; style-src 'self' 'unsafe-inline'; img-src 'self' data:;
Content-Type: text/event-stream
X-Accel-Buffering: no
@@ -162,14 +162,14 @@ Connection: close
Cache-Control: no-transform
Cache-Control: no-cache
Connection: keep-alive
Content-Security-Policy: default-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data:;
Content-Security-Policy: default-src 'self' 'wasm-unsafe-eval' blob: https://cdn.jsdelivr.net https://*.duckdb.org; style-src 'self' 'unsafe-inline'; img-src 'self' data:;
Content-Type: text/event-stream
X-Accel-Buffering: no
/* snapshot: Test_handler_streamLogs_error_std */
HTTP/1.1 400 Bad Request
Connection: close
Content-Security-Policy: default-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data:;
Content-Security-Policy: default-src 'self' 'wasm-unsafe-eval' blob: https://cdn.jsdelivr.net https://*.duckdb.org; style-src 'self' 'unsafe-inline'; img-src 'self' data:;
Content-Type: text/plain; charset=utf-8
X-Content-Type-Options: nosniff
@@ -181,7 +181,7 @@ Connection: close
Cache-Control: no-transform
Cache-Control: no-cache
Connection: keep-alive
Content-Security-Policy: default-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data:;
Content-Security-Policy: default-src 'self' 'wasm-unsafe-eval' blob: https://cdn.jsdelivr.net https://*.duckdb.org; style-src 'self' 'unsafe-inline'; img-src 'self' data:;
Content-Type: text/event-stream
X-Accel-Buffering: no
@@ -197,7 +197,7 @@ Connection: close
Cache-Control: no-transform
Cache-Control: no-cache
Connection: keep-alive
Content-Security-Policy: default-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data:;
Content-Security-Policy: default-src 'self' 'wasm-unsafe-eval' blob: https://cdn.jsdelivr.net https://*.duckdb.org; style-src 'self' 'unsafe-inline'; img-src 'self' data:;
Content-Type: text/event-stream
X-Accel-Buffering: no
@@ -210,7 +210,7 @@ Connection: close
Cache-Control: no-transform
Cache-Control: no-cache
Connection: keep-alive
Content-Security-Policy: default-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data:;
Content-Security-Policy: default-src 'self' 'wasm-unsafe-eval' blob: https://cdn.jsdelivr.net https://*.duckdb.org; style-src 'self' 'unsafe-inline'; img-src 'self' data:;
Content-Type: text/event-stream
X-Accel-Buffering: no

5
internal/web/csp.go
View File

@@ -6,7 +6,10 @@ import (
func cspHeaders(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Security-Policy", "default-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data:;")
w.Header().Set(
"Content-Security-Policy",
"default-src 'self' 'wasm-unsafe-eval' blob: https://cdn.jsdelivr.net https://*.duckdb.org; style-src 'self' 'unsafe-inline'; img-src 'self' data:;",
)
next.ServeHTTP(w, r)
})
}

26
internal/web/logs.go
View File

@@ -120,6 +120,13 @@ func (h *handler) fetchLogsBetweenDates(w http.ResponseWriter, r *http.Request)
}
}
onlyComplex := r.URL.Query().Has("jsonOnly")
everything := r.URL.Query().Has("everything")
if everything {
from = time.Time{}
to = time.Now()
}
minimum := 0
if r.URL.Query().Has("minimum") {
minimum, err = strconv.Atoi(r.URL.Query().Get("minimum"))
@@ -134,6 +141,7 @@ func (h *handler) fetchLogsBetweenDates(w http.ResponseWriter, r *http.Request)
}
}
encoder := json.NewEncoder(w)
for {
if buffer.Len() > minimum {
break
@@ -154,12 +162,23 @@ func (h *handler) fetchLogsBetweenDates(w http.ResponseWriter, r *http.Request)
buffer.Push(event)
}
} else {
buffer.Push(event)
if onlyComplex {
if _, ok := event.Message.(string); ok {
continue
}
}
if everything {
if err := encoder.Encode(event); err != nil {
log.Error().Err(err).Msg("error encoding log event")
}
} else {
buffer.Push(event)
}
}
}
if from.Before(containerService.Container.Created) {
log.Debug().Msg("reached beginning of logs")
if everything || from.Before(containerService.Container.Created) {
break
}
@@ -169,7 +188,6 @@ func (h *handler) fetchLogsBetweenDates(w http.ResponseWriter, r *http.Request)
log.Debug().Int("buffer_size", buffer.Len()).Msg("sending logs to client")
encoder := json.NewEncoder(w)
for _, event := range buffer.Data() {
if err := encoder.Encode(event); err != nil {
log.Error().Err(err).Msg("error encoding log event")

1
package.json
View File

@@ -29,6 +29,7 @@
"docs:preview": "vitepress preview docs"
},
"dependencies": {
"@duckdb/duckdb-wasm": "1.28.1-dev106.0",
"@iconify-json/carbon": "^1.2.1",
"@iconify-json/cil": "^1.2.0",
"@iconify-json/ic": "^1.2.0",

680
pnpm-lock.yaml generated
View File

File diff suppressed because it is too large Load Diff