Fetch more (#209)

* Adds code to fetch more

* Adds working in progress

* Adds debugging test

* Cleans up and creates a new component

* Adds debug logs

* Adds debounce for messages

* Fixes scrolling

* Fixes go code to handle length

* Fixes tests

* Adds loader

* Fixes tests

assets/components/InfiniteLoader.vue

@@ -0,0 +1,38 @@
+<template lang="html">
+  <div ref="observer" class="control" :class="{ 'is-loading': isLoading }"></div>
+</template>
+
+<script>
+export default {
+  name: "InfiniteLoader",
+  data() {
+    return {
+      scrollingParent: null,
+      isLoading: false
+    };
+  },
+  props: {
+    onLoadMore: Function,
+    enabled: Boolean
+  },
+  mounted() {
+    this.scrollingParent = this.$el.closest("[data-scrolling]");
+    const intersectionObserver = new IntersectionObserver(
+      async entries => {
+        if (entries[0].intersectionRatio <= 0) return;
+        if (this.onLoadMore && this.enabled) {
+          const previousHeight = this.scrollingParent.scrollHeight;
+          this.isLoading = true;
+          await this.onLoadMore();
+          this.isLoading = false;
+          this.$nextTick(() => (this.scrollingParent.scrollTop += this.scrollingParent.scrollHeight - previousHeight));
+        }
+      },
+      { threshholds: 1 }
+    );
+
+    intersectionObserver.observe(this.$refs.observer);
+  }
+};
+</script>
+<style scoped lang="scss"></style>

assets/components/LogEventSource.spec.js

@@ -3,15 +3,26 @@ import { sources } from "eventsourcemock";
 import { shallowMount, mount, createLocalVue } from "@vue/test-utils";
 import Vuex from "vuex";
 import MockDate from "mockdate";
+import debounce from "lodash.debounce";
 import LogEventSource from "./LogEventSource.vue";
 import LogViewer from "./LogViewer.vue";
 
+jest.mock("lodash.debounce", () => jest.fn(fn => fn));
+
 describe("<LogEventSource />", () => {
   beforeEach(() => {
     global.BASE_PATH = "";
     global.EventSource = EventSource;
     MockDate.set("6/12/2019", 0);
     window.scrollTo = jest.fn();
+
+    const observe = jest.fn();
+    const unobserve = jest.fn();
+    global.IntersectionObserver = jest.fn(() => ({
+      observe,
+      unobserve
+    }));
+    debounce.mockClear();
   });
 
   afterEach(() => MockDate.reset());
@@ -48,7 +59,17 @@ describe("<LogEventSource />", () => {
 
   test("renders correctly", async () => {
     const wrapper = createLogEventSource();
-    expect(wrapper.element).toMatchSnapshot();
+    expect(wrapper.element).toMatchInlineSnapshot(`
+      <div>
+        <div
+          class="control"
+        />
+         
+        <ul
+          class="events"
+        />
+      </div>
+    `);
   });
 
   test("should connect to EventSource", async () => {
@@ -68,15 +89,15 @@ describe("<LogEventSource />", () => {
     const wrapper = createLogEventSource();
     sources["/api/logs/stream?id=abc"].emitOpen();
     sources["/api/logs/stream?id=abc"].emitMessage({ data: `2019-06-12T10:55:42.459034602Z "This is a message."` });
+
     const [message, _] = wrapper.vm.messages;
     const { key, ...messageWithoutKey } = message;
 
-    expect(key).toBeGreaterThanOrEqual(0);
-
+    expect(key).toBe("2019-06-12T10:55:42.459034602Z");
     expect(messageWithoutKey).toMatchInlineSnapshot(`
       Object {
         "date": 2019-06-12T10:55:42.459Z,
-        "message": " \\"This is a message.\\"",
+        "message": "\\"This is a message.\\"",
       }
     `);
   });
@@ -89,12 +110,12 @@ describe("<LogEventSource />", () => {
 
     const { key, ...messageWithoutKey } = message;
 
-    expect(key).toBeGreaterThanOrEqual(0);
+    expect(key).toBe("2019-06-12T10:55:42.459034602Z");
 
     expect(messageWithoutKey).toMatchInlineSnapshot(`
       Object {
         "date": 2019-06-12T10:55:42.459Z,
-        "message": " \\"This is a message.\\"",
+        "message": "\\"This is a message.\\"",
       }
     `);
   });
@@ -106,7 +127,7 @@ describe("<LogEventSource />", () => {
 
     expect(wrapper.find("ul.events")).toMatchInlineSnapshot(`
       <ul class="events">
-        <li><span class="date">today at 10:55 AM</span> <span class="text"> "This is a message."</span></li>
+        <li><span class="date">today at 10:55 AM</span> <span class="text">"This is a message."</span></li>
       </ul>
     `);
   });
@@ -120,7 +141,7 @@ describe("<LogEventSource />", () => {
 
     expect(wrapper.find("ul.events")).toMatchInlineSnapshot(`
       <ul class="events">
-        <li><span class="date">today at 10:55 AM</span> <span class="text"> <span style="color:#000">black<span style="color:#AAA">white</span></span></span></li>
+        <li><span class="date">today at 10:55 AM</span> <span class="text"><span style="color:#000">black<span style="color:#AAA">white</span></span></span></li>
       </ul>
     `);
   });
@@ -134,7 +155,7 @@ describe("<LogEventSource />", () => {
 
     expect(wrapper.find("ul.events")).toMatchInlineSnapshot(`
       <ul class="events">
-        <li><span class="date">today at 10:55 AM</span> <span class="text"> &lt;test&gt;foo bar&lt;/test&gt;</span></li>
+        <li><span class="date">today at 10:55 AM</span> <span class="text">&lt;test&gt;foo bar&lt;/test&gt;</span></li>
       </ul>
     `);
   });
@@ -151,7 +172,7 @@ describe("<LogEventSource />", () => {
 
     expect(wrapper.find("ul.events")).toMatchInlineSnapshot(`
       <ul class="events">
-        <li><span class="date">today at 10:55 AM</span> <span class="text"> This is a <mark>test</mark> &lt;hi&gt;&lt;/hi&gt;</span></li>
+        <li><span class="date">today at 10:55 AM</span> <span class="text">This is a <mark>test</mark> &lt;hi&gt;&lt;/hi&gt;</span></li>
       </ul>
     `);
   });

assets/components/LogEventSource.vue

@@ -1,15 +1,18 @@
 <template lang="html">
   <div>
+    <infinite-loader :onLoadMore="loadOlderLogs" :enabled="messages.length > 100"></infinite-loader>
     <slot v-bind:messages="messages"></slot>
   </div>
 </template>
 
 <script>
-let nextId = 0;
+import debounce from "lodash.debounce";
+import InfiniteLoader from "./InfiniteLoader";
+
 function parseMessage(data) {
   const date = new Date(data.substring(0, 30));
-  const message = data.substring(30);
-  const key = nextId++;
+  const key = data.substring(0, 30);
+  const message = data.substring(30).trim();
   return {
     key,
     date,
@@ -20,9 +23,13 @@ function parseMessage(data) {
 export default {
   props: ["id"],
   name: "LogEventSource",
+  components: {
+    InfiniteLoader
+  },
   data() {
     return {
-      messages: []
+      messages: [],
+      buffer: []
     };
   },
   created() {
@@ -37,11 +44,37 @@ export default {
         this.es = null;
       }
       this.es = new EventSource(`${BASE_PATH}/api/logs/stream?id=${this.id}`);
-      this.es.onmessage = e => this.messages.push(parseMessage(e.data));
-      this.es.onerror = function(e) {
-        console.log("EventSource failed." + e);
+      const flushBuffer = debounce(
+        () => {
+          this.messages.push(...this.buffer);
+          this.buffer = [];
+        },
+        250,
+        { maxWait: 1000 }
+      );
+      this.es.onmessage = e => {
+        this.buffer.push(parseMessage(e.data));
+        flushBuffer();
       };
+      this.es.onerror = e => console.log("EventSource failed." + e);
       this.$once("hook:beforeDestroy", () => this.es.close());
+    },
+    async loadOlderLogs() {
+      if (this.messages.length < 100) return;
+
+      const to = this.messages[0].date;
+      const from = new Date(to);
+      from.setMinutes(from.getMinutes() - 10);
+      const logs = await (
+        await fetch(`/api/logs?id=${this.id}&from=${from.toISOString()}&to=${to.toISOString()}`)
+      ).text();
+      if (logs) {
+        const newMessages = logs
+          .trim()
+          .split("\n")
+          .map(line => parseMessage(line));
+        this.messages.unshift(...newMessages);
+      }
     }
   },
   watch: {

assets/components/LogViewer.vue

@@ -47,6 +47,7 @@ export default {
             }));
         } catch (e) {
           if (e instanceof SyntaxError) {
+            console.info(`Ignoring SytaxError from search.`, e);
             return messages;
           }
           throw e;

assets/components/ScrollableLogsWithSource.vue

@@ -0,0 +1,19 @@
+<template lang="html">
+  <scrollable-view>
+    <log-viewer-with-source :id="id"></log-viewer-with-source>
+  </scrollable-view>
+</template>
+
+<script>
+import ScrollableView from "./ScrollableView";
+import LogViewerWithSource from "./LogViewerWithSource";
+
+export default {
+  props: ["id"],
+  name: "ScrollableLogsWithSource",
+  components: {
+    LogViewerWithSource,
+    ScrollableView
+  }
+};
+</script>

assets/components/ScrollableView.vue

@@ -3,7 +3,7 @@
     <header v-if="$slots.header">
       <slot name="header"></slot>
     </header>
-    <main ref="content" @scroll.passive="onScroll">
+    <main ref="content" @scroll.passive="onScroll" data-scrolling>
       <slot></slot>
     </main>
     <div class="scroll-bar-notification">
@@ -40,6 +40,7 @@ export default {
       }
     }).observe(content, { childList: true, subtree: true });
   },
+
   methods: {
     scrollToBottom(behavior = "instant") {
       const { content } = this.$refs;
@@ -54,8 +55,7 @@ export default {
       const { content } = this.$refs;
       this.paused = content.scrollTop + content.clientHeight + 1 < content.scrollHeight;
     }
-  },
-  watch: {}
+  }
 };
 </script>
 <style scoped lang="scss">

assets/components/__snapshots__/LogEventSource.spec.js.snap

@@ -1,9 +0,0 @@
-// Jest Snapshot v1, https://goo.gl/fbAQLP
-
-exports[`<LogEventSource /> renders correctly 1`] = `
-<div>
-  <ul
-    class="events"
-  />
-</div>
-`;

assets/pages/Container.vue

@@ -1,23 +1,19 @@
 <template lang="html">
   <div>
     <search></search>
-    <scrollable-view>
-      <log-viewer-with-source :id="id"></log-viewer-with-source>
-    </scrollable-view>
+    <scrollable-logs-with-source :id="id"></scrollable-logs-with-source>
   </div>
 </template>
 
 <script>
-import LogViewerWithSource from "../components/LogViewerWithSource";
+import ScrollableLogsWithSource from "../components/ScrollableLogsWithSource";
 import Search from "../components/Search";
-import ScrollableView from "../components/ScrollableView";
 
 export default {
   props: ["id", "name"],
   name: "Container",
   components: {
-    LogViewerWithSource,
-    ScrollableView,
+    ScrollableLogsWithSource,
     Search
   },
   metaInfo() {
@@ -28,4 +24,3 @@ export default {
   }
 };
 </script>
-<style lang="scss" scoped></style>

assets/styles.scss

@@ -18,7 +18,3 @@ body {
 h1.title {
   font-family: "Gafata", sans-serif;
 }
-
-.column {
-  min-width: 0;
-}

docker/client.go

@@ -10,6 +10,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/events"
@@ -36,6 +37,7 @@ type Client interface {
 	FindContainer(string) (Container, error)
 	ContainerLogs(context.Context, string, int) (<-chan string, <-chan error)
 	Events(context.Context) (<-chan events.Message, <-chan error)
+	ContainerLogsBetweenDates(context.Context, string, time.Time, time.Time) ([]string, error)
 }
 
 // NewClient creates a new instance of Client
@@ -190,3 +192,45 @@ func (d *dockerClient) ContainerLogs(ctx context.Context, id string, tailSize in
 func (d *dockerClient) Events(ctx context.Context) (<-chan events.Message, <-chan error) {
 	return d.cli.Events(ctx, types.EventsOptions{})
 }
+
+func (d *dockerClient) ContainerLogsBetweenDates(ctx context.Context, id string, from time.Time, to time.Time) ([]string, error) {
+	options := types.ContainerLogsOptions{
+		ShowStdout: true,
+		ShowStderr: true,
+		Timestamps: true,
+		Since:      strconv.FormatInt(from.Unix(), 10),
+		Until:      strconv.FormatInt(to.Unix(), 10),
+	}
+	reader, _ := d.cli.ContainerLogs(ctx, id, options)
+
+	defer reader.Close()
+
+	var messages []string
+	hdr := make([]byte, 8)
+	var buffer bytes.Buffer
+
+	for {
+		_, err := reader.Read(hdr)
+		if err != nil {
+			if err == io.EOF {
+				break
+			} else {
+				return nil, err
+			}
+		}
+		count := binary.BigEndian.Uint32(hdr[4:])
+		_, err = io.CopyN(&buffer, reader, int64(count))
+
+		if err != nil {
+			if err == io.EOF {
+				break
+			} else {
+				return nil, err
+			}
+		}
+		messages = append(messages, strings.TrimSpace(buffer.String()))
+		buffer.Reset()
+	}
+
+	return messages, nil
+}

main.go

@@ -34,9 +34,9 @@ var (
 )
 
 type handler struct {
-	client docker.Client
+	client  docker.Client
 	showAll bool
-	box    packr.Box
+	box     packr.Box
 }
 
 func init() {
@@ -91,6 +91,7 @@ func createRoutes(base string, h *handler) *mux.Router {
 	s := r.PathPrefix(base).Subrouter()
 	s.HandleFunc("/api/containers.json", h.listContainers)
 	s.HandleFunc("/api/logs/stream", h.streamLogs)
+	s.HandleFunc("/api/logs", h.fetchLogsBetweenDates)
 	s.HandleFunc("/api/events/stream", h.streamEvents)
 	s.HandleFunc("/version", h.version)
 	s.PathPrefix("/").Handler(http.StripPrefix(base, http.HandlerFunc(h.index)))
@@ -108,9 +109,9 @@ func main() {
 
 	box := packr.NewBox("./static")
 	r := createRoutes(base, &handler{
-		client: dockerClient,
+		client:  dockerClient,
 		showAll: showAll,
-		box: box,
+		box:     box,
 	})
 	srv := &http.Server{Addr: addr, Handler: r}
 
@@ -170,6 +171,20 @@ func (h *handler) listContainers(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+func (h *handler) fetchLogsBetweenDates(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "text/plain; charset=UTF-8")
+
+	from, _ := time.Parse(time.RFC3339, r.URL.Query().Get("from"))
+	to, _ := time.Parse(time.RFC3339, r.URL.Query().Get("to"))
+	id := r.URL.Query().Get("id")
+
+	messages, _ := h.client.ContainerLogsBetweenDates(r.Context(), id, from, to)
+
+	for _, m := range messages {
+		fmt.Fprintln(w, m)
+	}
+}
+
 func (h *handler) streamLogs(w http.ResponseWriter, r *http.Request) {
 	id := r.URL.Query().Get("id")
 	if id == "" {

package-lock.json

@@ -7647,6 +7647,11 @@
       "integrity": "sha1-GVhwRQ9aExkkeN9Lw9I9LeoZB7Y=",
       "dev": true
     },
+    "lodash.debounce": {
+      "version": "4.0.8",
+      "resolved": "https://registry.npmjs.org/lodash.debounce/-/lodash.debounce-4.0.8.tgz",
+      "integrity": "sha1-gteb/zCmfEAF/9XiUVMArZyk168="
+    },
     "lodash.memoize": {
       "version": "4.1.2",
       "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz",
@@ -10530,9 +10535,9 @@
       }
     },
     "splitpanes": {
-      "version": "2.1.1",
-      "resolved": "https://registry.npmjs.org/splitpanes/-/splitpanes-2.1.1.tgz",
-      "integrity": "sha512-K4jlE6gLKElsjtoUqzX6x0Uq4k7rnSBAMmxOJNWYTTIAuO7seC0+x3ADGpUgqtazMYdq0nAfWB9+NI/t/tW0mw=="
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/splitpanes/-/splitpanes-2.1.2.tgz",
+      "integrity": "sha512-IiVeC1wk7yVq4QZ3VTj6FbOTUJQmy/gFxpR1pDk67P+Pj8V0daUsPeoBNcKmcvJp2v3272GHSeLTNEyDCKRXSQ=="
     },
     "sprintf-js": {
       "version": "1.0.3",
@@ -11549,9 +11554,9 @@
       "dev": true
     },
     "vue": {
-      "version": "2.6.10",
-      "resolved": "https://registry.npmjs.org/vue/-/vue-2.6.10.tgz",
-      "integrity": "sha512-ImThpeNU9HbdZL3utgMCq0oiMzAkt1mcgy3/E6zWC/G6AaQoeuFdsl9nDhTDU3X1R6FK7nsIUuRACVcjI+A2GQ=="
+      "version": "2.6.11",
+      "resolved": "https://registry.npmjs.org/vue/-/vue-2.6.11.tgz",
+      "integrity": "sha512-VfPwgcGABbGAue9+sfrD4PuwFar7gPb1yl1UK1MwXoQPAw0BKSqWfoYCT/ThFrdEVWoI51dBuyCoiNU9bZDZxQ=="
     },
     "vue-hot-reload-api": {
       "version": "2.3.4",
@@ -11591,9 +11596,9 @@
       "integrity": "sha512-8iSa4mGNXBjyuSZFCCO4fiKfvzqk+mhL0lnKuGcQtO1eoj8nq3CmbEG8FwK5QqoqwDgsjsf1GDuisDX4cdb/aQ=="
     },
     "vue-template-compiler": {
-      "version": "2.6.10",
-      "resolved": "https://registry.npmjs.org/vue-template-compiler/-/vue-template-compiler-2.6.10.tgz",
-      "integrity": "sha512-jVZkw4/I/HT5ZMvRnhv78okGusqe0+qH2A0Em0Cp8aq78+NK9TII263CDVz2QXZsIT+yyV/gZc/j/vlwa+Epyg==",
+      "version": "2.6.11",
+      "resolved": "https://registry.npmjs.org/vue-template-compiler/-/vue-template-compiler-2.6.11.tgz",
+      "integrity": "sha512-KIq15bvQDrcCjpGjrAhx4mUlyyHfdmTaoNfeoATHLAiWB+MU3cx4lOzMwrnUh9cCxy0Lt1T11hAFY6TQgroUAA==",
       "dev": true,
       "requires": {
         "de-indent": "^1.0.2",

package.json

@@ -27,9 +27,10 @@
   "ansi-to-html": "^0.6.13",
   "bulma": "^0.8.0",
   "date-fns": "^2.8.1",
-  "splitpanes": "^2.1.1",
+  "lodash.debounce": "^4.0.8",
+  "splitpanes": "^2.1.2",
   "store": "^2.0.12",
-  "vue": "^2.6.10",
+  "vue": "^2.6.11",
   "vue-meta": "^2.3.1",
   "vue-router": "^3.1.3",
   "vuex": "^3.1.2"
@@ -54,7 +55,7 @@
   "sass": "^1.23.7",
   "vue-hot-reload-api": "^2.3.4",
   "vue-jest": "^3.0.5",
-  "vue-template-compiler": "^2.6.10"
+  "vue-template-compiler": "^2.6.11"
  },
  "husky": {
   "hooks": {