feat: retries to fetch more logs if no logs are fetched when scrolling up (#3230)

Makefile

@@ -22,7 +22,7 @@ fake_assets:
 
 .PHONY: test
 test: fake_assets generate
-	go test -cover -race -count 1 -timeout 20s ./...
+	go test -cover -race -count 1 -timeout 40s ./...
 
 .PHONY: build
 build: dist generate

assets/composable/eventStreams.ts

@@ -210,7 +210,7 @@ function useLogStream(url: Ref<string>, loadMoreUrl?: Ref<string>) {
       const stopWatcher = watchOnce(url, () => abortController.abort("stream changed"));
       const logs = await (
         await fetch(
-          `${loadMoreUrl.value}&${new URLSearchParams({ from: from.toISOString(), to: to.toISOString() }).toString()}`,
+          `${loadMoreUrl.value}&${new URLSearchParams({ from: from.toISOString(), to: to.toISOString(), fill: "1" }).toString()}`,
           { signal },
         )
       ).text();

internal/utils/ring_buffer.go

@@ -45,6 +45,12 @@ func (r *RingBuffer[T]) Push(data T) {
 	}
 }
 
+func (r *RingBuffer[T]) Len() int {
+	r.mutex.RLock()
+	defer r.mutex.RUnlock()
+	return len(r.data)
+}
+
 func (r *RingBuffer[T]) Clear() {
 	r.mutex.Lock()
 	defer r.mutex.Unlock()

internal/web/__snapshots__/web.snapshot

@@ -89,6 +89,15 @@ Content-Type: application/x-jsonl; charset=UTF-8
 {"m":"INFO Testing stdout logs...","ts":1589396137772,"id":466600245,"l":"info","s":"stdout","c":"123456"}
 {"m":"INFO Testing stderr logs...","ts":1589396197772,"id":1101501603,"l":"info","s":"stderr","c":"123456"}
 
+/* snapshot: Test_handler_between_dates_with_fill */
+HTTP/1.1 200 OK
+Connection: close
+Content-Security-Policy: default-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data:;
+Content-Type: application/x-jsonl; charset=UTF-8
+
+{"m":"INFO Testing stdout logs...","ts":1589396137772,"id":466600245,"l":"info","s":"stdout","c":"123456"}
+{"m":"INFO Testing stderr logs...","ts":1589396197772,"id":1101501603,"l":"info","s":"stderr","c":"123456"}
+
 /* snapshot: Test_handler_download_logs */
 INFO Testing logs...
 

internal/web/logs.go

@@ -103,6 +103,13 @@ func (h *handler) fetchLogsBetweenDates(w http.ResponseWriter, r *http.Request)
 		return
 	}
 
+	buffer := utils.NewRingBuffer[*docker.LogEvent](500)
+	delta := to.Sub(from)
+
+	for {
+		if buffer.Len() > 0 {
+			break
+		}
 		events, err := containerService.LogsBetweenDates(r.Context(), from, to, stdTypes)
 		if err != nil {
 			log.Error().Err(err).Msg("error fetching logs")
@@ -110,12 +117,17 @@ func (h *handler) fetchLogsBetweenDates(w http.ResponseWriter, r *http.Request)
 			return
 		}
 
-	buffer := utils.NewRingBuffer[*docker.LogEvent](500)
-
 		for event := range events {
 			buffer.Push(event)
 		}
 
+		if !r.URL.Query().Has("fill") { // only auto fill if fill query parameter is set
+			break
+		}
+
+		from = from.Add(-delta)
+	}
+
 	encoder := json.NewEncoder(w)
 	for _, event := range buffer.Data() {
 		if err := encoder.Encode(event); err != nil {

internal/web/logs_test.go

@@ -243,7 +243,7 @@ func Test_handler_between_dates(t *testing.T) {
 	require.NoError(t, err, "NewRequest should not return an error.")
 
 	from, _ := time.Parse(time.RFC3339, "2018-01-01T00:00:00Z")
-	to, _ := time.Parse(time.RFC3339, "2018-01-01T010:00:00Z")
+	to, _ := time.Parse(time.RFC3339, "2018-01-01T10:00:00Z")
 
 	q := req.URL.Query()
 	q.Add("from", from.Format(time.RFC3339))
@@ -276,6 +276,51 @@ func Test_handler_between_dates(t *testing.T) {
 	mockedClient.AssertExpectations(t)
 }
 
+func Test_handler_between_dates_with_fill(t *testing.T) {
+	id := "123456"
+	req, err := http.NewRequest("GET", "/api/hosts/localhost/containers/"+id+"/logs", nil)
+	require.NoError(t, err, "NewRequest should not return an error.")
+
+	from, _ := time.Parse(time.RFC3339, "2018-01-01T00:00:00Z")
+	to, _ := time.Parse(time.RFC3339, "2018-01-01T10:00:00Z")
+
+	q := req.URL.Query()
+	q.Add("from", from.Format(time.RFC3339))
+	q.Add("to", to.Format(time.RFC3339))
+	q.Add("stdout", "true")
+	q.Add("stderr", "true")
+	q.Add("fill", "true")
+
+	req.URL.RawQuery = q.Encode()
+
+	mockedClient := new(MockedClient)
+
+	first := makeMessage("2020-05-13T18:55:37.772853839Z INFO Testing stdout logs...\n", docker.STDOUT)
+	second := makeMessage("2020-05-13T18:56:37.772853839Z INFO Testing stderr logs...\n", docker.STDERR)
+	data := append(first, second...)
+
+	mockedClient.On("ContainerLogsBetweenDates", mock.Anything, id, from, to, docker.STDALL).
+		Return(io.NopCloser(bytes.NewReader([]byte{})), nil).
+		Once()
+	mockedClient.On("ContainerLogsBetweenDates", mock.Anything, id, time.Date(2017, time.December, 31, 14, 0, 0, 0, time.UTC), to, docker.STDALL).
+		Return(io.NopCloser(bytes.NewReader(data)), nil).
+		Once()
+	mockedClient.On("FindContainer", id).Return(docker.Container{ID: id}, nil)
+	mockedClient.On("Host").Return(docker.Host{
+		ID: "localhost",
+	})
+	mockedClient.On("ListContainers").Return([]docker.Container{
+		{ID: id, Name: "test", Host: "localhost", State: "running"},
+	}, nil)
+	mockedClient.On("ContainerEvents", mock.Anything, mock.AnythingOfType("chan<- docker.ContainerEvent")).Return(nil)
+
+	handler := createDefaultHandler(mockedClient)
+	rr := httptest.NewRecorder()
+	handler.ServeHTTP(rr, req)
+	abide.AssertHTTPResponse(t, t.Name(), rr.Result())
+	mockedClient.AssertExpectations(t)
+}
+
 func makeMessage(message string, stream docker.StdType) []byte {
 	data := make([]byte, 8)
 	binary.BigEndian.PutUint32(data[4:], uint32(len(message)))