From ba915fe84b57b05ab3b7540333d3280b3ab147ac Mon Sep 17 00:00:00 2001 From: Sebastian Benjamin Date: Thu, 15 May 2025 14:02:05 -0700 Subject: [PATCH 1/3] Stream JSON results to client --- .../components/VariantTableWidget.tsx | 31 +++++-- jbrowse/src/client/JBrowse/utils.ts | 81 +++++++++++++------ .../org/labkey/jbrowse/JBrowseController.java | 10 ++- .../labkey/jbrowse/JBrowseLuceneSearch.java | 28 +++---- 4 files changed, 101 insertions(+), 49 deletions(-) diff --git a/jbrowse/src/client/JBrowse/VariantSearch/components/VariantTableWidget.tsx b/jbrowse/src/client/JBrowse/VariantSearch/components/VariantTableWidget.tsx index e50da12a9..98f6ac0f9 100644 --- a/jbrowse/src/client/JBrowse/VariantSearch/components/VariantTableWidget.tsx +++ b/jbrowse/src/client/JBrowse/VariantSearch/components/VariantTableWidget.tsx @@ -68,7 +68,6 @@ const VariantTableWidget = observer(props => { return obj })) - setTotalHits(data.totalHits) setDataLoaded(true) } @@ -96,12 +95,35 @@ const VariantTableWidget = observer(props => { currentUrl.searchParams.set("sortDirection", sort.toString()); if (pushToHistory) { - window.history.pushState(null, "", currentUrl.toString()); + window.history.pushState(null, "", currentUrl.toString()); } setFilters(passedFilters); - setDataLoaded(false) - fetchLuceneQuery(passedFilters, sessionId, trackGUID, page, pageSize, field, sort, (json)=>{handleSearch(json)}, (error) => {setDataLoaded(true); setError(error)}); + setDataLoaded(false); + setFeatures([]); + + fetchLuceneQuery( + passedFilters, + sessionId, + trackGUID, + page, + pageSize, + field, + sort, + (row) => { + setFeatures(prev => { + row.id = prev.length; + row.trackId = trackId; + return [...prev, row]; + }); + }, + () => setDataLoaded(true), + (error) => { + console.error("Stream error:", error); + setError(error); + setDataLoaded(true); + } + ); } const handleExport = () => { @@ -274,7 +296,6 @@ const VariantTableWidget = observer(props => { const [filterModalOpen, setFilterModalOpen] = useState(false); const [filters, setFilters] = useState([]); - const [totalHits, setTotalHits] = useState(0); const [fieldTypeInfo, setFieldTypeInfo] = useState([]); const [allowedGroupNames, setAllowedGroupNames] = useState([]); const [promotedFilters, setPromotedFilters] = useState>(null); diff --git a/jbrowse/src/client/JBrowse/utils.ts b/jbrowse/src/client/JBrowse/utils.ts index 0b51d8ef7..1b5f5a7d3 100644 --- a/jbrowse/src/client/JBrowse/utils.ts +++ b/jbrowse/src/client/JBrowse/utils.ts @@ -328,23 +328,24 @@ export function serializeLocationToEncodedSearchString(contig, start, end) { return createEncodedFilterString(filters) } -export async function fetchLuceneQuery(filters, sessionId, trackGUID, offset, pageSize, sortField, sortReverseString, successCallback, failureCallback) { +export async function fetchLuceneQuery(filters, sessionId, trackGUID, offset, pageSize, sortField, sortReverseString, + handleRow, handleComplete, handleError) { if (!offset) { offset = 0 } if (!sessionId) { - failureCallback("There was an error: " + "Lucene query: no session ID") + handleError("There was an error: " + "Lucene query: no session ID") return } if (!trackGUID) { - failureCallback("There was an error: " + "Lucene query: no track ID") + handleError("There was an error: " + "Lucene query: no track ID") return } if (!filters) { - failureCallback("There was an error: " + "Lucene query: no filters") + handleError("There was an error: " + "Lucene query: no filters") return } @@ -358,27 +359,59 @@ export async function fetchLuceneQuery(filters, sessionId, trackGUID, offset, pa sortReverse = false } - return Ajax.request({ - url: ActionURL.buildURL('jbrowse', 'luceneQuery.api'), - method: 'GET', - success: async function(res){ - let jsonRes = JSON.parse(res.response); - successCallback(jsonRes) - }, - failure: function(res) { - console.error("There was an error: " + res.status + "\n Status Body: " + res.responseText + "\n Session ID:" + sessionId) - failureCallback("There was an error: status " + res.status) - }, - params: { - "searchString": encoded, - "sessionId": sessionId, - "trackId": trackGUID, - "offset": offset, - "pageSize": pageSize, - "sortField": sortField ?? "genomicPosition", - "sortReverse": sortReverse - }, + const params = new URLSearchParams({ + searchString: encoded, + sessionId, + trackId: trackGUID, + offset: offset, + pageSize: pageSize, + sortField: sortField ?? "genomicPosition", + sortReverse: sortReverse, }); + + try { + const response = await fetch(`/jbrowse/luceneQuery.api?${params.toString()}`); + if (!response.ok || !response.body) { + throw new Error(`HTTP error ${response.status}`); + } + + const reader = response.body.getReader(); + const decoder = new TextDecoder("utf-8"); + let buffer = ''; + + while (true) { + const { done, value } = await reader.read(); + if (done) break; + + buffer += decoder.decode(value, { stream: true }); + + let boundary; + while ((boundary = buffer.indexOf('\n')) >= 0) { + const line = buffer.slice(0, boundary).trim(); + buffer = buffer.slice(boundary + 1); + if (line) { + try { + const parsed = JSON.parse(line); + handleRow(parsed); + } catch (err) { + console.error('Failed to parse line:', line, err); + } + } + } + } + + if (buffer.trim()) { + try { + handleRow(JSON.parse(buffer)); + } catch (err) { + console.error('Final line parse error:', buffer, err); + } + } + + handleComplete(); + } catch (error) { + handleError(error.toString()); + } } export class FieldModel { diff --git a/jbrowse/src/org/labkey/jbrowse/JBrowseController.java b/jbrowse/src/org/labkey/jbrowse/JBrowseController.java index 6dc297b95..f19c67247 100644 --- a/jbrowse/src/org/labkey/jbrowse/JBrowseController.java +++ b/jbrowse/src/org/labkey/jbrowse/JBrowseController.java @@ -962,14 +962,18 @@ public ApiResponse execute(LuceneQueryForm form, BindException errors) try { - return new ApiSimpleResponse(searcher.doSearchJSON( + HttpServletResponse response = getViewContext().getResponse(); + response.setContentType("application/x-ndjson"); + searcher.doSearchJSON( getUser(), PageFlowUtil.decode(form.getSearchString()), form.getPageSize(), form.getOffset(), form.getSortField(), - form.getSortReverse() - )); + form.getSortReverse(), + response + ); + return null; } catch (Exception e) { diff --git a/jbrowse/src/org/labkey/jbrowse/JBrowseLuceneSearch.java b/jbrowse/src/org/labkey/jbrowse/JBrowseLuceneSearch.java index e50875cb4..82249a343 100644 --- a/jbrowse/src/org/labkey/jbrowse/JBrowseLuceneSearch.java +++ b/jbrowse/src/org/labkey/jbrowse/JBrowseLuceneSearch.java @@ -1,6 +1,7 @@ package org.labkey.jbrowse; import jakarta.servlet.http.HttpServletResponse; +import org.apache.catalina.connector.Response; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.Logger; import org.apache.lucene.analysis.Analyzer; @@ -202,9 +203,9 @@ public String extractFieldName(String queryString) return parts.length > 0 ? parts[0].trim() : null; } - public JSONObject doSearchJSON(User u, String searchString, final int pageSize, final int offset, String sortField, boolean sortReverse) throws IOException, ParseException { + public void doSearchJSON(User u, String searchString, final int pageSize, final int offset, String sortField, boolean sortReverse, HttpServletResponse response) throws IOException, ParseException { SearchConfig searchConfig = createSearchConfig(u, searchString, pageSize, offset, sortField, sortReverse); - return paginateJSON(searchConfig); + paginateJSON(searchConfig, response); } public void doSearchCSV(User u, String searchString, String sortField, boolean sortReverse, HttpServletResponse response) throws IOException, ParseException { @@ -330,32 +331,26 @@ else if (numericQueryParserFields.containsKey(fieldName)) return new SearchConfig(cacheEntry, query, pageSize, offset, sort, fieldsList); } - private JSONObject paginateJSON(SearchConfig c) throws IOException, ParseException { + private void paginateJSON(SearchConfig c, HttpServletResponse response) throws IOException, ParseException { IndexSearcher searcher = c.cacheEntry.indexSearcher; TopDocs topDocs; + PrintWriter writer = response.getWriter(); if (c.offset == 0) { topDocs = searcher.search(c.query, c.pageSize, c.sort); } else { TopFieldDocs prev = searcher.search(c.query, c.pageSize * c.offset, c.sort); - long totalHits = prev.totalHits.value; ScoreDoc[] prevHits = prev.scoreDocs; if (prevHits.length < c.pageSize * c.offset) { - JSONObject results = new JSONObject(); - results.put("data", Collections.emptyList()); - results.put("totalHits", totalHits); - return results; + return; } ScoreDoc lastDoc = prevHits[c.pageSize * c.offset - 1]; topDocs = searcher.searchAfter(lastDoc, c.query, c.pageSize, c.sort); } - JSONObject results = new JSONObject(); - List data = new ArrayList<>(topDocs.scoreDocs.length); - for (ScoreDoc sd : topDocs.scoreDocs) { Document doc = searcher.storedFields().document(sd.doc); @@ -366,12 +361,10 @@ private JSONObject paginateJSON(SearchConfig c) throws IOException, ParseExcepti String[] vals = doc.getValues(name); elem.put(name, vals.length > 1 ? Arrays.asList(vals) : vals[0]); } - data.add(elem); - } - results.put("data", data); - results.put("totalHits", topDocs.totalHits.value); - return results; + writer.println(elem); + writer.flush(); + } } private void exportCSV(SearchConfig c, HttpServletResponse response) throws IOException @@ -648,8 +641,9 @@ public void cacheDefaultQuery() { try { + HttpServletResponse response = new Response(); JBrowseLuceneSearch.clearCache(_jsonFile.getObjectId()); - doSearchJSON(_user, ALL_DOCS, 100, 0, GENOMIC_POSITION, false); + doSearchJSON(_user, ALL_DOCS, 100, 0, GENOMIC_POSITION, false, response); } catch (ParseException | IOException e) { From 5d3bf8db50081b151959e6145a748c9322318bc9 Mon Sep 17 00:00:00 2001 From: Sebastian Benjamin Date: Mon, 19 May 2025 10:17:19 -0700 Subject: [PATCH 2/3] Use ActionURL to build lucene queries --- jbrowse/src/client/JBrowse/utils.ts | 7 ++++--- jbrowse/src/org/labkey/jbrowse/JBrowseLuceneSearch.java | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/jbrowse/src/client/JBrowse/utils.ts b/jbrowse/src/client/JBrowse/utils.ts index 1b5f5a7d3..ad1870022 100644 --- a/jbrowse/src/client/JBrowse/utils.ts +++ b/jbrowse/src/client/JBrowse/utils.ts @@ -359,7 +359,7 @@ export async function fetchLuceneQuery(filters, sessionId, trackGUID, offset, pa sortReverse = false } - const params = new URLSearchParams({ + const params = { searchString: encoded, sessionId, trackId: trackGUID, @@ -367,10 +367,11 @@ export async function fetchLuceneQuery(filters, sessionId, trackGUID, offset, pa pageSize: pageSize, sortField: sortField ?? "genomicPosition", sortReverse: sortReverse, - }); + }; try { - const response = await fetch(`/jbrowse/luceneQuery.api?${params.toString()}`); + const url = ActionURL.buildURL('jbrowse', 'luceneQuery.api', null, params); + const response = await fetch(url); if (!response.ok || !response.body) { throw new Error(`HTTP error ${response.status}`); } diff --git a/jbrowse/src/org/labkey/jbrowse/JBrowseLuceneSearch.java b/jbrowse/src/org/labkey/jbrowse/JBrowseLuceneSearch.java index 82249a343..34d8c82f5 100644 --- a/jbrowse/src/org/labkey/jbrowse/JBrowseLuceneSearch.java +++ b/jbrowse/src/org/labkey/jbrowse/JBrowseLuceneSearch.java @@ -363,8 +363,9 @@ private void paginateJSON(SearchConfig c, HttpServletResponse response) throws I } writer.println(elem); - writer.flush(); } + + writer.flush(); } private void exportCSV(SearchConfig c, HttpServletResponse response) throws IOException From 324a8560997e8fed1dbc4a03425f7994c33604d1 Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 21 May 2025 09:37:05 -0700 Subject: [PATCH 3/3] Switch to ExportAction --- .../src/org/labkey/jbrowse/JBrowseController.java | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/jbrowse/src/org/labkey/jbrowse/JBrowseController.java b/jbrowse/src/org/labkey/jbrowse/JBrowseController.java index f19c67247..d7c75b483 100644 --- a/jbrowse/src/org/labkey/jbrowse/JBrowseController.java +++ b/jbrowse/src/org/labkey/jbrowse/JBrowseController.java @@ -28,6 +28,7 @@ import org.json.JSONObject; import org.labkey.api.action.ApiResponse; import org.labkey.api.action.ApiSimpleResponse; +import org.labkey.api.action.ExportAction; import org.labkey.api.action.MutatingApiAction; import org.labkey.api.action.ReadOnlyApiAction; import org.labkey.api.action.SimpleApiJsonForm; @@ -944,10 +945,10 @@ else if (!isValidUUID(form.getTrackId())) } @RequiresPermission(ReadPermission.class) - public static class LuceneQueryAction extends ReadOnlyApiAction + public static class LuceneQueryAction extends ExportAction { @Override - public ApiResponse execute(LuceneQueryForm form, BindException errors) + public void export(LuceneQueryForm form, HttpServletResponse response, BindException errors) throws Exception { JBrowseLuceneSearch searcher; try @@ -957,12 +958,11 @@ public ApiResponse execute(LuceneQueryForm form, BindException errors) catch (IllegalArgumentException e) { errors.reject(ERROR_MSG, e.getMessage()); - return null; + return; } try { - HttpServletResponse response = getViewContext().getResponse(); response.setContentType("application/x-ndjson"); searcher.doSearchJSON( getUser(), @@ -973,18 +973,16 @@ public ApiResponse execute(LuceneQueryForm form, BindException errors) form.getSortReverse(), response ); - return null; } catch (Exception e) { _log.error("Error in JBrowse lucene query", e); errors.reject(ERROR_MSG, e.getMessage()); - return null; } } @Override - public void validateForm(LuceneQueryForm form, Errors errors) + public void validate(LuceneQueryForm form, BindException errors) { if ((form.getSearchString() == null || form.getSessionId() == null || form.getTrackId() == null)) {