diff --git a/jbrowse/src/client/JBrowse/VariantSearch/components/VariantTableWidget.tsx b/jbrowse/src/client/JBrowse/VariantSearch/components/VariantTableWidget.tsx index c7a5ebb14..e3c711e78 100644 --- a/jbrowse/src/client/JBrowse/VariantSearch/components/VariantTableWidget.tsx +++ b/jbrowse/src/client/JBrowse/VariantSearch/components/VariantTableWidget.tsx @@ -13,6 +13,9 @@ import { GridToolbarExport } from '@mui/x-data-grid'; import SearchIcon from '@mui/icons-material/Search'; +import LinkIcon from '@mui/icons-material/Link'; +import DownloadIcon from '@mui/icons-material/Download' +import { ActionURL } from '@labkey/api'; import React, { useEffect, useState } from 'react'; import { getConf } from '@jbrowse/core/configuration'; import { AppBar, Box, Button, Dialog, Paper, Popover, Toolbar, Tooltip, Typography } from '@mui/material'; @@ -92,6 +95,27 @@ const VariantTableWidget = observer(props => { fetchLuceneQuery(passedFilters, sessionId, trackGUID, page, pageSize, field, sort, (json)=>{handleSearch(json)}, (error) => {setDataLoaded(true); setError(error)}); } + const handleExport = () => { + const currentUrl = new URL(window.location.href); + + const searchString = createEncodedFilterString(filters, true); + const sortField = sortModel[0]?.field ?? 'genomicPosition'; + const sortDirection = sortModel[0]?.sort ?? false; + + const sortReverse = (sortDirection === 'desc'); + + const rawUrl = ActionURL.buildURL('jbrowse', 'luceneCSVExport.api'); + const exportUrl = new URL(rawUrl, window.location.origin); + + exportUrl.searchParams.set('sessionId', sessionId); + exportUrl.searchParams.set('trackId', trackGUID); + exportUrl.searchParams.set('searchString', searchString); + exportUrl.searchParams.set('sortField', sortField); + exportUrl.searchParams.set('sortReverse', sortReverse.toString()); + + window.location.href = exportUrl.toString(); + }; + const TableCellWithPopover = (props: { value: any }) => { const { value } = props; const fullDisplayValue = value ? (Array.isArray(value) ? value.join(', ') : value) : '' @@ -184,7 +208,7 @@ const VariantTableWidget = observer(props => { ); } - function CustomToolbar({ setFilterModalOpen }) { + function CustomToolbar({ setFilterModalOpen, handleExport }) { return ( @@ -197,9 +221,14 @@ const VariantTableWidget = observer(props => { Search - + + @@ -207,7 +236,10 @@ const VariantTableWidget = observer(props => { } const ToolbarWithProps = () => ( - + ); const handleOffsetChange = (newOffset: number) => { diff --git a/jbrowse/src/org/labkey/jbrowse/JBrowseController.java b/jbrowse/src/org/labkey/jbrowse/JBrowseController.java index caaf68229..6dc297b95 100644 --- a/jbrowse/src/org/labkey/jbrowse/JBrowseController.java +++ b/jbrowse/src/org/labkey/jbrowse/JBrowseController.java @@ -20,6 +20,7 @@ import htsjdk.variant.variantcontext.Genotype; import htsjdk.variant.variantcontext.VariantContext; import htsjdk.variant.vcf.VCFFileReader; +import jakarta.servlet.http.HttpServletResponse; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; @@ -78,6 +79,8 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -891,6 +894,55 @@ public void validateForm(ResolveVcfFieldsForm form, Errors errors) } } + @RequiresPermission(ReadPermission.class) + public static class LuceneCSVExportAction extends ReadOnlyApiAction + { + @Override + public ApiResponse execute(LuceneQueryForm form, BindException errors) + { + try + { + JBrowseLuceneSearch searcher = JBrowseLuceneSearch.create(form.getSessionId(), form.getTrackId(), getUser()); + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd_HH-mm-ss"); + String timestamp = LocalDateTime.now().format(formatter); + String filename = "mGAP_results_" + timestamp + ".csv"; + + HttpServletResponse response = getViewContext().getResponse(); + response.setContentType("text/csv"); + response.setHeader("Content-Disposition", "attachment; filename=\"" + filename + "\""); + + searcher.doSearchCSV( + getUser(), + PageFlowUtil.decode(form.getSearchString()), + form.getSortField(), + form.getSortReverse(), + response + ); + + return null; + } + catch (Exception e) + { + _log.error("Error in JBrowse lucene query", e); + errors.reject(ERROR_MSG, e.getMessage()); + return null; + } + } + + @Override + public void validateForm(LuceneQueryForm form, Errors errors) + { + if ((form.getSearchString() == null || form.getSessionId() == null || form.getTrackId() == null)) + { + errors.reject(ERROR_MSG, "Must provide search string, track ID, and the JBrowse session ID"); + } + else if (!isValidUUID(form.getTrackId())) + { + errors.reject(ERROR_MSG, "Invalid track ID: " + form.getTrackId()); + } + } + } + @RequiresPermission(ReadPermission.class) public static class LuceneQueryAction extends ReadOnlyApiAction { @@ -910,7 +962,14 @@ public ApiResponse execute(LuceneQueryForm form, BindException errors) try { - return new ApiSimpleResponse(searcher.doSearch(getUser(), PageFlowUtil.decode(form.getSearchString()), form.getPageSize(), form.getOffset(), form.getSortField(), form.getSortReverse())); + return new ApiSimpleResponse(searcher.doSearchJSON( + getUser(), + PageFlowUtil.decode(form.getSearchString()), + form.getPageSize(), + form.getOffset(), + form.getSortField(), + form.getSortReverse() + )); } catch (Exception e) { diff --git a/jbrowse/src/org/labkey/jbrowse/JBrowseLuceneSearch.java b/jbrowse/src/org/labkey/jbrowse/JBrowseLuceneSearch.java index f886d1c86..df0adb665 100644 --- a/jbrowse/src/org/labkey/jbrowse/JBrowseLuceneSearch.java +++ b/jbrowse/src/org/labkey/jbrowse/JBrowseLuceneSearch.java @@ -1,5 +1,6 @@ package org.labkey.jbrowse; +import jakarta.servlet.http.HttpServletResponse; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.Logger; import org.apache.lucene.analysis.Analyzer; @@ -20,6 +21,7 @@ import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.QueryCachingPolicy; +import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; import org.apache.lucene.search.TopFieldDocs; @@ -50,6 +52,7 @@ import java.io.File; import java.io.IOException; +import java.io.PrintWriter; import java.net.URLDecoder; import java.nio.charset.StandardCharsets; import java.text.DecimalFormat; @@ -189,7 +192,17 @@ public String extractFieldName(String queryString) return parts.length > 0 ? parts[0].trim() : null; } - public JSONObject doSearch(User u, String searchString, final int pageSize, final int offset, String sortField, boolean sortReverse) throws IOException, ParseException + public JSONObject doSearchJSON(User u, String searchString, final int pageSize, final int offset, String sortField, boolean sortReverse) throws IOException, ParseException { + SearchConfig searchConfig = createSearchConfig(u, searchString, pageSize, offset, sortField, sortReverse); + return paginateJSON(searchConfig); + } + + public void doSearchCSV(User u, String searchString, String sortField, boolean sortReverse, HttpServletResponse response) throws IOException, ParseException { + SearchConfig searchConfig = createSearchConfig(u, searchString, 0, 0, sortField, sortReverse); + exportCSV(searchConfig, response); + } + + private SearchConfig createSearchConfig(User u, String searchString, final int pageSize, final int offset, String sortField, boolean sortReverse) throws IOException, ParseException { searchString = tryUrlDecode(searchString); File indexPath = _jsonFile.getExpectedLocationOfLuceneIndex(true); @@ -199,6 +212,7 @@ public JSONObject doSearch(User u, String searchString, final int pageSize, fina Analyzer analyzer = new StandardAnalyzer(); List stringQueryParserFields = new ArrayList<>(); + List fieldsList = new ArrayList<>(); Map numericQueryParserFields = new HashMap<>(); PointsConfig intPointsConfig = new PointsConfig(new DecimalFormat(), Integer.class); PointsConfig doublePointsConfig = new PointsConfig(new DecimalFormat(), Double.class); @@ -208,6 +222,7 @@ public JSONObject doSearch(User u, String searchString, final int pageSize, fina for (Map.Entry entry : fields.entrySet()) { String field = entry.getKey(); + fieldsList.add(field); JBrowseFieldDescriptor descriptor = entry.getValue(); switch(descriptor.getType()) @@ -267,14 +282,14 @@ else if (numericQueryParserFields.containsKey(fieldName)) } catch (QueryNodeException e) { - _log.error("Unable to parse query for field " + fieldName + ": " + queryString, e); + _log.error("Unable to parse query for field {}: {}", fieldName, queryString, e); throw new IllegalArgumentException("Unable to parse query: " + queryString + " for field: " + fieldName); } } else { - _log.error("No such field(s), or malformed query: " + queryString + ", field: " + fieldName); + _log.error("No such field(s), or malformed query: {}, field: {}", queryString, fieldName); throw new IllegalArgumentException("No such field(s), or malformed query: " + queryString + ", field: " + fieldName); } @@ -302,43 +317,79 @@ else if (numericQueryParserFields.containsKey(fieldName)) sort = new Sort(new SortField(sortField + "_sort", fieldType, sortReverse)); } + return new SearchConfig(cacheEntry, query, pageSize, offset, sort, fieldsList); + } + + private JSONObject paginateJSON(SearchConfig c) throws IOException, ParseException { // Get chunks of size {pageSize}. Default to 1 chunk -- add to the offset to get more. // We then iterate over the range of documents we want based on the offset. This does grow in memory // linearly with the number of documents, but my understanding is that these are just score,id pairs // rather than full documents, so mem usage *should* still be pretty low. // Perform the search with sorting - TopFieldDocs topDocs = cacheEntry.indexSearcher.search(query, pageSize * (offset + 1), sort); - + TopFieldDocs topDocs = c.cacheEntry.indexSearcher.search(c.query, c.pageSize * (c.offset + 1), c.sort); JSONObject results = new JSONObject(); // Iterate over the doc list, (either to the total end or until the page ends) grab the requested docs, // and add to returned results List data = new ArrayList<>(); - for (int i = pageSize * offset; i < Math.min(pageSize * (offset + 1), topDocs.scoreDocs.length); i++) + for (int i = c.pageSize * c.offset; i < Math.min(c.pageSize * (c.offset + 1), topDocs.scoreDocs.length); i++) { JSONObject elem = new JSONObject(); - Document doc = cacheEntry.indexSearcher.storedFields().document(topDocs.scoreDocs[i].doc); + Document doc = c.cacheEntry.indexSearcher.storedFields().document(topDocs.scoreDocs[i].doc); - for (IndexableField field : doc.getFields()) { + for (IndexableField field : doc.getFields()) + { String fieldName = field.name(); String[] fieldValues = doc.getValues(fieldName); - if (fieldValues.length > 1) { + if (fieldValues.length > 1) + { elem.put(fieldName, fieldValues); - } else { + } + else + { elem.put(fieldName, fieldValues[0]); } } - data.add(elem); } results.put("data", data); results.put("totalHits", topDocs.totalHits.value); - //TODO: we should probably stream this return results; } + private void exportCSV(SearchConfig c, HttpServletResponse response) throws IOException + { + PrintWriter writer = response.getWriter(); + IndexSearcher searcher = c.cacheEntry.indexSearcher; + TopFieldDocs topDocs = searcher.search(c.query, Integer.MAX_VALUE, c.sort); + + writer.println(String.join(",", c.fields)); + + for (ScoreDoc scoreDoc : topDocs.scoreDocs) + { + Document doc = searcher.storedFields().document(scoreDoc.doc); + List rowValues = new ArrayList<>(); + + for (String fieldName : c.fields) + { + String[] values = doc.getValues(fieldName); + String value = values.length > 0 + ? String.join(",", values) + : ""; + + // Escape strings + value = "\"" + value.replace("\"", "\"\"") + "\""; + rowValues.add(value); + } + + writer.println(String.join(",", rowValues)); + } + + writer.flush(); + } + public static class DefaultJBrowseFieldCustomizer extends AbstractJBrowseFieldCustomizer { public DefaultJBrowseFieldCustomizer() @@ -583,7 +634,7 @@ public void cacheDefaultQuery() try { JBrowseLuceneSearch.clearCache(_jsonFile.getObjectId()); - doSearch(_user, ALL_DOCS, 100, 0, GENOMIC_POSITION, false); + doSearchJSON(_user, ALL_DOCS, 100, 0, GENOMIC_POSITION, false); } catch (ParseException | IOException e) { @@ -641,4 +692,22 @@ public void shutdownStarted() JBrowseLuceneSearch.emptyCache(); } } + + private class SearchConfig { + CacheEntry cacheEntry; + Query query; + int pageSize; + int offset; + Sort sort; + List fields; + + public SearchConfig(CacheEntry cacheEntry, Query query, int pageSize, int offset, Sort sort, List fields) { + this.cacheEntry = cacheEntry; + this.query = query; + this.pageSize = pageSize; + this.offset = offset; + this.sort = sort; + this.fields = fields; + } + } } diff --git a/jbrowse/test/src/org/labkey/test/tests/external/labModules/JBrowseTest.java b/jbrowse/test/src/org/labkey/test/tests/external/labModules/JBrowseTest.java index b3bb3bc57..8ef002a10 100644 --- a/jbrowse/test/src/org/labkey/test/tests/external/labModules/JBrowseTest.java +++ b/jbrowse/test/src/org/labkey/test/tests/external/labModules/JBrowseTest.java @@ -15,6 +15,7 @@ */ package org.labkey.test.tests.external.labModules; +import au.com.bytecode.opencsv.CSVReader; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.json.JSONArray; @@ -50,6 +51,7 @@ import org.openqa.selenium.support.Color; import java.io.File; +import java.io.FileReader; import java.io.IOException; import java.util.List; import java.util.Map; @@ -1896,6 +1898,40 @@ private void testLuceneSearchUI(String sessionId) waitAndClick(Locator.tagWithClass("button", "filter-form-select-button")); waitForElement(Locator.tagWithText("span", "0.029")); + // Test the CSV export on this discrete case prior to clearing -- there should be 2 rows + File downloadCSV = doAndWaitForDownload(() -> clickButton("Export CSV", 0)); + try (FileReader fileReader = new FileReader(downloadCSV)) + { + CSVReader csvReader = new CSVReader(fileReader); + String[] line = csvReader.readNext(); + Assert.assertArrayEquals(new String[] { + "contig", "start", "end", "ref", "alt", "genomicPosition", + "nCalled", "fractionHet", "variableSamples", "homozygousVarSamples", + "nHomVar", "nHet", "AF", "AC", "CADD_PH", "CLN_ALLELE", "OMIMD", "IMPACT" + }, line); + line = csvReader.readNext(); + Assert.assertArrayEquals(new String[] { + "1", "2", "2", "A", "T", "2", "1523", "0.9268293", + "m00004,m00013,m00029,m00101,m00391,m00435,m00458,m00598,m00610,m00789,m00801,m00816,m00969,m02168," + + "m02169,m02179,m02225,m02230,m02280,m02324,m02344,m02369,m02391,m03592,m03655,m03667,m03669," + + "m03684,m03694,m05192,m05264,m05270,m05273,m05550,m05551,m05557,m05622,m05640,m05658,m05659," + + "m05660,m05662,m05666,m05668,m05669,m05671,m05672,m05673,m05675,m05795,m05812,m05834,m05855," + + "m05861,m05895,m06088,m06969,m06970,m06985,m06986,m06988,m06993,m07003,m07006,m07007,m07025," + + "m07034,m07037,m07045,m07409,m07418,m07426,m07438,m07446,m07458,m07489,m07495,m07513,m07863," + + "m07929,m07951,m07952", + "m03694,m05671,m05672,m06970,m06993,m07951", + "6", "76", "0.029", "88", "7.292", "", "", "HIGH" + }, line); + line = csvReader.readNext(); + Assert.assertArrayEquals(new String[] { + "1","5336","5336","A","G","5336","1524","1.0","m05373","","0","1","3.281E-4","1","2.868","","","MODERATE" + }, line); + } + catch (Exception e) + { + Assert.fail(); + } + clearFilterDialog("IMPACT equals HIGH,MODERATE"); testLuceneColumnSerialization(sessionId);