From 38f32bd1f61f826e3e194962cf05b6480b31e5f6 Mon Sep 17 00:00:00 2001 From: labkey-jeckels Date: Sat, 18 Oct 2025 08:52:31 -0700 Subject: [PATCH 1/8] More file path improvements, mostly around pipeline jobs --- .../AnalysisScriptController.java | 4 +- .../flow/controllers/run/RunController.java | 6 +- .../script/AbstractExternalAnalysisJob.java | 3 +- .../labkey/flow/script/ScriptXarSource.java | 6 +- .../tests/luminex/LuminexPositivityTest.java | 3 +- .../test/tests/luminex/LuminexTest.java | 5 +- ms2/src/org/labkey/ms2/MS2Controller.java | 50 +------- ms2/src/org/labkey/ms2/MS2Importer.java | 46 +------ ms2/src/org/labkey/ms2/MS2Manager.java | 29 +++-- ms2/src/org/labkey/ms2/MascotDatImporter.java | 9 +- .../ms2/PepXmlExperimentDataHandler.java | 2 +- ms2/src/org/labkey/ms2/PepXmlImporter.java | 14 +- .../ProteinProphetExperimentDataHandler.java | 7 +- .../labkey/ms2/ProteinProphetImporter.java | 20 +-- ms2/src/org/labkey/ms2/SpectrumImporter.java | 12 +- .../AbstractMS2SearchPipelineJob.java | 10 +- .../pipeline/AbstractMS2SearchProtocol.java | 7 +- .../pipeline/ImportScanCountsUpgradeJob.java | 121 ------------------ .../ms2/pipeline/MS2ImportPipelineJob.java | 11 +- .../ms2/pipeline/MS2PipelineManager.java | 75 ++--------- .../ms2/pipeline/PipelineController.java | 7 +- .../pipeline/ProteinProphetPipelineJob.java | 10 +- .../ms2/pipeline/QuantitationAlgorithm.java | 17 +-- ms2/src/org/labkey/ms2/pipeline/TPPTask.java | 10 +- .../ms2/pipeline/comet/CometPipelineJob.java | 10 +- .../pipeline/comet/CometSearchProtocol.java | 5 +- .../mascot/MascotImportPipelineJob.java | 34 +++-- .../pipeline/mascot/MascotPipelineJob.java | 9 +- .../pipeline/mascot/MascotSearchProtocol.java | 6 +- .../ms2/pipeline/mascot/MascotSearchTask.java | 11 +- .../AbstractSequestSearchTaskFactory.java | 5 + .../pipeline/sequest/SequestPipelineJob.java | 10 +- .../sequest/SequestSearchProtocol.java | 5 +- .../pipeline/sequest/SequestSearchTask.java | 6 +- .../pipeline/tandem/XTandemPipelineJob.java | 10 +- .../tandem/XTandemSearchProtocol.java | 5 +- .../ms2/reader/AbstractMzxmlIterator.java | 5 +- ms2/src/org/labkey/ms2/reader/MS2Loader.java | 13 +- .../org/labkey/ms2/reader/MS2XmlLoader.java | 3 +- .../labkey/ms2/reader/MascotDatLoader.java | 5 +- .../org/labkey/ms2/reader/PepXmlLoader.java | 6 +- .../org/labkey/ms2/reader/ProtXmlReader.java | 20 +-- .../ms2/reader/SequentialMzxmlIterator.java | 5 +- 43 files changed, 215 insertions(+), 442 deletions(-) delete mode 100644 ms2/src/org/labkey/ms2/pipeline/ImportScanCountsUpgradeJob.java diff --git a/flow/src/org/labkey/flow/controllers/executescript/AnalysisScriptController.java b/flow/src/org/labkey/flow/controllers/executescript/AnalysisScriptController.java index 1b2ebe2cc8..0c7ceb52c9 100644 --- a/flow/src/org/labkey/flow/controllers/executescript/AnalysisScriptController.java +++ b/flow/src/org/labkey/flow/controllers/executescript/AnalysisScriptController.java @@ -291,7 +291,7 @@ else if (form.getFile() == null || form.getFile().length == 0) } else { - files.addAll(form.getValidatedFiles(getContainer())); + files.addAll(form.getValidatedFiles(getContainer()).stream().map(FileLike::toNioPathForRead).map(Path::toFile).toList()); } Set usedPaths = new HashSet<>(); @@ -390,7 +390,7 @@ protected ModelAndView uploadRuns(ImportRunsForm form, BindException errors) thr if (form.isCurrent()) files = Collections.singletonList(pr.resolvePath(form.getPath())); else - files = form.getValidatedFiles(form.getContainer()); + files = form.getValidatedFiles(form.getContainer()).stream().map(FileLike::toNioPathForRead).map(Path::toFile).toList(); // validate target study Container targetStudy = getTargetStudy(form.getTargetStudy(), errors); diff --git a/flow/src/org/labkey/flow/controllers/run/RunController.java b/flow/src/org/labkey/flow/controllers/run/RunController.java index af765567b3..30501da07d 100644 --- a/flow/src/org/labkey/flow/controllers/run/RunController.java +++ b/flow/src/org/labkey/flow/controllers/run/RunController.java @@ -604,13 +604,13 @@ else if (_wells != null && !_wells.isEmpty()) return _success = true; } - private void writeManifest(String manifestJson, String dir) throws IOException + private void writeManifest(String manifestJson, File dir) throws IOException { if (manifestJson == null || manifestJson.isEmpty()) return; - File file = new File(dir,MANIFEST_FILENAME); + File file = FileUtil.appendName(dir, MANIFEST_FILENAME); FileOutputStream statisticsFile = new FileOutputStream(file); try (PrintWriter pw = PrintWriters.getPrintWriter(statisticsFile)) @@ -694,7 +694,7 @@ URLHelper onExportComplete(ExportAnalysisForm form, VirtualFile vf, SampleIdMap< ViewBackgroundInfo vbi = new ViewBackgroundInfo(getContainer(), getUser(), null); ExportAnalysisManifest analysisManifest = buildExportAnalysisManifest(form, files); - writeManifest(analysisManifest.toJSON(), vf.getLocation()); + writeManifest(analysisManifest.toJSON(), location); PipelineJob job = new ExportToScriptJob(_guid, _exportToScriptPath, _exportToScriptCommandLine, _exportToScriptFormat, form.getLabel(), location, _exportToScriptTimeout, _exportToScriptDeleteOnComplete, vbi, root); String jobGuid = null; diff --git a/flow/src/org/labkey/flow/script/AbstractExternalAnalysisJob.java b/flow/src/org/labkey/flow/script/AbstractExternalAnalysisJob.java index c3ed81687c..0c1bd8b8e2 100644 --- a/flow/src/org/labkey/flow/script/AbstractExternalAnalysisJob.java +++ b/flow/src/org/labkey/flow/script/AbstractExternalAnalysisJob.java @@ -36,6 +36,7 @@ import org.labkey.api.pipeline.PipelineService; import org.labkey.api.query.FieldKey; import org.labkey.api.security.User; +import org.labkey.api.util.FileUtil; import org.labkey.api.util.Pair; import org.labkey.api.view.ActionURL; import org.labkey.api.view.ViewBackgroundInfo; @@ -383,7 +384,7 @@ protected FlowRun saveAnalysis(User user, Container container, FlowExperiment ex MultiValuedMap sampleIdToNameMap) throws Exception { // Fake file URI set on the FCSFile/FCSAnalsyis ExpData to ensure it's recognized by the FlowDataHandler. - URI dataFileURI = new File(externalAnalysisFile.getParent(), "attributes.flowdata.xml").toURI(); + URI dataFileURI = FileUtil.appendName(externalAnalysisFile.getParentFile(), "attributes.flowdata.xml").toURI(); // Prepare comp matrices for saving Map compMatrixMap = new HashMap<>(); diff --git a/flow/src/org/labkey/flow/script/ScriptXarSource.java b/flow/src/org/labkey/flow/script/ScriptXarSource.java index a356d85faa..f5a3e2fe82 100644 --- a/flow/src/org/labkey/flow/script/ScriptXarSource.java +++ b/flow/src/org/labkey/flow/script/ScriptXarSource.java @@ -22,6 +22,8 @@ import org.labkey.api.exp.XarSource; import org.labkey.api.pipeline.PipelineJob; import org.labkey.api.util.FileUtil; +import org.labkey.vfs.FileLike; +import org.labkey.vfs.FileSystemLike; import java.io.File; import java.io.FileWriter; @@ -85,8 +87,8 @@ public ExperimentArchiveDocument getDocument() @Override - public Path getLogFilePath() + public FileLike getLogFilePath() { - return _logFile.toPath(); + return FileSystemLike.wrapFile(_logFile); } } diff --git a/luminex/test/src/org/labkey/test/tests/luminex/LuminexPositivityTest.java b/luminex/test/src/org/labkey/test/tests/luminex/LuminexPositivityTest.java index df8576f455..6a05b042f0 100644 --- a/luminex/test/src/org/labkey/test/tests/luminex/LuminexPositivityTest.java +++ b/luminex/test/src/org/labkey/test/tests/luminex/LuminexPositivityTest.java @@ -19,6 +19,7 @@ import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.labkey.api.util.FileUtil; import org.labkey.test.BaseWebDriverTest; import org.labkey.test.Locator; import org.labkey.test.TestFileUtils; @@ -57,7 +58,7 @@ public final class LuminexPositivityTest extends LuminexTest private Boolean _expectedNegativeControlValue = false; private Boolean _newNegativeControlValue = false; private static final String _negControlAnalyte = _analyteNames.get(1); - private static final File POSITIVITY_RTRANSFORM_SCRIPT_FILE = new File(TestFileUtils.getLabKeyRoot(), "server/modules/commonAssays/luminex/resources/transformscripts/description_parsing_example.pl"); + private static final File POSITIVITY_RTRANSFORM_SCRIPT_FILE = FileUtil.appendPath(TestFileUtils.getLabKeyRoot(), org.labkey.api.util.Path.parse("server/modules/commonAssays/luminex/resources/transformscripts/description_parsing_example.pl")); private static final String RUN_ID_BASE = "Positivity"; @BeforeClass diff --git a/luminex/test/src/org/labkey/test/tests/luminex/LuminexTest.java b/luminex/test/src/org/labkey/test/tests/luminex/LuminexTest.java index 165c177565..aed80ce32a 100644 --- a/luminex/test/src/org/labkey/test/tests/luminex/LuminexTest.java +++ b/luminex/test/src/org/labkey/test/tests/luminex/LuminexTest.java @@ -19,6 +19,7 @@ import org.apache.commons.lang3.StringUtils; import org.junit.BeforeClass; import org.labkey.api.query.QueryKey; +import org.labkey.api.util.FileUtil; import org.labkey.remoteapi.CommandException; import org.labkey.remoteapi.Connection; import org.labkey.remoteapi.assay.GetProtocolCommand; @@ -101,8 +102,8 @@ public abstract class LuminexTest extends BaseWebDriverTest public static final File TEST_ASSAY_MULTIPLE_STANDARDS_3 = TestFileUtils.getSampleData("luminex/plate 3_IgA-Biot (Standard1).xls"); public static final File TEST_ASSAY_MULTIPLE_STANDARDS_3_XLSX = TestFileUtils.getSampleData("luminex/plate 3_IgA-Biot (Standard1).xls"); - public static final File RTRANSFORM_SCRIPT_FILE_LABKEY = new File(TestFileUtils.getLabKeyRoot(), "server/modules/commonAssays/luminex/resources/transformscripts/labkey_luminex_transform.R"); - public static final File RTRANSFORM_SCRIPT_FILE_LAB = new File(TestFileUtils.getLabKeyRoot(), "server/modules/commonAssays/luminex/resources/transformscripts/tomaras_luminex_transform.R"); + public static final File RTRANSFORM_SCRIPT_FILE_LABKEY = FileUtil.appendPath(TestFileUtils.getLabKeyRoot(), org.labkey.api.util.Path.parse("server/modules/commonAssays/luminex/resources/transformscripts/labkey_luminex_transform.R")); + public static final File RTRANSFORM_SCRIPT_FILE_LAB = FileUtil.appendPath(TestFileUtils.getLabKeyRoot(), org.labkey.api.util.Path.parse("server/modules/commonAssays/luminex/resources/transformscripts/tomaras_luminex_transform.R")); public static final String ASSAY_DATA_FILE_LOCATION_MULTIPLE_FIELD = "__primaryFile__"; diff --git a/ms2/src/org/labkey/ms2/MS2Controller.java b/ms2/src/org/labkey/ms2/MS2Controller.java index c29bf50815..512455407d 100644 --- a/ms2/src/org/labkey/ms2/MS2Controller.java +++ b/ms2/src/org/labkey/ms2/MS2Controller.java @@ -166,7 +166,6 @@ import org.labkey.ms2.peptideview.PeptidesView; import org.labkey.ms2.peptideview.QueryPeptideMS2RunView; import org.labkey.ms2.pipeline.AbstractMS2SearchTask; -import org.labkey.ms2.pipeline.ImportScanCountsUpgradeJob; import org.labkey.ms2.pipeline.ProteinProphetPipelineJob; import org.labkey.ms2.pipeline.TPPTask; import org.labkey.ms2.pipeline.mascot.MascotClientImpl; @@ -186,6 +185,7 @@ import org.labkey.ms2.query.SpectraCountConfiguration; import org.labkey.ms2.reader.PeptideProphetSummary; import org.labkey.ms2.reader.SensitivitySummary; +import org.labkey.vfs.FileLike; import org.springframework.validation.BindException; import org.springframework.validation.Errors; import org.springframework.web.servlet.ModelAndView; @@ -4553,7 +4553,7 @@ public void validateCommand(PipelinePathForm target, Errors errors) @Override public boolean handlePost(PipelinePathForm form, BindException errors) throws Exception { - for (File f : form.getValidatedFiles(getContainer())) + for (FileLike f : form.getValidatedFiles(getContainer())) { if (f.isFile()) { @@ -5271,49 +5271,6 @@ public Form getForm() } } - @RequiresSiteAdmin - public class ImportMSScanCountsUpgradeAction extends FormViewAction - { - @Override - public void validateCommand(Object target, Errors errors) - { - } - - @Override - public ModelAndView getView(Object o, boolean reshow, BindException errors) - { - return new JspView<>("/org/labkey/ms2/pipeline/importMSScanCounts.jsp"); - } - - @Override - public boolean handlePost(Object o, BindException errors) throws Exception - { - PipeRoot root = PipelineService.get().findPipelineRoot(getContainer()); - if (root == null || !root.isValid()) - { - throw new NotFoundException("No pipeline root found for " + getContainer()); - } - - ViewBackgroundInfo info = getViewBackgroundInfo(); - PipelineJob job = new ImportScanCountsUpgradeJob(info, root); - PipelineService.get().queueJob(job); - - return true; - } - - @Override - public ActionURL getSuccessURL(Object o) - { - return urlProvider(PipelineUrls.class).urlBegin(getContainer()); - } - - @Override - public void addNavTrail(NavTree root) - { - root.addChild("Load MS scan counts"); - } - } - public static class TestCase extends AbstractActionPermissionTest { @Override @@ -5345,8 +5302,7 @@ controller.new EditElutionGraphAction() // @RequiresSiteAdmin assertForRequiresSiteAdmin(user, controller.new PurgeRunsAction(), - controller.new ShowMS2AdminAction(), - controller.new ImportMSScanCountsUpgradeAction() + controller.new ShowMS2AdminAction() ); } } diff --git a/ms2/src/org/labkey/ms2/MS2Importer.java b/ms2/src/org/labkey/ms2/MS2Importer.java index 09546e5ee9..e6549ae63e 100644 --- a/ms2/src/org/labkey/ms2/MS2Importer.java +++ b/ms2/src/org/labkey/ms2/MS2Importer.java @@ -41,6 +41,7 @@ import org.labkey.api.util.massSpecDataFileType; import org.labkey.ms2.reader.AbstractMzxmlIterator; import org.labkey.ms2.reader.SimpleScan; +import org.labkey.vfs.FileLike; import javax.xml.stream.XMLStreamException; import java.io.File; @@ -302,14 +303,14 @@ protected int createRun() return asInteger(returnMap.get("Run")); } - protected boolean isMzXmlFile(File file) + protected boolean isMzXmlFile(FileLike file) { String name = file.getName().toLowerCase(); return name.endsWith(".mzxml") || name.endsWith(".mzxml.gz"); } - protected int createFraction(User user, Container c, int runId, String path, File file) throws IOException + protected int createFraction(User user, Container c, int runId, String path, FileLike file) throws IOException { MS2Fraction fraction = new MS2Fraction(); fraction.setRun(runId); @@ -321,7 +322,7 @@ protected int createFraction(User user, Container c, int runId, String path, Fil int totalScans = loadScanCounts(file, fraction); _log.info("Finished parsing to get scan counts. Total: " + totalScans + ", MS1: " + fraction.getMS1ScanCount() + ", MS2: " + fraction.getMS2ScanCount() + ", MS3: " + fraction.getMS3ScanCount() + ", MS4:" + fraction.getMS4ScanCount()); - fraction.setMzXmlURL(FileUtil.resolveFile(FileUtil.getAbsoluteCaseSensitiveFile(file)).toPath().toUri().toString()); + fraction.setMzXmlURL(FileUtil.resolveFile(FileUtil.getAbsoluteCaseSensitiveFile(file.toNioPathForRead().toFile())).toPath().toUri().toString()); massSpecDataFileType msdft = new massSpecDataFileType(); String peptideFileName = msdft.getBaseName(file); // strip off .mzxml or .mzxml.gz @@ -343,7 +344,7 @@ protected int createFraction(User user, Container c, int runId, String path, Fil } /** @return the total number of scans in the file */ - public static int loadScanCounts(File mzXmlFile, MS2Fraction fraction) throws IOException + public static int loadScanCounts(FileLike mzXmlFile, MS2Fraction fraction) throws IOException { int scanCount = 0; if (NetworkDrive.exists(mzXmlFile)) @@ -545,43 +546,6 @@ WHERE Fraction IN (SELECT Fraction FROM ms2.Fractions WHERE Run = ?) AND _updateSeqIdEndOfLookupStringSql = sql.getSQL(); } - private static final String _updateSeqIdInexactMatchSql; - - static - { - SQLFragment sql = new SQLFragment(); - - /* - UPDATE ms2.peptidesdata p SET SeqId = ( - SELECT - CASE (SELECT count(*) FROM prot.fastasequences fs - WHERE fs.FastaId = ? AND fs.LookupString LIKE CONCAT('%', Protein, '%')) - WHEN 1 THEN - (SELECT SeqId FROM prot.fastasequences fs - WHERE fs.FastaId = ? AND fs.LookupString LIKE CONCAT('%', Protein, '%')) - ELSE - NULL - END - ) - WHERE p.SeqId IS NULL AND FRACTION = ? - */ - - sql.append("UPDATE ").append(MS2Manager.getTableInfoPeptidesData()); - sql.append(" SET SeqId = ("); - sql.append(" SELECT " ); - sql.append(" CASE (SELECT COUNT(*) FROM ").append(ProteinSchema.getTableInfoFastaSequences()).append(" fs "); - sql.append(" WHERE fs.FastaId = ? AND fs.LookupString LIKE ").append(MS2Manager.getSqlDialect().concatenate("'%'", "Protein", "'%'")).append(")"); - sql.append(" WHEN 1 THEN " ); - sql.append(" (SELECT SeqId FROM ").append(ProteinSchema.getTableInfoFastaSequences()).append(" fs "); - sql.append(" WHERE fs.FastaId = ? AND fs.LookupString LIKE ").append(MS2Manager.getSqlDialect().concatenate("'%'", "Protein", "'%'")).append(")"); - sql.append(" ELSE "); - sql.append(" NULL"); - sql.append(" END)"); - sql.append(" WHERE SeqId IS NULL AND Fraction = ?"); - - _updateSeqIdInexactMatchSql = sql.getSQL(); - } - private static final SQLFragment _updateSequencePositionSql; static diff --git a/ms2/src/org/labkey/ms2/MS2Manager.java b/ms2/src/org/labkey/ms2/MS2Manager.java index 11d1d9f724..d4156c24ad 100644 --- a/ms2/src/org/labkey/ms2/MS2Manager.java +++ b/ms2/src/org/labkey/ms2/MS2Manager.java @@ -339,7 +339,7 @@ private static ExpRun wrapRun(MS2Run run, User user) throws ExperimentException try (DbScope.Transaction transaction = ExperimentService.get().getSchema().getScope().ensureTransaction()) { Container container = run.getContainer(); - final FileLike pepXMLFile = FileSystemLike.wrapFile(new File(run.getPath(), run.getFileName())); + final FileLike pepXMLFile = FileSystemLike.wrapFile(new File(run.getPath())).resolveChild(run.getFileName()); // Check if this ExpData existingPepXmlData = ExperimentService.get().getExpDataByURL(pepXMLFile, container); @@ -377,7 +377,7 @@ private static ExpRun wrapRun(MS2Run run, User user) throws ExperimentException XarSource source = new AbstractFileXarSource("Wrap MS2 Run", container, user) { @Override - public Path getLogFilePath() + public FileLike getLogFilePath() { throw new UnsupportedOperationException(); } @@ -476,7 +476,7 @@ public static List getRunIds(List runs) return runIds; } - public static ProteinProphetFile getProteinProphetFile(File f, Container c) + public static ProteinProphetFile getProteinProphetFile(FileLike f, Container c) { String sql = "SELECT " + getTableInfoProteinProphetFiles() + ".* FROM " + @@ -487,14 +487,15 @@ public static ProteinProphetFile getProteinProphetFile(File f, Container c) getTableInfoRuns() + ".Container = ? AND " + getTableInfoRuns() + ".Deleted = ?"; + File file = f.toNioPathForRead().toFile(); String path; try { - path = f.getCanonicalPath(); + path = file.getCanonicalPath(); } catch (IOException e) { - path = f.getAbsolutePath(); + path = file.getAbsolutePath(); } return new SqlSelector(getSchema(), sql, path, c, Boolean.FALSE).getObject(ProteinProphetFile.class); @@ -558,7 +559,7 @@ private static MS2Run[] getRuns(String whereClause, Object... params) } public static MS2Importer.RunInfo addMascotRunToQueue(ViewBackgroundInfo info, - File file, + FileLike file, String description, PipeRoot root) throws IOException { MS2Importer importer = createImporter(file, info, description, null, new XarContext(description, info.getContainer(), info.getUser())); @@ -576,7 +577,7 @@ public static MS2Importer.RunInfo addMascotRunToQueue(ViewBackgroundInfo info, } public static MS2Importer.RunInfo addRunToQueue(ViewBackgroundInfo info, - File file, + FileLike file, String description, PipeRoot root) throws IOException { MS2Importer importer = createImporter(file, info, description, null, new XarContext(description, info.getContainer(), info.getUser())); @@ -594,7 +595,7 @@ public static MS2Importer.RunInfo addRunToQueue(ViewBackgroundInfo info, } public static MS2Run addRun(ViewBackgroundInfo info, Logger log, - File file, + FileLike file, boolean restart, XarContext context) throws IOException, XMLStreamException { MS2Importer importer = createImporter(file, info, file.getName() + (context.getJobDescription() != null ? " - " + context.getJobDescription() : ""), log, context); @@ -610,7 +611,7 @@ public static boolean endsWithExtOrExtDotGZ(String name,String ext) } public static MS2Run importRun(ViewBackgroundInfo info, Logger log, - File file, + FileLike file, MS2Importer.RunInfo runInfo, XarContext context) throws IOException, XMLStreamException { @@ -618,11 +619,11 @@ public static MS2Run importRun(ViewBackgroundInfo info, Logger log, return importer.upload(runInfo); } - private static MS2Importer createImporter(File file, ViewBackgroundInfo info, String description, Logger log, XarContext context) throws IOException + private static MS2Importer createImporter(FileLike file, ViewBackgroundInfo info, String description, Logger log, XarContext context) throws IOException { Container c = info.getContainer(); - String fileName = file.getPath(); + String fileName = file.toNioPathForRead().toFile().getPath(); if (endsWithExtOrExtDotGZ(fileName,".xml") || fileName.endsWith(".pepXML")) return new PepXmlImporter(info.getUser(), c, description, fileName, log, context); else if (fileName.toLowerCase().endsWith(".dat")) @@ -736,7 +737,7 @@ public static void markAsDeleted(List runIds, Container c, User user) MS2Run run = getRun(runId.intValue()); if (run != null) { - File file = new File(run.getPath(), run.getFileName()); + final FileLike file = FileSystemLike.wrapFile(new File(run.getPath())).resolveChild(run.getFileName()); ExpData data = ExperimentService.get().getExpDataByURL(file, c); if (data != null) { @@ -1135,8 +1136,8 @@ public static Pair getSpectrumFromMzXML(MS2Fraction fraction, private static Pair getSpectrumFromDat(@NotNull MS2Fraction fraction, int scan) throws SpectrumException { - File f = new File(getRun(fraction.getRun()).getPath(), fraction.getFileName()); - NetworkDrive.ensureDrive(f.getPath()); + final FileLike f = FileSystemLike.wrapFile(new File(getRun(fraction.getRun()).getPath())).resolveChild(fraction.getFileName()); + NetworkDrive.ensureDrive(f); try (MascotDatLoader loader = new MascotDatLoader(f, LOG)) { return loader.loadSpectrum(scan); diff --git a/ms2/src/org/labkey/ms2/MascotDatImporter.java b/ms2/src/org/labkey/ms2/MascotDatImporter.java index 74c5a971ed..e56f4824a5 100644 --- a/ms2/src/org/labkey/ms2/MascotDatImporter.java +++ b/ms2/src/org/labkey/ms2/MascotDatImporter.java @@ -25,12 +25,13 @@ import org.labkey.api.util.NetworkDrive; import org.labkey.ms2.reader.MS2Loader; import org.labkey.ms2.reader.MascotDatLoader; +import org.labkey.vfs.FileLike; +import org.labkey.vfs.FileSystemLike; import javax.xml.stream.XMLStreamException; import java.io.File; import java.io.IOException; import java.sql.SQLException; -import java.util.HashMap; import java.util.Map; /** @@ -59,11 +60,11 @@ public String getType() public void importRun(MS2Progress progress) throws IOException, XMLStreamException { - File f = new File(_path + "/" + _fileName); - NetworkDrive.ensureDrive(f.getPath()); + FileLike f = FileSystemLike.wrapFile(new File(_path)).resolveChild(_fileName); + NetworkDrive.ensureDrive(f); _fractionId = createFraction(_user, _container, _runId, _path, f); MS2Loader.PeptideFraction fraction = new MS2Loader.PeptideFraction(); - fraction.setSpectrumPath(f.getPath()); + fraction.setSpectrumPath(f.toNioPathForRead().toFile().getPath()); try (MascotDatLoader loader = new MascotDatLoader(f, _log)) { diff --git a/ms2/src/org/labkey/ms2/PepXmlExperimentDataHandler.java b/ms2/src/org/labkey/ms2/PepXmlExperimentDataHandler.java index 4947a7efec..1e82ef129e 100644 --- a/ms2/src/org/labkey/ms2/PepXmlExperimentDataHandler.java +++ b/ms2/src/org/labkey/ms2/PepXmlExperimentDataHandler.java @@ -133,7 +133,7 @@ public boolean accept(File f) return; } - MS2Run run = MS2Manager.addRun(info, log, dataFile, restart, context); + MS2Run run = MS2Manager.addRun(info, log, dataFileLike, restart, context); if (run == null || run.statusId != MS2Importer.STATUS_SUCCESS) { diff --git a/ms2/src/org/labkey/ms2/PepXmlImporter.java b/ms2/src/org/labkey/ms2/PepXmlImporter.java index 240e85efd0..59524e516a 100644 --- a/ms2/src/org/labkey/ms2/PepXmlImporter.java +++ b/ms2/src/org/labkey/ms2/PepXmlImporter.java @@ -36,6 +36,8 @@ import org.labkey.ms2.reader.PepXmlLoader.PeptideIterator; import org.labkey.ms2.reader.PeptideProphetHandler; import org.labkey.ms2.reader.PeptideProphetSummary; +import org.labkey.vfs.FileLike; +import org.labkey.vfs.FileSystemLike; import javax.xml.stream.XMLStreamException; import java.io.File; @@ -63,8 +65,8 @@ public void importRun(MS2Progress progress) throws XMLStreamException, IOExcepti { boolean runUpdated = false; // Set to true after we update the run information (after importing the first fraction) - File f = new File(_path + "/" + _fileName); - NetworkDrive.ensureDrive(f.getPath()); + FileLike f = FileSystemLike.wrapFile(new File(_path)).resolveChild(_fileName); + NetworkDrive.ensureDrive(f); loader = new PepXmlLoader(f, _log); PeptideProphetSummary summary = loader.getPeptideProphetSummary(); @@ -186,7 +188,7 @@ private void writeFractionInfo(MS2Loader.PeptideFraction fraction) throws IOExce mzXMLFile = FT_MZXML.getFile(pepXmlDir.getParentFile().getParentFile(), newFilename); } - if (mzXMLFile == null || !NetworkDrive.exists(mzXMLFile)) + if (!NetworkDrive.exists(mzXMLFile)) { // If not there, look in the same directory as the MS2 results mzXMLFile = FT_MZXML.getFile(pepXmlDir, newFilename); @@ -206,7 +208,7 @@ private void writeFractionInfo(MS2Loader.PeptideFraction fraction) throws IOExce } File mzXMLFile = getMzXMLFile(fraction); - _fractionId = createFraction(_user, _container, _runId, _path, mzXMLFile); + _fractionId = createFraction(_user, _container, _runId, _path, FileSystemLike.wrapFile(mzXMLFile)); } @@ -256,9 +258,9 @@ protected void processSpectrumFile(PepXmlFraction fraction, Set scans, } } - SpectrumImporter sl = new SpectrumImporter(gzFileName, "", mzXmlFile, scans, progress, _fractionId, _log, shouldLoadSpectra, shouldLoadRetentionTimes); + SpectrumImporter sl = new SpectrumImporter(gzFileName, "", FileSystemLike.wrapFile(mzXmlFile), scans, progress, _fractionId, _log, shouldLoadSpectra, shouldLoadRetentionTimes); sl.upload(); - updateFractionSpectrumFileName(sl.getFile()); + updateFractionSpectrumFileName(sl.getFile() == null ? null : sl.getFile().toNioPathForRead().toFile()); } diff --git a/ms2/src/org/labkey/ms2/ProteinProphetExperimentDataHandler.java b/ms2/src/org/labkey/ms2/ProteinProphetExperimentDataHandler.java index 539d16aa81..a3158fc238 100644 --- a/ms2/src/org/labkey/ms2/ProteinProphetExperimentDataHandler.java +++ b/ms2/src/org/labkey/ms2/ProteinProphetExperimentDataHandler.java @@ -30,7 +30,6 @@ import org.labkey.api.view.ViewBackgroundInfo; import org.labkey.ms2.pipeline.TPPTask; import org.labkey.vfs.FileLike; -import org.labkey.vfs.FileSystemLike; import javax.xml.stream.XMLStreamException; import java.io.File; @@ -48,9 +47,8 @@ public DataType getDataType() } @Override - public void importFile(@NotNull ExpData data, @NotNull FileLike dataFileLike, @NotNull ViewBackgroundInfo info, @NotNull Logger log, @NotNull XarContext context) throws ExperimentException + public void importFile(@NotNull ExpData data, @NotNull FileLike dataFile, @NotNull ViewBackgroundInfo info, @NotNull Logger log, @NotNull XarContext context) throws ExperimentException { - File dataFile = FileSystemLike.toFile(dataFileLike); if (context.getJob() != null && "false".equalsIgnoreCase(context.getJob().getParameters().get(IMPORT_PROPHET_RESULTS))) { log.info("Skipping import of file " + dataFile); @@ -75,9 +73,8 @@ public void importFile(@NotNull ExpData data, @NotNull FileLike dataFileLike, @N @Override public ActionURL getContentURL(ExpData data) { - File dataFile = data.getFile(); MS2Run run = null; - ProteinProphetFile ppFile = MS2Manager.getProteinProphetFile(dataFile, data.getContainer()); + ProteinProphetFile ppFile = MS2Manager.getProteinProphetFile(data.getFileLike(), data.getContainer()); if (ppFile != null) { run = MS2Manager.getRun(ppFile.getRun()); diff --git a/ms2/src/org/labkey/ms2/ProteinProphetImporter.java b/ms2/src/org/labkey/ms2/ProteinProphetImporter.java index 2ec2f927c6..ec84444687 100644 --- a/ms2/src/org/labkey/ms2/ProteinProphetImporter.java +++ b/ms2/src/org/labkey/ms2/ProteinProphetImporter.java @@ -40,6 +40,8 @@ import org.labkey.ms2.reader.ITraqProteinQuantitation; import org.labkey.ms2.reader.ProtXmlReader; import org.labkey.ms2.reader.ProteinGroup; +import org.labkey.vfs.FileLike; +import org.labkey.vfs.FileSystemLike; import javax.xml.stream.XMLStreamException; import java.io.BufferedInputStream; @@ -66,14 +68,14 @@ public class ProteinProphetImporter { private static final Logger _log = LogManager.getLogger(ProteinProphetImporter.class); - private final File _file; + private final FileLike _file; private final String _experimentRunLSID; private final XarContext _context; private final SqlDialect _dialect = MS2Manager.getSchema().getSqlDialect(); private static final int STREAM_BUFFER_SIZE = 128 * 1024; - public ProteinProphetImporter(File f, String experimentRunLSID, XarContext context) + public ProteinProphetImporter(FileLike f, String experimentRunLSID, XarContext context) { _file = f; _experimentRunLSID = experimentRunLSID; @@ -336,7 +338,7 @@ private ProteinProphetFile insertProteinProphetFile(ViewBackgroundInfo info, MS2 throws IOException, XMLStreamException { ProteinProphetFile file = new ProteinProphetFile(parser); - file.setFilePath(_file.getCanonicalPath()); + file.setFilePath(_file.toNioPathForRead().toFile().getCanonicalPath()); file.setRun(run.getRun()); Table.insert(info.getUser(), MS2Manager.getTableInfoProteinProphetFiles(), file); @@ -355,20 +357,20 @@ private MS2Run importRun(ViewBackgroundInfo info, Logger log) throws IOException // First, see if our usual XAR lookups can find it String pepXMLFileName = pepXMLFileNameOriginal + ((attempts>1)?".gz":""); attemptedFiles.add(pepXMLFileName); - pepXMLFile = _context.findFile(pepXMLFileName, _file.getParentFile()); + pepXMLFile = _context.findFile(pepXMLFileName, _file.getParent().toNioPathForRead().toFile()); if (pepXMLFile == null) { // Second, try the file name in the XML in the current directory - pepXMLFile = FileUtil.appendName(_file.getParentFile(), new File(pepXMLFileName).getName()); + pepXMLFile = FileUtil.appendName(_file.getParent().toNioPathForRead().toFile(), new File(pepXMLFileName).getName()); attemptedFiles.add(pepXMLFile.getAbsolutePath()); if (!NetworkDrive.exists(pepXMLFile)) { // Third, try replacing the .pep-prot.xml on the file name with .pep.xml // and looking in the same directory - if (TPPTask.isProtXMLFile(_file)) + if (TPPTask.isProtXMLFile(_file.getParent().toNioPathForRead().toFile())) { String baseName = TPPTask.FT_PROT_XML.getBaseName(_file); - pepXMLFile = TPPTask.getPepXMLFile(_file.getParentFile(), baseName); + pepXMLFile = TPPTask.getPepXMLFile(_file.getParent().toNioPathForRead().toFile(), baseName); attemptedFiles.add(pepXMLFile.getAbsolutePath()); if (!NetworkDrive.exists(pepXMLFile)) { @@ -383,7 +385,7 @@ private MS2Run importRun(ViewBackgroundInfo info, Logger log) throws IOException } log.info("Resolved referenced PepXML file to " + pepXMLFile.getPath()); - MS2Run run = MS2Manager.addRun(info, log, pepXMLFile, false, _context); + MS2Run run = MS2Manager.addRun(info, log, FileSystemLike.wrapFile(pepXMLFile), false, _context); if (_experimentRunLSID != null && run.getExperimentRunLSID() == null) { run.setExperimentRunLSID(_experimentRunLSID); @@ -514,7 +516,7 @@ private int insertProteinGroup(ProtXmlReader.Protein protein, PreparedStatement return groupId; } - private String getPepXMLFileName() throws FileNotFoundException, XMLStreamException + private String getPepXMLFileName() throws IOException, XMLStreamException { SimpleXMLStreamReader parser = null; InputStream fIn = null; diff --git a/ms2/src/org/labkey/ms2/SpectrumImporter.java b/ms2/src/org/labkey/ms2/SpectrumImporter.java index 1d70d07c5f..b5d0a486f8 100644 --- a/ms2/src/org/labkey/ms2/SpectrumImporter.java +++ b/ms2/src/org/labkey/ms2/SpectrumImporter.java @@ -22,6 +22,8 @@ import org.labkey.api.util.NetworkDrive; import org.labkey.api.util.Pair; import org.labkey.ms2.reader.*; +import org.labkey.vfs.FileLike; +import org.labkey.vfs.FileSystemLike; import javax.xml.stream.XMLStreamException; import java.io.File; @@ -45,15 +47,15 @@ public class SpectrumImporter private Logger _log; private MS2Importer.MS2Progress _progress; - private Set _scans; + private Set _scans; private final int _fractionId; private SimpleScanIterator _scanIterator; - private File _file = null; + private FileLike _file = null; private final boolean _shouldImportSpectra; private final boolean _shouldImportRetentionTime; - protected SpectrumImporter(String gzFileName, String dtaFileNamePrefix, File mzXmlFile, Set scans, MS2Importer.MS2Progress progress, int fractionId, Logger log, boolean shouldImportSpectra, boolean shouldImportRetentionTime) + protected SpectrumImporter(String gzFileName, String dtaFileNamePrefix, FileLike mzXmlFile, Set scans, MS2Importer.MS2Progress progress, int fractionId, Logger log, boolean shouldImportSpectra, boolean shouldImportRetentionTime) { _scans = scans; _progress = progress; @@ -72,7 +74,7 @@ protected SpectrumImporter(String gzFileName, String dtaFileNamePrefix, File mzX if (NetworkDrive.exists(gz)) { - _file = gz; + _file = FileSystemLike.wrapFile(gz); _scanIterator = new TarIterator(gz, dtaFileNamePrefix); } else @@ -287,7 +289,7 @@ protected void finalize() throws Throwable } - public File getFile() + public FileLike getFile() { return _file; } diff --git a/ms2/src/org/labkey/ms2/pipeline/AbstractMS2SearchPipelineJob.java b/ms2/src/org/labkey/ms2/pipeline/AbstractMS2SearchPipelineJob.java index 5385f9dbfe..5f5a970ffc 100644 --- a/ms2/src/org/labkey/ms2/pipeline/AbstractMS2SearchPipelineJob.java +++ b/ms2/src/org/labkey/ms2/pipeline/AbstractMS2SearchPipelineJob.java @@ -29,10 +29,10 @@ import org.labkey.api.util.NetworkDrive; import org.labkey.api.util.PepXMLFileType; import org.labkey.api.view.ViewBackgroundInfo; +import org.labkey.vfs.FileLike; import java.io.File; import java.io.IOException; -import java.nio.file.Path; import java.util.*; /** @@ -82,10 +82,10 @@ public AbstractMS2SearchPipelineJob(AbstractMS2SearchProtocol protocol, ViewBackgroundInfo info, PipeRoot root, String protocolName, - Path fileParameters, - List filesInput) throws IOException + FileLike fileParameters, + List filesInput) throws IOException { - super(protocol, providerName, info, root, protocolName, fileParameters, filesInput, true, false); + super(protocol, providerName, info, root, protocolName, fileParameters, filesInput, true); _dirSequenceRoot = MS2PipelineManager.getSequenceDatabaseRoot(info.getContainer(), false); @@ -106,7 +106,7 @@ public AbstractMS2SearchPipelineJob(AbstractMS2SearchProtocol protocol, } } - public AbstractMS2SearchPipelineJob(AbstractMS2SearchPipelineJob job, File fileFraction) + public AbstractMS2SearchPipelineJob(AbstractMS2SearchPipelineJob job, FileLike fileFraction) { super(job, fileFraction); diff --git a/ms2/src/org/labkey/ms2/pipeline/AbstractMS2SearchProtocol.java b/ms2/src/org/labkey/ms2/pipeline/AbstractMS2SearchProtocol.java index d2dbf04e63..a7f069d98a 100644 --- a/ms2/src/org/labkey/ms2/pipeline/AbstractMS2SearchProtocol.java +++ b/ms2/src/org/labkey/ms2/pipeline/AbstractMS2SearchProtocol.java @@ -28,6 +28,7 @@ import org.labkey.api.util.FileType; import org.labkey.api.util.massSpecDataFileType; import org.labkey.api.view.ViewBackgroundInfo; +import org.labkey.vfs.FileLike; import java.io.File; import java.io.IOException; @@ -70,12 +71,12 @@ public File getDirSeqRoot() @Override public abstract JOB createPipelineJob(ViewBackgroundInfo info, PipeRoot root, - List filesInput, - Path fileParameters, + List filesInput, + FileLike fileParameters, @Nullable Map variableMap) throws IOException; @Override - protected void save(Path path, Map addParams, Map instanceParams) throws IOException + protected void save(FileLike path, Map addParams, Map instanceParams) throws IOException { if (addParams == null) addParams = new HashMap<>(); diff --git a/ms2/src/org/labkey/ms2/pipeline/ImportScanCountsUpgradeJob.java b/ms2/src/org/labkey/ms2/pipeline/ImportScanCountsUpgradeJob.java deleted file mode 100644 index e6e8714759..0000000000 --- a/ms2/src/org/labkey/ms2/pipeline/ImportScanCountsUpgradeJob.java +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Copyright (c) 2011-2018 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.ms2.pipeline; - -import org.labkey.api.data.CompareType; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.Sort; -import org.labkey.api.data.Table; -import org.labkey.api.data.TableSelector; -import org.labkey.api.pipeline.PipeRoot; -import org.labkey.api.pipeline.PipelineJob; -import org.labkey.api.query.FieldKey; -import org.labkey.api.util.DateUtil; -import org.labkey.api.util.FileUtil; -import org.labkey.api.util.URLHelper; -import org.labkey.api.view.ViewBackgroundInfo; -import org.labkey.ms2.MS2Fraction; -import org.labkey.ms2.MS2Importer; -import org.labkey.ms2.MS2Manager; - -import java.io.File; -import java.io.IOException; -import java.io.Serializable; -import java.net.URI; - -/** - * User: jeckels - * Date: 7/1/11 - */ -public class ImportScanCountsUpgradeJob extends PipelineJob implements Serializable -{ - private static final String UPGRADE_RUN = "Attempting to import scan counts for MS2 fraction #%s %s"; - private static final String UPGRADE_EXCEPTION = "An unexpected error occurred attempting to import scan counts for the MS2 fraction: %s, skipping."; - private static final String UPGRADE_STATS = "Upgrade job complete. Number of fractions checked: %s. Number of records update: %s."; - - // For serialization - protected ImportScanCountsUpgradeJob() {} - - public ImportScanCountsUpgradeJob(ViewBackgroundInfo info, PipeRoot root) throws IOException - { - super(null, info, root); - - File logFile = FileUtil.createTempFile("importScanCounts", ".log", root.ensureSystemDirectory()); - setLogFile(logFile); - } - - @Override - public URLHelper getStatusHref() - { - return null; - } - - @Override - public String getDescription() - { - return "Load MS2 scan counts for existing data"; - } - - @Override - public void run() - { - setStatus("IMPORTING", "Job started at: " + DateUtil.nowISO()); - - int count = 0; - int imported = 0; - - try - { - SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("ScanCount"), null, CompareType.ISBLANK); - filter.addCondition(FieldKey.fromParts("MzXmlURL"), null, CompareType.NONBLANK); - MS2Fraction[] fractions = new TableSelector(MS2Manager.getTableInfoFractions(), filter, new Sort("Fraction")).getArray(MS2Fraction.class); - info("Found " + fractions.length + " fractions to process"); - - for (MS2Fraction fraction : fractions) - { - info(String.format(UPGRADE_RUN, count++, fraction.getMzXmlURL())); - try - { - File f = new File(new URI(fraction.getMzXmlURL())); - MS2Importer.loadScanCounts(f, fraction); - if (fraction.getScanCount() != null) - { - Table.update(null, MS2Manager.getTableInfoFractions(), fraction, fraction.getFraction()); - imported++; - } - else - { - info("Unable to load scan counts for " + fraction.getMzXmlURL() + ", file may have been deleted"); - } - } - catch (Exception e) - { - error(String.format(UPGRADE_EXCEPTION, fraction.getMzXmlURL()), e); - } - } - } - catch (Exception e) - { - error("Import failure", e); - setStatus(TaskStatus.error, "Job finished at: " + DateUtil.nowISO()); - } - finally - { - info(String.format(UPGRADE_STATS, count, imported)); - setStatus(TaskStatus.complete, "Job finished at: " + DateUtil.nowISO()); - } - } -} diff --git a/ms2/src/org/labkey/ms2/pipeline/MS2ImportPipelineJob.java b/ms2/src/org/labkey/ms2/pipeline/MS2ImportPipelineJob.java index bcf51be43f..36131f6466 100644 --- a/ms2/src/org/labkey/ms2/pipeline/MS2ImportPipelineJob.java +++ b/ms2/src/org/labkey/ms2/pipeline/MS2ImportPipelineJob.java @@ -31,6 +31,7 @@ import org.labkey.ms2.MS2Importer; import org.labkey.ms2.MS2Manager; import org.labkey.ms2.MS2Run; +import org.labkey.vfs.FileLike; import javax.xml.stream.XMLStreamException; import java.io.File; @@ -42,13 +43,13 @@ */ public class MS2ImportPipelineJob extends PipelineJob { - protected final File _file; + protected final FileLike _file; private final String _description; private final MS2Importer.RunInfo _runInfo; @JsonCreator protected MS2ImportPipelineJob( - @JsonProperty("_file") File file, + @JsonProperty("_file") FileLike file, @JsonProperty("_description") String description, @JsonProperty("_runInfo") MS2Importer.RunInfo runInfo) { @@ -58,7 +59,7 @@ protected MS2ImportPipelineJob( _runInfo = runInfo; } - public MS2ImportPipelineJob(ViewBackgroundInfo info, File file, String description, MS2Importer.RunInfo runInfo, PipeRoot root) + public MS2ImportPipelineJob(ViewBackgroundInfo info, FileLike file, String description, MS2Importer.RunInfo runInfo, PipeRoot root) { super(MS2PipelineProvider.name, info, root); _file = file; @@ -70,8 +71,8 @@ public MS2ImportPipelineJob(ViewBackgroundInfo info, File file, String descripti // Use the PipeRoot to resolve the desired location for the .log file, which may // not be in the same directory as the imported file if there is a supplemental pipeline // directory configured. See issue 27440. - String relativePath = getPipeRoot().relativePath(_file.getParentFile()); - relativePath += File.separator + FT_LOG.getName(_file.getParentFile(), basename); + String relativePath = getPipeRoot().relativePath(_file.getParent()); + relativePath += File.separator + FT_LOG.getName(_file.getParent(), basename); File logFile = getPipeRoot().resolvePath(relativePath); setLogFile(logFile); diff --git a/ms2/src/org/labkey/ms2/pipeline/MS2PipelineManager.java b/ms2/src/org/labkey/ms2/pipeline/MS2PipelineManager.java index 976b056534..1794d8443f 100644 --- a/ms2/src/org/labkey/ms2/pipeline/MS2PipelineManager.java +++ b/ms2/src/org/labkey/ms2/pipeline/MS2PipelineManager.java @@ -21,12 +21,12 @@ import org.labkey.api.pipeline.*; import org.labkey.api.pipeline.cmd.ConvertTaskId; import org.labkey.api.security.User; -import org.labkey.api.util.FileType; import org.labkey.api.util.FileUtil; import org.labkey.api.util.NetworkDrive; +import org.labkey.api.util.Path; import org.labkey.api.view.NotFoundException; import org.labkey.ms2.pipeline.mascot.MascotSearchTask; -import org.labkey.ms2.pipeline.tandem.XTandemSearchTask; +import org.labkey.vfs.FileSystemLike; import java.io.*; import java.net.URI; @@ -62,7 +62,7 @@ public static class UploadFileFilter extends PipelineProvider.FileEntryFilter @Override public boolean accept(File file) { - if (MascotSearchTask.isNativeOutputFile(file)) + if (MascotSearchTask.isNativeOutputFile(FileSystemLike.wrapFile(file))) return true; if (TPPTask.isPepXMLFile(file)) @@ -77,15 +77,6 @@ public boolean accept(File file) } } - public static class XtanXmlFileFilter extends PipelineProvider.FileEntryFilter - { - @Override - public boolean accept(File file) - { - return XTandemSearchTask.getNativeFileType(FileType.gzSupportLevel.SUPPORT_GZ).isType(file); - } - } - public static PipelineProvider.FileEntryFilter getUploadFilter() { return new UploadFileFilter(); @@ -93,7 +84,7 @@ public static PipelineProvider.FileEntryFilter getUploadFilter() public static PipelineProvider.FileEntryFilter getAnalyzeFilter() { - TaskFactory factory = PipelineJobService.get().getTaskFactory(MZXML_CONVERTER_TASK_ID); + TaskFactory factory = PipelineJobService.get().getTaskFactory(MZXML_CONVERTER_TASK_ID); if (factory != null) return new PipelineProvider.FileTypesEntryFilter(factory.getInputTypes()); @@ -107,56 +98,6 @@ public boolean accept(File f) }; } - public static List getSequenceDirList(File dir, String path) - { - File[] dbFiles = dir.listFiles(new SequenceDbFileFilter()); - - if (dbFiles == null) - return null; - - ArrayList dirList = new ArrayList<>(); - - for (File dbFile : dbFiles) - { - if (dbFile.isDirectory()) - dirList.add(path + dbFile.getName() + "/"); - else - dirList.add(dbFile.getName()); - } - - return dirList; - } - - public static List addSequenceDbPaths(File dir, String path, List m) - { - File[] dbFiles = dir.listFiles(new SequenceDbFileFilter()); - - if (dbFiles == null) - return null; - ArrayList listSubdirs = new ArrayList<>(); - int fileCount = 0; - for (File dbFile : dbFiles) - { - if (dbFile.isDirectory()) - { - listSubdirs.add(dbFile); - m.add(path + dbFile.getName() + "/"); - } - else - { - fileCount++; - } - } - if(fileCount == 0) - { - m.remove(path); - } - for (File subdir : listSubdirs) - { - addSequenceDbPaths(subdir, path + subdir.getName() + "/", m); - } - return m; - } public static File getSequenceDBFile(File fileRoot, String name) { @@ -219,14 +160,14 @@ private static File getSequenceDatabaseRoot(PipeRoot root) return root.resolvePath(DEFAULT_FASTA_DIR); } - public static File getLocalMascotFile(String sequenceRoot, String db, String release) + public static File getLocalMascotFile(File sequenceRoot, String db, String release) { - return new File(sequenceRoot+File.separator+"mascot"+File.separator+db, release); + return FileUtil.appendPath(sequenceRoot, Path.parse("mascot/" + db + "/" + release)); } - public static File getLocalMascotFileHash(String sequenceRoot, String db, String release) + public static File getLocalMascotFileHash(File sequenceRoot, String db, String release) { - return new File(sequenceRoot+File.separator+"mascot"+File.separator+db, release+".hash"); + return FileUtil.appendPath(sequenceRoot, Path.parse("mascot/" + db + "/" + release+".hash")); } public static boolean exists(File file, Set knownFiles, Set checkedDirectories) diff --git a/ms2/src/org/labkey/ms2/pipeline/PipelineController.java b/ms2/src/org/labkey/ms2/pipeline/PipelineController.java index 3107275d13..73b6ad1249 100644 --- a/ms2/src/org/labkey/ms2/pipeline/PipelineController.java +++ b/ms2/src/org/labkey/ms2/pipeline/PipelineController.java @@ -48,6 +48,7 @@ import org.labkey.ms2.pipeline.sequest.SequestPipelineProvider; import org.labkey.ms2.pipeline.tandem.XTandemPipelineProvider; import org.labkey.ms2.pipeline.tandem.XTandemSearchProtocolFactory; +import org.labkey.vfs.FileLike; import org.springframework.validation.BindException; import org.springframework.validation.Errors; import org.springframework.web.servlet.ModelAndView; @@ -98,7 +99,7 @@ public void validateCommand(PipelinePathForm target, Errors errors) @Override public boolean handlePost(PipelinePathForm form, BindException errors) { - for (File file : form.getValidatedFiles(getContainer())) + for (FileLike file : form.getValidatedFiles(getContainer())) { if (!file.isFile()) { @@ -125,8 +126,8 @@ public boolean handlePost(PipelinePathForm form, BindException errors) { // If the data was created by our pipeline, try to get the name // to look like the normal generated name. - protocolName = file.getParentFile().getName(); - dirDataOriginal = file.getParentFile().getParentFile(); + protocolName = file.getParent().getName(); + dirDataOriginal = file.getParent().getParent().toNioPathForRead().toFile(); if (dirDataOriginal != null && dirDataOriginal.getName().equals(XTandemSearchProtocolFactory.get().getName())) { diff --git a/ms2/src/org/labkey/ms2/pipeline/ProteinProphetPipelineJob.java b/ms2/src/org/labkey/ms2/pipeline/ProteinProphetPipelineJob.java index 27d0724572..f6bf59d195 100644 --- a/ms2/src/org/labkey/ms2/pipeline/ProteinProphetPipelineJob.java +++ b/ms2/src/org/labkey/ms2/pipeline/ProteinProphetPipelineJob.java @@ -25,10 +25,10 @@ import org.labkey.ms2.*; import org.labkey.api.exp.ExperimentException; import org.labkey.api.exp.XarContext; +import org.labkey.vfs.FileLike; import javax.xml.stream.XMLStreamException; -import java.io.File; import java.io.IOException; import java.sql.SQLException; @@ -39,20 +39,20 @@ */ public class ProteinProphetPipelineJob extends PipelineJob { - private final File _file; + private final FileLike _file; @JsonCreator - protected ProteinProphetPipelineJob(@JsonProperty("_file") File file) + protected ProteinProphetPipelineJob(@JsonProperty("_file") FileLike file) { _file = file; } - public ProteinProphetPipelineJob(ViewBackgroundInfo info, File file, PipeRoot root) + public ProteinProphetPipelineJob(ViewBackgroundInfo info, FileLike file, PipeRoot root) { super(ProteinProphetPipelineProvider.NAME, info, root); _file = file; - setLogFile(new File(_file.getParentFile(), _file.getName() + ".log")); + setLogFile(_file.getParent().resolveChild(_file.getName() + ".log").toNioPathForWrite()); } @Override diff --git a/ms2/src/org/labkey/ms2/pipeline/QuantitationAlgorithm.java b/ms2/src/org/labkey/ms2/pipeline/QuantitationAlgorithm.java index 32b9b3dfdb..bf6e94aa24 100644 --- a/ms2/src/org/labkey/ms2/pipeline/QuantitationAlgorithm.java +++ b/ms2/src/org/labkey/ms2/pipeline/QuantitationAlgorithm.java @@ -22,6 +22,7 @@ import org.labkey.api.pipeline.WorkDirectory; import org.labkey.api.util.NetworkDrive; import org.labkey.api.util.Pair; +import org.labkey.vfs.FileLike; import java.io.File; import java.io.FileNotFoundException; @@ -40,7 +41,7 @@ public enum QuantitationAlgorithm xpress { @Override - public String[] getCommand(Map params, String pathMzXml, TPPTask.Factory factory, Pair configFile) + public String[] getCommand(Map params, String pathMzXml, TPPTask.Factory factory, Pair configFile) { return new String[] { "-X" + StringUtils.join(getCommonXpressQ3Params(params, pathMzXml).iterator(), ' ') }; } @@ -48,7 +49,7 @@ public String[] getCommand(Map params, String pathMzXml, TPPTask libra { @Override - public String[] getCommand(Map params, String pathMzXml, TPPTask.Factory factory, Pair configFile) throws PipelineJobException + public String[] getCommand(Map params, String pathMzXml, TPPTask.Factory factory, Pair configFile) throws PipelineJobException { String normalizationChannelString = params.get(ParameterNames.LIBRA_NORMALIZATION_CHANNEL_PARAM); if (normalizationChannelString == null) @@ -59,7 +60,7 @@ public String[] getCommand(Map params, String pathMzXml, TPPTask } @Override - protected Pair getConfigFile(Map params, PipeRoot root, WorkDirectory wd) throws PipelineJobException, IOException + protected Pair getConfigFile(Map params, PipeRoot root, WorkDirectory wd) throws PipelineJobException, IOException { String libraConfigName = params.get(ParameterNames.LIBRA_CONFIG_NAME_PARAM); if (libraConfigName == null) @@ -71,13 +72,13 @@ protected Pair getConfigFile(Map params, PipeRoot throw new PipelineJobException("Libra configuration files containing a space are not supported"); } LibraProtocolFactory factory = new LibraProtocolFactory(); - Path result = factory.getProtocolFile(root, libraConfigName, false); + FileLike result = factory.getProtocolFile(root, libraConfigName, false); if (!NetworkDrive.exists(result)) { throw new PipelineJobException("Libra config file does not exist: " + result); } - wd.inputFile(result.toFile(), true); - return new Pair<>(result.toFile(), TPPTask.LIBRA_CONFIG_INPUT_ROLE); + wd.inputFile(result, true); + return new Pair<>(result, TPPTask.LIBRA_CONFIG_INPUT_ROLE); }}; protected List getCommonXpressQ3Params(Map params, String pathMzXml) @@ -154,10 +155,10 @@ else if ("]".equals(aa)) /** @return Libra config file, in its original location, or null if we're not using Libra */ protected @Nullable - Pair getConfigFile(Map params, PipeRoot root, WorkDirectory wd) throws PipelineJobException, IOException + Pair getConfigFile(Map params, PipeRoot root, WorkDirectory wd) throws PipelineJobException, IOException { return null; } - public abstract String[] getCommand(Map params, String pathMzXml, TPPTask.Factory factory, Pair configFile) throws PipelineJobException, FileNotFoundException; + public abstract String[] getCommand(Map params, String pathMzXml, TPPTask.Factory factory, Pair configFile) throws PipelineJobException, FileNotFoundException; } diff --git a/ms2/src/org/labkey/ms2/pipeline/TPPTask.java b/ms2/src/org/labkey/ms2/pipeline/TPPTask.java index 2269e2da58..9ae3d92317 100644 --- a/ms2/src/org/labkey/ms2/pipeline/TPPTask.java +++ b/ms2/src/org/labkey/ms2/pipeline/TPPTask.java @@ -38,6 +38,7 @@ import org.labkey.api.util.Pair; import org.labkey.api.util.PepXMLFileType; import org.labkey.api.util.ProtXMLFileType; +import org.labkey.vfs.FileLike; import java.io.File; import java.io.FileNotFoundException; @@ -124,6 +125,11 @@ public static boolean isPepXMLFile(File file) return FT_PEP_XML.isType(file); } + public static boolean isPepXMLFile(FileLike file) + { + return FT_PEP_XML.isType(file); + } + public static File getProtXMLFile(File dirAnalysis, String baseName) { return FT_PROT_XML.newFile(dirAnalysis, baseName); @@ -288,7 +294,7 @@ public RecordedActionSet run() throws PipelineJobException QuantitationAlgorithm quantitationAlgorithm = getQuantitionAlgorithm(params); // Non-null if we're doing Libra quantitation - @Nullable Pair quantConfigFile = quantitationAlgorithm == null ? null : quantitationAlgorithm.getConfigFile(params, getJob().getPipeRoot(), _wd); + @Nullable Pair quantConfigFile = quantitationAlgorithm == null ? null : quantitationAlgorithm.getConfigFile(params, getJob().getPipeRoot(), _wd); // First step takes all the pepXMLs as inputs and either runs PeptideProphet (non-join) or rolls them up (join) RecordedAction pepXMLAction = new RecordedAction(PEPTIDE_PROPHET_ACTION_NAME); @@ -651,7 +657,7 @@ private QuantitationAlgorithm getQuantitionAlgorithm(Map params) return null; } - private String[] getQuantitationCmd(Map params, String pathMzXml, Pair configFile) throws FileNotFoundException, PipelineJobException + private String[] getQuantitationCmd(Map params, String pathMzXml, Pair configFile) throws FileNotFoundException, PipelineJobException { QuantitationAlgorithm paramAlgorithm = getQuantitionAlgorithm(params); if (paramAlgorithm == null) diff --git a/ms2/src/org/labkey/ms2/pipeline/comet/CometPipelineJob.java b/ms2/src/org/labkey/ms2/pipeline/comet/CometPipelineJob.java index dc681db3d2..66b3bc6aad 100644 --- a/ms2/src/org/labkey/ms2/pipeline/comet/CometPipelineJob.java +++ b/ms2/src/org/labkey/ms2/pipeline/comet/CometPipelineJob.java @@ -24,10 +24,10 @@ import org.labkey.ms2.pipeline.AbstractMS2SearchPipelineJob; import org.labkey.ms2.pipeline.sequest.SequestPipelineProvider; import org.labkey.ms2.pipeline.sequest.SequestSearchTask; +import org.labkey.vfs.FileLike; import java.io.File; import java.io.IOException; -import java.nio.file.Path; import java.util.List; /** @@ -48,8 +48,8 @@ public CometPipelineJob(CometSearchProtocol protocol, ViewBackgroundInfo info, PipeRoot root, String name, - List filesMzXML, - Path fileInputXML + List filesMzXML, + FileLike fileInputXML ) throws IOException { super(protocol, SequestPipelineProvider.name, info, root, name, fileInputXML, filesMzXML); @@ -58,13 +58,13 @@ public CometPipelineJob(CometSearchProtocol protocol, writeInputFilesToLog(); } - public CometPipelineJob(CometPipelineJob job, File fileFraction) + public CometPipelineJob(CometPipelineJob job, FileLike fileFraction) { super(job, fileFraction); } @Override - public AbstractFileAnalysisJob createSingleFileJob(File file) + public AbstractFileAnalysisJob createSingleFileJob(FileLike file) { return new CometPipelineJob(this, file); } diff --git a/ms2/src/org/labkey/ms2/pipeline/comet/CometSearchProtocol.java b/ms2/src/org/labkey/ms2/pipeline/comet/CometSearchProtocol.java index 75cd134d8d..5d2afec024 100644 --- a/ms2/src/org/labkey/ms2/pipeline/comet/CometSearchProtocol.java +++ b/ms2/src/org/labkey/ms2/pipeline/comet/CometSearchProtocol.java @@ -22,6 +22,7 @@ import org.labkey.api.util.FileUtil; import org.labkey.api.view.ViewBackgroundInfo; import org.labkey.ms2.pipeline.AbstractMS2SearchProtocol; +import org.labkey.vfs.FileLike; import java.io.File; import java.io.IOException; @@ -48,8 +49,8 @@ public CometSearchProtocolFactory getFactory() @Override public CometPipelineJob createPipelineJob(ViewBackgroundInfo info, - PipeRoot root, List filesInput, - Path fileParameters, @Nullable Map variableMap + PipeRoot root, List filesInput, + FileLike fileParameters, @Nullable Map variableMap ) throws IOException { return new CometPipelineJob(this, info, root, getName(), diff --git a/ms2/src/org/labkey/ms2/pipeline/mascot/MascotImportPipelineJob.java b/ms2/src/org/labkey/ms2/pipeline/mascot/MascotImportPipelineJob.java index 19c8abc0ad..1107593cb8 100644 --- a/ms2/src/org/labkey/ms2/pipeline/mascot/MascotImportPipelineJob.java +++ b/ms2/src/org/labkey/ms2/pipeline/mascot/MascotImportPipelineJob.java @@ -18,14 +18,13 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import org.apache.commons.io.FileUtils; import org.labkey.api.pipeline.PipeRoot; import org.labkey.api.util.FileUtil; import org.labkey.api.view.ViewBackgroundInfo; import org.labkey.ms2.MS2Importer; import org.labkey.ms2.pipeline.MS2ImportPipelineJob; +import org.labkey.vfs.FileLike; -import java.io.File; import java.io.IOException; /** @@ -36,14 +35,14 @@ public class MascotImportPipelineJob extends MS2ImportPipelineJob { @JsonCreator protected MascotImportPipelineJob( - @JsonProperty("_file") File file, + @JsonProperty("_file") FileLike file, @JsonProperty("_description") String description, @JsonProperty("_runInfo") MS2Importer.RunInfo runInfo) { super(file, description, runInfo); } - public MascotImportPipelineJob(ViewBackgroundInfo info, File file, String description, + public MascotImportPipelineJob(ViewBackgroundInfo info, FileLike file, String description, MS2Importer.RunInfo runInfo, PipeRoot root) { super(info, file, description, runInfo, root); @@ -59,27 +58,27 @@ public void run() return; } - String _dirAnalysis = _file.getParent(); - String _baseName = FileUtil.getBaseName(_file); - File dirWork = new File(_dirAnalysis, _baseName + ".import.work"); - File workFile = new File(dirWork.getAbsolutePath(), _file.getName()); + FileLike dirAnalysis = _file.getParent(); + String baseName = FileUtil.getBaseName(_file); + FileLike dirWork = dirAnalysis.resolveChild(baseName + ".import.work"); + FileLike workFile = dirWork.resolveChild(_file.getName()); boolean completeStatus = false; try { if (!dirWork.exists() && !FileUtil.mkdir(dirWork)) { - getLogger().error("Failed create working folder "+dirWork.getAbsolutePath()+"."); + getLogger().error("Failed create working folder "+dirWork+"."); return; } try { - FileUtils.copyFile(_file, workFile); + FileUtil.copyFile(_file, workFile); } catch (IOException x) { - getLogger().error("Failed to move Mascot result file to working folder as "+workFile.getAbsolutePath(), x); + getLogger().error("Failed to move Mascot result file to working folder as "+workFile, x); return; } @@ -90,12 +89,12 @@ public void run() if (!workFile.delete()) { - getLogger().error("Failed to delete " + workFile.getAbsolutePath()); + getLogger().error("Failed to delete " + workFile); return; } else if (!dirWork.delete()) { - getLogger().error("Failed to delete " + dirWork.getAbsolutePath()); + getLogger().error("Failed to delete " + dirWork); return; } else @@ -117,7 +116,14 @@ else if (!dirWork.delete()) } if (workFile.exists()) { - workFile.delete(); + try + { + workFile.delete(); + } + catch (IOException e) + { + getLogger().error("Failed to delete " + workFile, e); + } } } } diff --git a/ms2/src/org/labkey/ms2/pipeline/mascot/MascotPipelineJob.java b/ms2/src/org/labkey/ms2/pipeline/mascot/MascotPipelineJob.java index a85df4dd45..94e65d4a0a 100644 --- a/ms2/src/org/labkey/ms2/pipeline/mascot/MascotPipelineJob.java +++ b/ms2/src/org/labkey/ms2/pipeline/mascot/MascotPipelineJob.java @@ -22,6 +22,7 @@ import org.labkey.api.pipeline.file.AbstractFileAnalysisJob; import org.labkey.api.view.ViewBackgroundInfo; import org.labkey.ms2.pipeline.AbstractMS2SearchPipelineJob; +import org.labkey.vfs.FileLike; import java.io.File; import java.io.IOException; @@ -54,8 +55,8 @@ public MascotPipelineJob(MascotSearchProtocol protocol, ViewBackgroundInfo info, PipeRoot root, String name, - List filesMzXML, - Path fileInputXML) throws IOException + List filesMzXML, + FileLike fileInputXML) throws IOException { super(protocol, MascotCPipelineProvider.name, info, root, name, fileInputXML, filesMzXML); @@ -69,7 +70,7 @@ public MascotPipelineJob(MascotSearchProtocol protocol, writeInputFilesToLog(); } - public MascotPipelineJob(MascotPipelineJob job, File fileFraction) + public MascotPipelineJob(MascotPipelineJob job, FileLike fileFraction) { super(job, fileFraction); @@ -110,7 +111,7 @@ public TaskId getTaskPipelineId() } @Override - public AbstractFileAnalysisJob createSingleFileJob(File file) + public AbstractFileAnalysisJob createSingleFileJob(FileLike file) { return new MascotPipelineJob(this, file); } diff --git a/ms2/src/org/labkey/ms2/pipeline/mascot/MascotSearchProtocol.java b/ms2/src/org/labkey/ms2/pipeline/mascot/MascotSearchProtocol.java index fae22f0de7..179b4d2659 100644 --- a/ms2/src/org/labkey/ms2/pipeline/mascot/MascotSearchProtocol.java +++ b/ms2/src/org/labkey/ms2/pipeline/mascot/MascotSearchProtocol.java @@ -20,9 +20,9 @@ import org.labkey.api.pipeline.PipeRoot; import org.labkey.api.view.ViewBackgroundInfo; import org.labkey.ms2.pipeline.AbstractMS2SearchProtocol; +import org.labkey.vfs.FileLike; import java.io.IOException; -import java.nio.file.Path; import java.util.List; import java.util.Map; @@ -45,8 +45,8 @@ public MascotSearchProtocolFactory getFactory() @Override public MascotPipelineJob createPipelineJob(ViewBackgroundInfo info, - PipeRoot root, List filesInput, - Path fileParameters, @Nullable Map variableMap + PipeRoot root, List filesInput, + FileLike fileParameters, @Nullable Map variableMap ) throws IOException { return new MascotPipelineJob(this, info, root, getName(), diff --git a/ms2/src/org/labkey/ms2/pipeline/mascot/MascotSearchTask.java b/ms2/src/org/labkey/ms2/pipeline/mascot/MascotSearchTask.java index 6525798d6a..b13bf7d76c 100644 --- a/ms2/src/org/labkey/ms2/pipeline/mascot/MascotSearchTask.java +++ b/ms2/src/org/labkey/ms2/pipeline/mascot/MascotSearchTask.java @@ -34,6 +34,7 @@ import org.labkey.ms2.pipeline.MS2PipelineManager; import org.labkey.ms2.pipeline.MS2SearchJobSupport; import org.labkey.ms2.pipeline.TPPTask; +import org.labkey.vfs.FileLike; import java.io.BufferedInputStream; import java.io.BufferedReader; @@ -77,7 +78,7 @@ public static File getNativeOutputFile(File dirAnalysis, String baseName) return FT_MASCOT_DAT.newFile(dirAnalysis, baseName); } - public static boolean isNativeOutputFile(File file) + public static boolean isNativeOutputFile(FileLike file) { return FT_MASCOT_DAT.isType(file); } @@ -277,8 +278,8 @@ else if (exceptionClass.contains("java.io.filenotfoundexception")) long nmascotFileTimestamp= smascotFileTimestamp == null ? -1 : Long.parseLong(smascotFileTimestamp); File dirSequenceRoot = getJobSupport().getSequenceRootDirectory(); - File localDB = MS2PipelineManager.getLocalMascotFile(dirSequenceRoot.getPath(), sequenceDB, sequenceRelease); - File localDBHash = MS2PipelineManager.getLocalMascotFileHash(dirSequenceRoot.getPath(), sequenceDB, sequenceRelease); + File localDB = MS2PipelineManager.getLocalMascotFile(dirSequenceRoot, sequenceDB, sequenceRelease); + File localDBHash = MS2PipelineManager.getLocalMascotFileHash(dirSequenceRoot, sequenceDB, sequenceRelease); File localDBParent = localDB.getParentFile(); FileUtil.mkdirs(localDBParent); long filesize=0; @@ -338,7 +339,7 @@ else if (exceptionClass.contains("java.io.filenotfoundexception")) } // 2. translate Mascot result file to pep.xml format - File fileSequenceDatabase = MS2PipelineManager.getLocalMascotFile(dirSequenceRoot.getPath(), sequenceDB, sequenceRelease); + File fileSequenceDatabase = MS2PipelineManager.getLocalMascotFile(dirSequenceRoot, sequenceDB, sequenceRelease); String exePath = PipelineJobService.get().getExecutablePath("Mascot2XML", null, "tpp", ver, getJob().getLogger()); String[] args = { @@ -363,7 +364,7 @@ else if (exceptionClass.contains("java.io.filenotfoundexception")) // three possibilities: basename.xml, basename.pep.xml, basename.pep.xml.gz if (fileOutputPepXML.getName().endsWith(".gz")&&!fileWorkPepXMLRaw.getName().endsWith(".gz")) { - fileWorkPepXMLRaw = new File(fileWorkPepXMLRaw.getParent(),fileWorkPepXMLRaw.getName()+".gz"); + fileWorkPepXMLRaw = FileUtil.appendName(fileWorkPepXMLRaw.getParentFile(), fileWorkPepXMLRaw.getName()+".gz"); } if (!fileOutputPepXML.renameTo(fileWorkPepXMLRaw)) { diff --git a/ms2/src/org/labkey/ms2/pipeline/sequest/AbstractSequestSearchTaskFactory.java b/ms2/src/org/labkey/ms2/pipeline/sequest/AbstractSequestSearchTaskFactory.java index e7e8f83b90..1bb11fa290 100644 --- a/ms2/src/org/labkey/ms2/pipeline/sequest/AbstractSequestSearchTaskFactory.java +++ b/ms2/src/org/labkey/ms2/pipeline/sequest/AbstractSequestSearchTaskFactory.java @@ -62,6 +62,11 @@ public String getGroupParameterName() return "sequest"; } + public File getSequestInstallDirAsFile() + { + return _sequestInstallDir; + } + public String getSequestInstallDir() { return _sequestInstallDir == null ? null : _sequestInstallDir.getAbsolutePath(); diff --git a/ms2/src/org/labkey/ms2/pipeline/sequest/SequestPipelineJob.java b/ms2/src/org/labkey/ms2/pipeline/sequest/SequestPipelineJob.java index 036ce9a55e..1bb8af055f 100644 --- a/ms2/src/org/labkey/ms2/pipeline/sequest/SequestPipelineJob.java +++ b/ms2/src/org/labkey/ms2/pipeline/sequest/SequestPipelineJob.java @@ -21,10 +21,10 @@ import org.labkey.api.pipeline.file.AbstractFileAnalysisJob; import org.labkey.api.view.ViewBackgroundInfo; import org.labkey.ms2.pipeline.AbstractMS2SearchPipelineJob; +import org.labkey.vfs.FileLike; import java.io.File; import java.io.IOException; -import java.nio.file.Path; import java.util.List; /** @@ -48,8 +48,8 @@ public SequestPipelineJob(SequestSearchProtocol protocol, ViewBackgroundInfo info, PipeRoot root, String name, - List filesMzXML, - Path fileInputXML + List filesMzXML, + FileLike fileInputXML ) throws IOException { super(protocol, SequestPipelineProvider.name, info, root, name, fileInputXML, filesMzXML); @@ -58,13 +58,13 @@ public SequestPipelineJob(SequestSearchProtocol protocol, writeInputFilesToLog(); } - public SequestPipelineJob(SequestPipelineJob job, File fileFraction) + public SequestPipelineJob(SequestPipelineJob job, FileLike fileFraction) { super(job, fileFraction); } @Override - public AbstractFileAnalysisJob createSingleFileJob(File file) + public AbstractFileAnalysisJob createSingleFileJob(FileLike file) { return new SequestPipelineJob(this, file); } diff --git a/ms2/src/org/labkey/ms2/pipeline/sequest/SequestSearchProtocol.java b/ms2/src/org/labkey/ms2/pipeline/sequest/SequestSearchProtocol.java index c0310d207a..3a188a6730 100644 --- a/ms2/src/org/labkey/ms2/pipeline/sequest/SequestSearchProtocol.java +++ b/ms2/src/org/labkey/ms2/pipeline/sequest/SequestSearchProtocol.java @@ -23,6 +23,7 @@ import org.labkey.api.util.FileUtil; import org.labkey.api.view.ViewBackgroundInfo; import org.labkey.ms2.pipeline.AbstractMS2SearchProtocol; +import org.labkey.vfs.FileLike; import java.io.File; import java.io.IOException; @@ -50,8 +51,8 @@ public SequestSearchProtocolFactory getFactory() @Override public SequestPipelineJob createPipelineJob(ViewBackgroundInfo info, - PipeRoot root, List filesInput, - Path fileParameters, @Nullable Map variableMap + PipeRoot root, List filesInput, + FileLike fileParameters, @Nullable Map variableMap ) throws IOException { return new SequestPipelineJob(this, info, root, getName(), diff --git a/ms2/src/org/labkey/ms2/pipeline/sequest/SequestSearchTask.java b/ms2/src/org/labkey/ms2/pipeline/sequest/SequestSearchTask.java index 056caf6347..3c15f33319 100644 --- a/ms2/src/org/labkey/ms2/pipeline/sequest/SequestSearchTask.java +++ b/ms2/src/org/labkey/ms2/pipeline/sequest/SequestSearchTask.java @@ -216,7 +216,7 @@ private List getFASTAOrIndexFiles(List actions) throws Pip // Invoke makedb List args = new ArrayList<>(); - File makeDBExecutable = new File(_factory.getSequestInstallDir(), "makedb"); + File makeDBExecutable = FileUtil.appendName(_factory.getSequestInstallDirAsFile(), "makedb"); args.add(makeDBExecutable.getAbsolutePath()); args.add("-O" + indexFileBase); args.add("-P" + fileWorkParams.getAbsolutePath()); @@ -295,7 +295,7 @@ public RecordedActionSet run() throws PipelineJobException // Perform Sequest search List sequestArgs = new ArrayList<>(); - File sequestExecutable = new File(_factory.getSequestInstallDir(), "sequest"); + File sequestExecutable = FileUtil.appendName(_factory.getSequestInstallDirAsFile(), "sequest"); sequestArgs.add(sequestExecutable.getAbsolutePath()); sequestArgs.addAll(_factory.getSequestOptions()); sequestArgs.add("-R" + dtaListFile.getAbsolutePath()); @@ -312,7 +312,7 @@ public RecordedActionSet run() throws PipelineJobException // out2xml assumes that the mzXML file base name will match the DTA directory name, so rename the file // temporarily - File guidMzXMLFile = new File(localMzXML.getParent(), AbstractMS2SearchProtocol.FT_MZXML.getDefaultName(dtaDirName)); + File guidMzXMLFile = FileUtil.appendName(localMzXML.getParentFile(), AbstractMS2SearchProtocol.FT_MZXML.getDefaultName(dtaDirName)); if (useGUIDFilename) { localMzXML.renameTo(guidMzXMLFile); diff --git a/ms2/src/org/labkey/ms2/pipeline/tandem/XTandemPipelineJob.java b/ms2/src/org/labkey/ms2/pipeline/tandem/XTandemPipelineJob.java index c563b0295a..00e65a8f24 100644 --- a/ms2/src/org/labkey/ms2/pipeline/tandem/XTandemPipelineJob.java +++ b/ms2/src/org/labkey/ms2/pipeline/tandem/XTandemPipelineJob.java @@ -22,10 +22,10 @@ import org.labkey.api.pipeline.file.AbstractFileAnalysisJob; import org.labkey.api.view.ViewBackgroundInfo; import org.labkey.ms2.pipeline.AbstractMS2SearchPipelineJob; +import org.labkey.vfs.FileLike; import java.io.File; import java.io.IOException; -import java.nio.file.Path; import java.util.List; /** @@ -56,8 +56,8 @@ public XTandemPipelineJob(XTandemSearchProtocol protocol, ViewBackgroundInfo info, PipeRoot root, String name, - List filesMzXML, - Path fileInputXML) throws IOException + List filesMzXML, + FileLike fileInputXML) throws IOException { super(protocol, XTandemPipelineProvider.name, info, root, name, fileInputXML, filesMzXML); @@ -65,7 +65,7 @@ public XTandemPipelineJob(XTandemSearchProtocol protocol, writeInputFilesToLog(); } - public XTandemPipelineJob(XTandemPipelineJob job, File fileFraction) + public XTandemPipelineJob(XTandemPipelineJob job, FileLike fileFraction) { super(job, fileFraction); } @@ -77,7 +77,7 @@ public TaskId getTaskPipelineId() } @Override - public AbstractFileAnalysisJob createSingleFileJob(File file) + public AbstractFileAnalysisJob createSingleFileJob(FileLike file) { return new XTandemPipelineJob(this, file); } diff --git a/ms2/src/org/labkey/ms2/pipeline/tandem/XTandemSearchProtocol.java b/ms2/src/org/labkey/ms2/pipeline/tandem/XTandemSearchProtocol.java index 0905186891..9562ac34d1 100644 --- a/ms2/src/org/labkey/ms2/pipeline/tandem/XTandemSearchProtocol.java +++ b/ms2/src/org/labkey/ms2/pipeline/tandem/XTandemSearchProtocol.java @@ -20,6 +20,7 @@ import org.labkey.api.pipeline.PipeRoot; import org.labkey.api.view.ViewBackgroundInfo; import org.labkey.ms2.pipeline.AbstractMS2SearchProtocol; +import org.labkey.vfs.FileLike; import java.io.IOException; import java.nio.file.Path; @@ -48,8 +49,8 @@ public XTandemSearchProtocolFactory getFactory() @Override public XTandemPipelineJob createPipelineJob(ViewBackgroundInfo info, - PipeRoot root, List filesInput, - Path fileParameters, @Nullable Map variableMap + PipeRoot root, List filesInput, + FileLike fileParameters, @Nullable Map variableMap ) throws IOException { return new XTandemPipelineJob(this, info, root, getName(), diff --git a/ms2/src/org/labkey/ms2/reader/AbstractMzxmlIterator.java b/ms2/src/org/labkey/ms2/reader/AbstractMzxmlIterator.java index 144b1ad01d..c1a0245d94 100644 --- a/ms2/src/org/labkey/ms2/reader/AbstractMzxmlIterator.java +++ b/ms2/src/org/labkey/ms2/reader/AbstractMzxmlIterator.java @@ -16,8 +16,9 @@ package org.labkey.ms2.reader; +import org.labkey.vfs.FileLike; + import javax.xml.stream.XMLStreamException; -import java.io.File; import java.io.IOException; public abstract class AbstractMzxmlIterator implements SimpleScanIterator @@ -37,7 +38,7 @@ public void remove() throw new UnsupportedOperationException(); } - public static AbstractMzxmlIterator createParser(File file, int scanLevelFilter) throws IOException, XMLStreamException + public static AbstractMzxmlIterator createParser(FileLike file, int scanLevelFilter) throws IOException, XMLStreamException { return new SequentialMzxmlIterator(file, scanLevelFilter); } diff --git a/ms2/src/org/labkey/ms2/reader/MS2Loader.java b/ms2/src/org/labkey/ms2/reader/MS2Loader.java index f57b54dc2a..76f89b9a95 100644 --- a/ms2/src/org/labkey/ms2/reader/MS2Loader.java +++ b/ms2/src/org/labkey/ms2/reader/MS2Loader.java @@ -18,11 +18,14 @@ import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; +import org.labkey.api.util.NetworkDrive; import org.labkey.ms2.MS2Modification; +import org.labkey.vfs.FileLike; import javax.xml.stream.XMLStreamException; import java.io.File; import java.io.FileNotFoundException; +import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -35,19 +38,19 @@ public abstract class MS2Loader { protected Logger _log; protected long _fileLength; - protected File _file; + protected FileLike _file; protected static final int STREAM_BUFFER_SIZE = 128 * 1024; - protected void init(File f, Logger log) throws FileNotFoundException, XMLStreamException + protected void init(FileLike f, Logger log) throws XMLStreamException, IOException { - if (f.exists()) + if (NetworkDrive.exists(f)) { _file = f; - _fileLength = f.length(); + _fileLength = f.getSize(); } else - throw new FileNotFoundException(f.getAbsolutePath()); + throw new FileNotFoundException(f.toString()); _log = log; } diff --git a/ms2/src/org/labkey/ms2/reader/MS2XmlLoader.java b/ms2/src/org/labkey/ms2/reader/MS2XmlLoader.java index c11085caf7..fb247d4aa0 100644 --- a/ms2/src/org/labkey/ms2/reader/MS2XmlLoader.java +++ b/ms2/src/org/labkey/ms2/reader/MS2XmlLoader.java @@ -19,6 +19,7 @@ import org.apache.logging.log4j.Logger; import org.labkey.api.reader.SimpleXMLStreamReader; import org.labkey.api.util.PossiblyGZIPpedFileInputStreamFactory; +import org.labkey.vfs.FileLike; import javax.xml.stream.XMLStreamException; import java.io.BufferedInputStream; @@ -33,7 +34,7 @@ public abstract class MS2XmlLoader extends MS2Loader protected SimpleXMLStreamReader _parser; @Override - protected void init(File f, Logger log) throws FileNotFoundException, XMLStreamException + protected void init(FileLike f, Logger log) throws IOException, XMLStreamException { super.init(f, log); _fIn = new BufferedInputStream(PossiblyGZIPpedFileInputStreamFactory.getStream(f), STREAM_BUFFER_SIZE); diff --git a/ms2/src/org/labkey/ms2/reader/MascotDatLoader.java b/ms2/src/org/labkey/ms2/reader/MascotDatLoader.java index 9b59bd48dd..4f2bd6f5cf 100644 --- a/ms2/src/org/labkey/ms2/reader/MascotDatLoader.java +++ b/ms2/src/org/labkey/ms2/reader/MascotDatLoader.java @@ -27,6 +27,7 @@ import org.labkey.ms2.MS2RunType; import org.labkey.ms2.SpectrumException; import org.labkey.ms2.pipeline.MS2PipelineManager; +import org.labkey.vfs.FileLike; import javax.xml.stream.XMLStreamException; import java.io.BufferedReader; @@ -204,10 +205,10 @@ public enum Section { private static final String _nonStandardAminoAcids = "BJOUXZ"; - public MascotDatLoader(File f, Logger log) throws IOException, XMLStreamException + public MascotDatLoader(FileLike f, Logger log) throws IOException, XMLStreamException { init(f, log); - _reader = new BufferedReader(new InputStreamReader(new FileInputStream(f), StringUtilsLabKey.DEFAULT_CHARSET)); + _reader = new BufferedReader(new InputStreamReader(f.openInputStream(), StringUtilsLabKey.DEFAULT_CHARSET)); findBoundaryMarker(); } diff --git a/ms2/src/org/labkey/ms2/reader/PepXmlLoader.java b/ms2/src/org/labkey/ms2/reader/PepXmlLoader.java index 4bcfcdc697..7235f4e4a3 100644 --- a/ms2/src/org/labkey/ms2/reader/PepXmlLoader.java +++ b/ms2/src/org/labkey/ms2/reader/PepXmlLoader.java @@ -22,11 +22,11 @@ import org.labkey.api.protein.fasta.FastaProtein; import org.labkey.api.reader.SimpleXMLStreamReader; import org.labkey.ms2.MS2Modification; +import org.labkey.vfs.FileLike; import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamReader; -import java.io.File; -import java.io.FileNotFoundException; +import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; @@ -41,7 +41,7 @@ public class PepXmlLoader extends MS2XmlLoader private final ArrayList _quantSummaries = new ArrayList<>(); - public PepXmlLoader(File f, Logger log) throws FileNotFoundException, XMLStreamException + public PepXmlLoader(FileLike f, Logger log) throws IOException, XMLStreamException { init(f, log); readAnalysisSummaries(); diff --git a/ms2/src/org/labkey/ms2/reader/ProtXmlReader.java b/ms2/src/org/labkey/ms2/reader/ProtXmlReader.java index 40395b0333..f4cfac4fc7 100644 --- a/ms2/src/org/labkey/ms2/reader/ProtXmlReader.java +++ b/ms2/src/org/labkey/ms2/reader/ProtXmlReader.java @@ -21,25 +21,24 @@ import org.labkey.ms2.IcatProteinQuantitation; import org.labkey.ms2.MS2Run; import org.labkey.ms2.pipeline.sequest.SequestRun; +import org.labkey.vfs.FileLike; import javax.xml.stream.XMLStreamException; -import java.io.File; -import java.io.FileNotFoundException; import java.io.IOException; import java.util.*; public class ProtXmlReader { - private final File _file; + private final FileLike _file; private final MS2Run _run; - public ProtXmlReader(File file, MS2Run run) + public ProtXmlReader(FileLike file, MS2Run run) { _file = file; _run = run; } - public ProteinGroupIterator iterator() throws FileNotFoundException, XMLStreamException + public ProteinGroupIterator iterator() throws IOException, XMLStreamException { return new ProteinGroupIterator(); } @@ -50,7 +49,7 @@ public class ProteinGroupIterator implements Iterator private ProteinGroup _nextProteinGroup = null; private final java.io.InputStream _fIn; - public ProteinGroupIterator() throws FileNotFoundException, XMLStreamException + public ProteinGroupIterator() throws IOException, XMLStreamException { // TPP treats .xml.gz as a native format _fIn = PossiblyGZIPpedFileInputStreamFactory.getStream(_file); @@ -147,15 +146,6 @@ private static int parseIntAllowingNulls(String s) return Integer.parseInt(s); } - private static float parseFloatAllowingNulls(String s) - { - if (s == null) - { - return 0; - } - return Float.parseFloat(s); - } - public static class Protein implements Cloneable { private String _proteinName; diff --git a/ms2/src/org/labkey/ms2/reader/SequentialMzxmlIterator.java b/ms2/src/org/labkey/ms2/reader/SequentialMzxmlIterator.java index eb029e17fe..c0e8544c15 100644 --- a/ms2/src/org/labkey/ms2/reader/SequentialMzxmlIterator.java +++ b/ms2/src/org/labkey/ms2/reader/SequentialMzxmlIterator.java @@ -22,6 +22,7 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; import org.apache.xmlbeans.GDuration; +import org.labkey.vfs.FileLike; import org.systemsbiology.jrap.Scan; import javax.xml.stream.XMLStreamException; @@ -37,13 +38,13 @@ public class SequentialMzxmlIterator extends AbstractMzxmlIterator { private static final Logger _log = LogManager.getLogger(SequentialMzxmlIterator.class); - private final File _file; + private final FileLike _file; private InputStream _in; private SimpleXMLStreamReader _parser; private SimpleScan _currentScan; private static final int STREAM_BUFFER_SIZE = 128 * 1024; - public SequentialMzxmlIterator(File file, int msLevel) throws FileNotFoundException, XMLStreamException + public SequentialMzxmlIterator(FileLike file, int msLevel) throws IOException, XMLStreamException { super(msLevel); _file = file; From 968b4e441311521763c54c0dd3e79d971d28b593 Mon Sep 17 00:00:00 2001 From: labkey-jeckels Date: Sun, 19 Oct 2025 16:42:34 -0700 Subject: [PATCH 2/8] Test fixes, --- luminex/src/org/labkey/luminex/LuminexExclusionPipelineJob.java | 2 +- ms2/src/org/labkey/ms2/pipeline/ProteinProphetPipelineJob.java | 2 +- .../labkey/api/protein/annotation/DefaultAnnotationLoader.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/luminex/src/org/labkey/luminex/LuminexExclusionPipelineJob.java b/luminex/src/org/labkey/luminex/LuminexExclusionPipelineJob.java index a48d694e05..4d1a01b065 100644 --- a/luminex/src/org/labkey/luminex/LuminexExclusionPipelineJob.java +++ b/luminex/src/org/labkey/luminex/LuminexExclusionPipelineJob.java @@ -44,7 +44,7 @@ public LuminexExclusionPipelineJob(ViewBackgroundInfo info, PipeRoot root, Lumin { super(LuminexAssayProvider.NAME, info, root); - setLogFile(root.getLogDirectoryFileLike(true).resolveChild(FileUtil.makeFileNameWithTimestamp("luminex_exclusion", "log")).toNioPathForWrite()); + setLogFile(root.getLogDirectoryFileLike(true).resolveChild(FileUtil.makeFileNameWithTimestamp("luminex_exclusion", "log"))); _form = form; _exclusionType = LuminexManager.ExclusionType.valueOf(form.getTableName()); diff --git a/ms2/src/org/labkey/ms2/pipeline/ProteinProphetPipelineJob.java b/ms2/src/org/labkey/ms2/pipeline/ProteinProphetPipelineJob.java index f6bf59d195..f1307a9e8f 100644 --- a/ms2/src/org/labkey/ms2/pipeline/ProteinProphetPipelineJob.java +++ b/ms2/src/org/labkey/ms2/pipeline/ProteinProphetPipelineJob.java @@ -52,7 +52,7 @@ public ProteinProphetPipelineJob(ViewBackgroundInfo info, FileLike file, PipeRoo super(ProteinProphetPipelineProvider.NAME, info, root); _file = file; - setLogFile(_file.getParent().resolveChild(_file.getName() + ".log").toNioPathForWrite()); + setLogFile(_file.getParent().resolveChild(_file.getName() + ".log")); } @Override diff --git a/protein/api-src/org/labkey/api/protein/annotation/DefaultAnnotationLoader.java b/protein/api-src/org/labkey/api/protein/annotation/DefaultAnnotationLoader.java index f793694d87..60c2a70826 100644 --- a/protein/api-src/org/labkey/api/protein/annotation/DefaultAnnotationLoader.java +++ b/protein/api-src/org/labkey/api/protein/annotation/DefaultAnnotationLoader.java @@ -60,7 +60,7 @@ public DefaultAnnotationLoader(File file, ViewBackgroundInfo info, PipeRoot pipe { throw new IOException("Could not create directory for log file: " + logDir); } - setLogFile(logDir.resolveChild(file.getName() + "." + DateUtil.formatDateTime(new Date(), FORMAT_STRING) + ".log").toNioPathForWrite()); + setLogFile(logDir.resolveChild(file.getName() + "." + DateUtil.formatDateTime(new Date(), FORMAT_STRING) + ".log")); } @Override From 3b42b68176be36e252625304ac1c1fccf179af6a Mon Sep 17 00:00:00 2001 From: labkey-jeckels Date: Mon, 20 Oct 2025 11:33:28 -0700 Subject: [PATCH 3/8] Fix pepXML finding --- ms2/src/org/labkey/ms2/ProteinProphetImporter.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ms2/src/org/labkey/ms2/ProteinProphetImporter.java b/ms2/src/org/labkey/ms2/ProteinProphetImporter.java index ec84444687..00915161f5 100644 --- a/ms2/src/org/labkey/ms2/ProteinProphetImporter.java +++ b/ms2/src/org/labkey/ms2/ProteinProphetImporter.java @@ -382,6 +382,10 @@ private MS2Run importRun(ViewBackgroundInfo info, Logger log) throws IOException } } } + if (NetworkDrive.exists(pepXMLFile)) + { + break; + } } log.info("Resolved referenced PepXML file to " + pepXMLFile.getPath()); From c334173c163374a0866ad2c0797698140abd4af0 Mon Sep 17 00:00:00 2001 From: labkey-jeckels Date: Mon, 20 Oct 2025 14:01:34 -0700 Subject: [PATCH 4/8] Test fixes --- ms2/src/org/labkey/ms2/pipeline/PipelineController.java | 9 +++++---- .../src/org/labkey/test/tests/ms2/SequestImportTest.java | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/ms2/src/org/labkey/ms2/pipeline/PipelineController.java b/ms2/src/org/labkey/ms2/pipeline/PipelineController.java index 73b6ad1249..116fb2974b 100644 --- a/ms2/src/org/labkey/ms2/pipeline/PipelineController.java +++ b/ms2/src/org/labkey/ms2/pipeline/PipelineController.java @@ -57,6 +57,7 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.net.URI; +import java.util.Collections; /** * PipelineController @@ -113,7 +114,7 @@ public boolean handlePost(PipelinePathForm form, BindException errors) // to look like the normal generated name. String protocolName; - File dirDataOriginal; + FileLike dirDataOriginal; String description; if (MascotSearchTask.isNativeOutputFile(file)) { @@ -127,14 +128,14 @@ public boolean handlePost(PipelinePathForm form, BindException errors) // If the data was created by our pipeline, try to get the name // to look like the normal generated name. protocolName = file.getParent().getName(); - dirDataOriginal = file.getParent().getParent().toNioPathForRead().toFile(); + dirDataOriginal = file.getParent().getParent(); if (dirDataOriginal != null && dirDataOriginal.getName().equals(XTandemSearchProtocolFactory.get().getName())) { - dirDataOriginal = dirDataOriginal.getParentFile(); + dirDataOriginal = dirDataOriginal.getParent(); } description = AbstractFileAnalysisJob. - getDataDescription(dirDataOriginal, baseName, AbstractFileAnalysisProtocol.LEGACY_JOINED_BASENAME, protocolName); + getDataDescription(dirDataOriginal, baseName, AbstractFileAnalysisProtocol.LEGACY_JOINED_BASENAME, protocolName, Collections.emptyList()); } ViewBackgroundInfo info = getViewBackgroundInfo(); diff --git a/ms2/test/src/org/labkey/test/tests/ms2/SequestImportTest.java b/ms2/test/src/org/labkey/test/tests/ms2/SequestImportTest.java index 52dbcb431c..14a2c6c708 100644 --- a/ms2/test/src/org/labkey/test/tests/ms2/SequestImportTest.java +++ b/ms2/test/src/org/labkey/test/tests/ms2/SequestImportTest.java @@ -94,7 +94,7 @@ private void verifyRunGrid() private void testFileUsageColumn() { final String[] runFiles = {"ipi.HUMAN.fasta.v2.31", "raftflow10.mzXML", "raftflow10.pep.xml"}; - final String runName = "sampledata/raftflow10 (raftflow)"; + final String runName = "raftflow10"; final String runProtocol = "MS2 Import"; final Locator.XPathLocator runLink = Locator.linkWithText(runName + " (" + runProtocol + ")"); From 1ffd4a9b64a893f59d05c06ac0ce0f512526488e Mon Sep 17 00:00:00 2001 From: labkey-jeckels Date: Mon, 20 Oct 2025 17:07:40 -0700 Subject: [PATCH 5/8] More FileLike --- .../controllers/executescript/AnalysisScriptController.java | 2 +- flow/src/org/labkey/flow/controllers/run/RunController.java | 3 ++- flow/src/org/labkey/flow/script/FlowJob.java | 2 +- ms2/src/org/labkey/ms2/pipeline/MS2ImportPipelineJob.java | 4 ++-- ms2/src/org/labkey/ms2/pipeline/mascot/MascotSearchTask.java | 4 ++-- ms2/src/org/labkey/ms2/pipeline/tandem/XTandemSearchTask.java | 2 +- ms2/src/org/labkey/ms2/pipeline/tandem/XTandemToXMLTask.java | 2 +- .../api-src/org/labkey/api/protein/fasta/FastaDbLoader.java | 2 +- 8 files changed, 11 insertions(+), 10 deletions(-) diff --git a/flow/src/org/labkey/flow/controllers/executescript/AnalysisScriptController.java b/flow/src/org/labkey/flow/controllers/executescript/AnalysisScriptController.java index 0c7ceb52c9..cd81a4be96 100644 --- a/flow/src/org/labkey/flow/controllers/executescript/AnalysisScriptController.java +++ b/flow/src/org/labkey/flow/controllers/executescript/AnalysisScriptController.java @@ -544,7 +544,7 @@ public class ImportAnalysisFromPipelineAction extends SimpleViewAction returns = mascotClient.getDBInfo(sequenceDB, sequenceRelease); String status = returns.get("STATUS"); - if (null == status || !"OK".equals(status)) + if (!"OK".equals(status)) { getJob().error("Failed to get database from Mascot server."); String exceptionMessage=returns.get("exceptionmessage"); diff --git a/ms2/src/org/labkey/ms2/pipeline/tandem/XTandemSearchTask.java b/ms2/src/org/labkey/ms2/pipeline/tandem/XTandemSearchTask.java index ddb1b3a33e..4faccac6aa 100644 --- a/ms2/src/org/labkey/ms2/pipeline/tandem/XTandemSearchTask.java +++ b/ms2/src/org/labkey/ms2/pipeline/tandem/XTandemSearchTask.java @@ -85,7 +85,7 @@ public Factory() } @Override - public PipelineJob.Task createTask(PipelineJob job) + public XTandemSearchTask createTask(PipelineJob job) { return new XTandemSearchTask(this, job); } diff --git a/ms2/src/org/labkey/ms2/pipeline/tandem/XTandemToXMLTask.java b/ms2/src/org/labkey/ms2/pipeline/tandem/XTandemToXMLTask.java index 33be2271b5..9526af5e9e 100644 --- a/ms2/src/org/labkey/ms2/pipeline/tandem/XTandemToXMLTask.java +++ b/ms2/src/org/labkey/ms2/pipeline/tandem/XTandemToXMLTask.java @@ -51,7 +51,7 @@ public Factory() } @Override - public PipelineJob.Task createTask(PipelineJob job) + public XTandemToXMLTask createTask(PipelineJob job) { return new XTandemToXMLTask(this, job); } diff --git a/protein/api-src/org/labkey/api/protein/fasta/FastaDbLoader.java b/protein/api-src/org/labkey/api/protein/fasta/FastaDbLoader.java index 4999eea89f..721809eaa0 100644 --- a/protein/api-src/org/labkey/api/protein/fasta/FastaDbLoader.java +++ b/protein/api-src/org/labkey/api/protein/fasta/FastaDbLoader.java @@ -115,7 +115,7 @@ public void run() { try { - setLogFile(getLogFile()); + setLogFile(getLogFileLike()); info("Starting annotation load for " + _file); setStatus("RUNNING"); parseFile(getLogger()); From 2feb8efca5da3b8c3d064835022695a9d6a2c4ae Mon Sep 17 00:00:00 2001 From: labkey-jeckels Date: Tue, 21 Oct 2025 09:57:21 -0700 Subject: [PATCH 6/8] Test fixes --- ms2/test/src/org/labkey/test/tests/ms2/MS2Test.java | 2 +- ms2/test/src/org/labkey/test/tests/ms2/PeaksTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ms2/test/src/org/labkey/test/tests/ms2/MS2Test.java b/ms2/test/src/org/labkey/test/tests/ms2/MS2Test.java index 90b0b0aba3..33be277521 100644 --- a/ms2/test/src/org/labkey/test/tests/ms2/MS2Test.java +++ b/ms2/test/src/org/labkey/test/tests/ms2/MS2Test.java @@ -1158,7 +1158,7 @@ private void pepXMLtest() { clickButton("Process and Import Data"); _fileBrowserHelper.importFile("pepXML/truncated.pep.xml", "Import Search Results"); - String ms2Run = "ms2pipe/truncated (pepXML)"; + String ms2Run = "truncated (pepXML)"; waitForRunningPipelineJobs(defaultWaitForPage); clickAndWait(Locator.linkWithText(ms2Run)); diff --git a/ms2/test/src/org/labkey/test/tests/ms2/PeaksTest.java b/ms2/test/src/org/labkey/test/tests/ms2/PeaksTest.java index 9e22aec9cc..fbfa6372bb 100644 --- a/ms2/test/src/org/labkey/test/tests/ms2/PeaksTest.java +++ b/ms2/test/src/org/labkey/test/tests/ms2/PeaksTest.java @@ -84,7 +84,7 @@ public void testPeaksImport() clickButton("Process and Import Data"); _fileBrowserHelper.importFile("peaks/peaks.pep.xml", "Import Search Results"); - String runLabel = "ms2pipe/peaks (peaks)"; + String runLabel = "peaks (peaks)"; waitForRunningPipelineJobs(MAX_WAIT_SECONDS * 1000); waitForElement(Locator.linkWithText(runLabel)); From 2602990224afebd85046a4cfb6548cf428538e7d Mon Sep 17 00:00:00 2001 From: labkey-jeckels Date: Wed, 22 Oct 2025 09:09:15 -0700 Subject: [PATCH 7/8] Test fixes --- ms2/test/src/org/labkey/test/tests/ms2/MS2Test.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ms2/test/src/org/labkey/test/tests/ms2/MS2Test.java b/ms2/test/src/org/labkey/test/tests/ms2/MS2Test.java index 33be277521..1d6a09cd3f 100644 --- a/ms2/test/src/org/labkey/test/tests/ms2/MS2Test.java +++ b/ms2/test/src/org/labkey/test/tests/ms2/MS2Test.java @@ -1168,7 +1168,7 @@ private void pepXMLtest() assertEquals("Incorrect number of graphs", 1, getElementCount(Locator.tag("img").withAttributeContaining("src", WebTestHelper.buildRelativeUrl("ms2", "MS2VerifyProject/ms2folder", "showPeptideProphetSensitivityPlot")))); assertEquals("Incorrect number of graphs", 6, getElementCount(Locator.tag("img").withAttributeContaining("src", WebTestHelper.buildRelativeUrl("ms2", "MS2VerifyProject/ms2folder", "showPeptideProphetDistributionPlot")))); assertEquals("Incorrect number of graphs", 6, getElementCount(Locator.tag("img").withAttributeContaining("src", WebTestHelper.buildRelativeUrl("ms2", "MS2VerifyProject/ms2folder", "showPeptideProphetObservedVsModelPlot")))); - assertTextPresent("PeptideProphet Details: ms2pipe/truncated (pepXML)"); + assertTextPresent("PeptideProphet Details: truncated (pepXML)"); getDriver().close(); switchToMainWindow(); } From 4945244f1de45388a0ce5206d2c196ec81dbac73 Mon Sep 17 00:00:00 2001 From: labkey-jeckels Date: Thu, 23 Oct 2025 18:34:40 -0700 Subject: [PATCH 8/8] Issue 54156: Assay XAR with transform script can't be edited if the transform script path is invalid for the given server Misc code cleanup --- .../labkey/flow/data/FlowAssayProvider.java | 35 +------------------ .../labkey/luminex/LuminexAssayProvider.java | 7 ++-- .../luminex/query/WellExclusionTable.java | 13 ++++--- .../matrix/ExpressionMatrixAssayProvider.java | 3 +- nab/src/org/labkey/nab/NabAssayProvider.java | 7 ++-- .../HighThroughputNabAssayProvider.java | 3 +- 6 files changed, 16 insertions(+), 52 deletions(-) diff --git a/flow/src/org/labkey/flow/data/FlowAssayProvider.java b/flow/src/org/labkey/flow/data/FlowAssayProvider.java index 6a1f2c4a59..3a5f51e1fd 100644 --- a/flow/src/org/labkey/flow/data/FlowAssayProvider.java +++ b/flow/src/org/labkey/flow/data/FlowAssayProvider.java @@ -27,10 +27,8 @@ import org.labkey.api.exp.OntologyManager; import org.labkey.api.exp.XarContext; import org.labkey.api.exp.api.ExpData; -import org.labkey.api.exp.api.ExpExperiment; import org.labkey.api.exp.api.ExpProtocol; import org.labkey.api.exp.api.ExpRun; -import org.labkey.api.exp.api.IAssayDomainType; import org.labkey.api.exp.property.Domain; import org.labkey.api.exp.property.DomainProperty; import org.labkey.api.gwt.client.DefaultValueType; @@ -68,7 +66,6 @@ import org.labkey.flow.script.FlowPipelineProvider; import org.labkey.flow.view.FlowQueryView; import org.labkey.flow.webparts.AnalysesWebPart; -import org.springframework.validation.BindException; import org.springframework.web.servlet.ModelAndView; import org.springframework.web.servlet.mvc.Controller; @@ -214,7 +211,7 @@ public Domain getResultsDomain(ExpProtocol protocol, boolean forUpdate) } @Override - public AssayRunCreator getRunCreator() + public AssayRunCreator getRunCreator() { throw new UnsupportedOperationException(); } @@ -399,18 +396,6 @@ public DefaultValueType getDefaultValueDefault(Domain domain) throw new UnsupportedOperationException(); } - @Override - public boolean hasCustomView(IAssayDomainType domainType, boolean details) - { - return false; - } - - @Override - public ModelAndView createBeginView(ViewContext context, ExpProtocol protocol) - { - return null; - } - @Override public ModelAndView createBatchesView(ViewContext context, ExpProtocol protocol) { @@ -423,12 +408,6 @@ public ModelAndView createBatchesView(ViewContext context, ExpProtocol protocol) return view; } - @Override - public ModelAndView createBatchDetailsView(ViewContext context, ExpProtocol protocol, ExpExperiment batch) - { - return null; - } - @Override public ModelAndView createRunsView(ViewContext context, ExpProtocol protocol) { @@ -437,18 +416,6 @@ public ModelAndView createRunsView(ViewContext context, ExpProtocol protocol) return new FlowQueryView(form); } - @Override - public ModelAndView createRunDetailsView(ViewContext context, ExpProtocol protocol, ExpRun run) - { - return null; - } - - @Override - public ModelAndView createResultsView(ViewContext context, ExpProtocol protocol, BindException errors) - { - return null; - } - @Override public ModelAndView createResultDetailsView(ViewContext context, ExpProtocol protocol, ExpData data, Object dataRowId) { diff --git a/luminex/src/org/labkey/luminex/LuminexAssayProvider.java b/luminex/src/org/labkey/luminex/LuminexAssayProvider.java index 315bc7385e..f315abe36e 100644 --- a/luminex/src/org/labkey/luminex/LuminexAssayProvider.java +++ b/luminex/src/org/labkey/luminex/LuminexAssayProvider.java @@ -51,7 +51,6 @@ import org.labkey.api.assay.AssayDataType; import org.labkey.api.assay.AssayPipelineProvider; import org.labkey.api.assay.AssayProtocolSchema; -import org.labkey.api.assay.AssayRunCreator; import org.labkey.api.assay.AssayRunDatabaseContext; import org.labkey.api.assay.AssayRunUploadContext; import org.labkey.api.assay.AssayTableMetadata; @@ -113,7 +112,7 @@ public LuminexProtocolSchema createProtocolSchema(User user, Container container public void registerLsidHandler() { super.registerLsidHandler(); - LsidManager.get().registerHandler(LUMINEX_DATA_ROW_LSID_PREFIX, new LsidManager.ExpObjectLsidHandler() + LsidManager.get().registerHandler(LUMINEX_DATA_ROW_LSID_PREFIX, new LsidManager.ExpObjectLsidHandler<>() { @Override public ExpData getObject(Lsid lsid) @@ -454,7 +453,7 @@ public void deleteProtocol(ExpProtocol protocol, User user, @Nullable final Stri } @Override - public AssayRunDatabaseContext createRunDatabaseContext(ExpRun run, User user, HttpServletRequest request) + public AssayRunDatabaseContext createRunDatabaseContext(ExpRun run, User user, HttpServletRequest request) { return new LuminexRunDatabaseContext(run, user, request); } @@ -466,7 +465,7 @@ public AssayRunAsyncContext createRunAsyncContext(AssayRunUploadContext co } @Override - public AssayRunCreator getRunCreator() + public LuminexRunCreator getRunCreator() { return new LuminexRunCreator(this); } diff --git a/luminex/src/org/labkey/luminex/query/WellExclusionTable.java b/luminex/src/org/labkey/luminex/query/WellExclusionTable.java index 3dda61f9bd..33d6dd7bc8 100644 --- a/luminex/src/org/labkey/luminex/query/WellExclusionTable.java +++ b/luminex/src/org/labkey/luminex/query/WellExclusionTable.java @@ -73,10 +73,10 @@ public WellExclusionTable(LuminexProtocolSchema schema, ContainerFilter cf, bool { super(LuminexProtocolSchema.getTableInfoWellExclusion(), schema, cf, filter); - getMutableColumn("DataId").setLabel("Data File"); - getMutableColumn("DataId").setFk(new ExpSchema(schema.getUser(), schema.getContainer()).getDataIdForeignKey(cf)); + getMutableColumnOrThrow("DataId").setLabel("Data File"); + getMutableColumnOrThrow("DataId").setFk(new ExpSchema(schema.getUser(), schema.getContainer()).getDataIdForeignKey(cf)); - getMutableColumn("Analytes").setFk(new MultiValuedForeignKey(new LookupForeignKey(cf, "WellExclusionId", null) + getMutableColumnOrThrow("Analytes").setFk(new MultiValuedForeignKey(new LookupForeignKey(cf, "WellExclusionId", null) { @Override public TableInfo getLookupTableInfo() @@ -84,7 +84,7 @@ public TableInfo getLookupTableInfo() return _userSchema.createWellExclusionAnalyteTable(getLookupContainerFilter()); } }, "AnalyteId")); - getMutableColumn("Analytes").setUserEditable(false); + getMutableColumnOrThrow("Analytes").setUserEditable(false); SQLFragment joinSQL = new SQLFragment(" FROM "); joinSQL.append(LuminexProtocolSchema.getTableInfoDataRow(), "dr"); @@ -123,8 +123,7 @@ public Object getDisplayValue(RenderContext ctx) if (null != result) { // get the list of unique wells (by splitting the concatenated string) - TreeSet uniqueWells = new TreeSet<>(); - uniqueWells.addAll(Arrays.asList(result.toString().split(MultiValuedRenderContext.VALUE_DELIMITER_REGEX))); + TreeSet uniqueWells = new TreeSet<>(Arrays.asList(result.toString().split(MultiValuedRenderContext.VALUE_DELIMITER_REGEX))); // put the unique wells back into a comma separated string StringBuilder sb = new StringBuilder(); @@ -332,7 +331,7 @@ private void rerunTransformScripts(BatchValidationException errors) throws Query for (ExpRun run : _runsToRefresh) { AssayProvider provider = AssayService.get().getProvider(run); - AssayRunDatabaseContext context = provider.createRunDatabaseContext(run, _userSchema.getUser(), null); + AssayRunDatabaseContext context = provider.createRunDatabaseContext(run, _userSchema.getUser(), null); provider.getRunCreator().saveExperimentRun(context, AssayService.get().findBatch(run), run, false); } } diff --git a/microarray/src/org/labkey/microarray/matrix/ExpressionMatrixAssayProvider.java b/microarray/src/org/labkey/microarray/matrix/ExpressionMatrixAssayProvider.java index 42e730c241..9274d256b0 100644 --- a/microarray/src/org/labkey/microarray/matrix/ExpressionMatrixAssayProvider.java +++ b/microarray/src/org/labkey/microarray/matrix/ExpressionMatrixAssayProvider.java @@ -26,7 +26,6 @@ import org.labkey.api.assay.AssayDataType; import org.labkey.api.assay.AssayPipelineProvider; import org.labkey.api.assay.AssayProtocolSchema; -import org.labkey.api.assay.AssayRunCreator; import org.labkey.api.assay.AssayTableMetadata; import org.labkey.api.assay.actions.AssayRunUploadForm; import org.labkey.api.assay.matrix.ColumnMappingProperty; @@ -93,7 +92,7 @@ public AssayTableMetadata getTableMetadata(@NotNull ExpProtocol protocol) } @Override - public AssayRunCreator getRunCreator() + public ExpressionMatrixRunCreator getRunCreator() { return new ExpressionMatrixRunCreator(this); } diff --git a/nab/src/org/labkey/nab/NabAssayProvider.java b/nab/src/org/labkey/nab/NabAssayProvider.java index 6028c1f8be..c5e4c0c5e8 100644 --- a/nab/src/org/labkey/nab/NabAssayProvider.java +++ b/nab/src/org/labkey/nab/NabAssayProvider.java @@ -23,7 +23,6 @@ import org.labkey.api.assay.AssayPipelineProvider; import org.labkey.api.assay.AssayProvider; import org.labkey.api.assay.AssayProviderSchema; -import org.labkey.api.assay.AssayRunCreator; import org.labkey.api.assay.AssaySchema; import org.labkey.api.assay.AssayUrls; import org.labkey.api.assay.actions.AssayRunUploadForm; @@ -32,6 +31,7 @@ import org.labkey.api.assay.dilution.DilutionDataHandler; import org.labkey.api.assay.nab.NabSpecimen; import org.labkey.api.assay.plate.Plate; +import org.labkey.api.assay.plate.PlateBasedRunCreator; import org.labkey.api.assay.plate.PlateSamplePropertyHelper; import org.labkey.api.assay.plate.WellGroup; import org.labkey.api.data.Container; @@ -55,6 +55,7 @@ import org.labkey.api.module.ModuleLoader; import org.labkey.api.pipeline.PipelineProvider; import org.labkey.api.query.FieldKey; +import org.labkey.api.query.SchemaKey; import org.labkey.api.security.User; import org.labkey.api.security.permissions.Permission; import org.labkey.api.study.assay.ParticipantVisitResolverType; @@ -201,7 +202,7 @@ protected void addPassThroughSampleWellGroupProperties(Container c, Domain sampl addProperty(sampleWellGroupDomain, SAMPLE_DILUTION_FACTOR_PROPERTY_NAME, SAMPLE_DILUTION_FACTOR_PROPERTY_CAPTION, PropertyType.DOUBLE).setRequired(true); DomainProperty method = addProperty(sampleWellGroupDomain, SAMPLE_METHOD_PROPERTY_NAME, SAMPLE_METHOD_PROPERTY_CAPTION, PropertyType.STRING); method.setImportAliasSet(new HashSet<>(Collections.singletonList("Well Method"))); - method.setLookup(new Lookup(lookupContainer, AssaySchema.NAME + "." + getResourceName(), NabProviderSchema.SAMPLE_PREPARATION_METHOD_TABLE_NAME)); + method.setLookup(new Lookup(lookupContainer, SchemaKey.fromParts(AssaySchema.NAME, getResourceName()), NabProviderSchema.SAMPLE_PREPARATION_METHOD_TABLE_NAME)); method.setRequired(true); } @@ -374,7 +375,7 @@ public PlateSamplePropertyHelper getVirusPropertyHelper(PlateUploadForm context, } @Override - public AssayRunCreator getRunCreator() + public PlateBasedRunCreator getRunCreator() { return new NabRunCreator(this); } diff --git a/nab/src/org/labkey/nab/multiplate/HighThroughputNabAssayProvider.java b/nab/src/org/labkey/nab/multiplate/HighThroughputNabAssayProvider.java index 3e361839b4..bb6ead6189 100644 --- a/nab/src/org/labkey/nab/multiplate/HighThroughputNabAssayProvider.java +++ b/nab/src/org/labkey/nab/multiplate/HighThroughputNabAssayProvider.java @@ -26,7 +26,6 @@ import org.labkey.api.exp.property.Domain; import org.labkey.api.assay.actions.AssayRunUploadForm; import org.labkey.api.assay.AssayDataType; -import org.labkey.api.assay.AssayRunCreator; import org.labkey.api.query.QuerySettings; import org.labkey.api.security.User; import org.labkey.api.study.assay.ParticipantVisitResolverType; @@ -107,7 +106,7 @@ public boolean supportsMultiVirusPlate() } @Override - public AssayRunCreator getRunCreator() + public PlateBasedRunCreator getRunCreator() { return new PlateBasedRunCreator<>(this); }