diff --git a/mGAP/src/org/labkey/mgap/pipeline/GenerateMgapTracksStep.java b/mGAP/src/org/labkey/mgap/pipeline/GenerateMgapTracksStep.java index 484b9c00e..c1d89037f 100644 --- a/mGAP/src/org/labkey/mgap/pipeline/GenerateMgapTracksStep.java +++ b/mGAP/src/org/labkey/mgap/pipeline/GenerateMgapTracksStep.java @@ -565,7 +565,7 @@ private Map getSampleToAlias(File input) throws PipelineJobExcep } @Override - public void performAdditionalMergeTasks(SequenceOutputHandler.JobContext ctx, PipelineJob job, TaskFileManager manager, ReferenceGenome genome, List orderedScatterOutputs, List orderedJobDirs) throws PipelineJobException + public void performAdditionalMergeTasks(SequenceOutputHandler.JobContext ctx, PipelineJob job, ReferenceGenome genome, List orderedScatterOutputs, List orderedJobDirs) throws PipelineJobException { job.getLogger().info("Merging additional track VCFs"); Map> trackToSamples = parseSampleMap(getSampleNameFile(getPipelineCtx().getSourceDirectory(true))); @@ -609,7 +609,7 @@ public void performAdditionalMergeTasks(SequenceOutputHandler.JobContext ctx, Pi so.setCategory(TRACK_CATEGORY); so.setLibrary_id(genome.getGenomeId()); so.setDescription("mGAP track: " + trackName + ", total samples: " + trackToSamples.get(trackName).size()); - manager.addSequenceOutput(so); + ctx.getFileManager().addSequenceOutput(so); } if (getAnnotationReferenceVcf() != null) @@ -653,7 +653,7 @@ public void performAdditionalMergeTasks(SequenceOutputHandler.JobContext ctx, Pi so.setCategory(TRACK_CATEGORY); so.setLibrary_id(genome.getGenomeId()); so.setDescription("These are novel sites in mGAP v" + releaseVersion + " for " + species); - manager.addSequenceOutput(so); + ctx.getFileManager().addSequenceOutput(so); } } diff --git a/mGAP/src/org/labkey/mgap/pipeline/GroupCompareStep.java b/mGAP/src/org/labkey/mgap/pipeline/GroupCompareStep.java index 1fd0e68ab..050514617 100644 --- a/mGAP/src/org/labkey/mgap/pipeline/GroupCompareStep.java +++ b/mGAP/src/org/labkey/mgap/pipeline/GroupCompareStep.java @@ -84,7 +84,7 @@ public GroupCompareStep create(PipelineContext ctx) } @Override - public void performAdditionalMergeTasks(SequenceOutputHandler.JobContext ctx, PipelineJob job, TaskFileManager manager, ReferenceGenome genome, List orderedScatterOutputs, List orderedJobDirs) throws PipelineJobException + public void performAdditionalMergeTasks(SequenceOutputHandler.JobContext ctx, PipelineJob job, ReferenceGenome genome, List orderedScatterOutputs, List orderedJobDirs) throws PipelineJobException { job.getLogger().info("Merging variant tables"); List toConcat = orderedScatterOutputs.stream().map(f -> { @@ -138,7 +138,7 @@ public void performAdditionalMergeTasks(SequenceOutputHandler.JobContext ctx, Pi so.setFile(combined); so.setCategory("Variant List"); so.setLibrary_id(genome.getGenomeId()); - manager.addSequenceOutput(so); + ctx.getFileManager().addSequenceOutput(so); } @Override diff --git a/mGAP/src/org/labkey/mgap/pipeline/mGapReleaseComparisonStep.java b/mGAP/src/org/labkey/mgap/pipeline/mGapReleaseComparisonStep.java index 1caeab897..4f1929ff5 100644 --- a/mGAP/src/org/labkey/mgap/pipeline/mGapReleaseComparisonStep.java +++ b/mGAP/src/org/labkey/mgap/pipeline/mGapReleaseComparisonStep.java @@ -109,7 +109,7 @@ public Output processVariants(File inputVCF, File outputDirectory, ReferenceGeno } @Override - public void performAdditionalMergeTasks(SequenceOutputHandler.JobContext ctx, PipelineJob job, TaskFileManager manager, ReferenceGenome genome, List orderedScatterOutputs, List orderedJobDirs) throws PipelineJobException + public void performAdditionalMergeTasks(SequenceOutputHandler.JobContext ctx, PipelineJob job, ReferenceGenome genome, List orderedScatterOutputs, List orderedJobDirs) throws PipelineJobException { job.getLogger().info("Merging missing sites VCFs"); List toConcat = orderedScatterOutputs.stream().map(f -> { @@ -142,6 +142,6 @@ public void performAdditionalMergeTasks(SequenceOutputHandler.JobContext ctx, Pi so.setFile(combined); so.setCategory("Missing Sites VCF"); so.setLibrary_id(genome.getGenomeId()); - manager.addSequenceOutput(so); + ctx.getFileManager().addSequenceOutput(so); } } diff --git a/mcc/package-lock.json b/mcc/package-lock.json index 9091b05dc..7a08fcbca 100644 --- a/mcc/package-lock.json +++ b/mcc/package-lock.json @@ -15,6 +15,7 @@ "@mui/styles": "^5.14.7", "@mui/x-data-grid": "^6.0.0", "chart.js": "^4.4.4", + "google-palette": "^1.1.1", "react": "^17.0.2", "react-chartjs-2": "^5.2.0", "react-dom": "^17.0.2", @@ -5642,10 +5643,11 @@ } }, "node_modules/cross-spawn": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", - "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", "dev": true, + "license": "MIT", "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", @@ -6573,9 +6575,9 @@ } }, "node_modules/express": { - "version": "4.21.1", - "resolved": "https://registry.npmjs.org/express/-/express-4.21.1.tgz", - "integrity": "sha512-YSFlK1Ee0/GC8QaO91tHcDxJiE/X4FbpAyQWkxAvG6AXCuR65YzK8ua6D9hvi/TzUfZMpc+BwuM1IPw8fmQBiQ==", + "version": "4.21.2", + "resolved": "https://registry.npmjs.org/express/-/express-4.21.2.tgz", + "integrity": "sha512-28HqgMZAmih1Czt9ny7qr6ek2qddF4FclbMzwhCREB6OFfH+rXAnuNCwo1/wFvrtbgsQDb4kSbX9de9lFbrXnA==", "dev": true, "license": "MIT", "dependencies": { @@ -6598,7 +6600,7 @@ "methods": "~1.1.2", "on-finished": "2.4.1", "parseurl": "~1.3.3", - "path-to-regexp": "0.1.10", + "path-to-regexp": "0.1.12", "proxy-addr": "~2.0.7", "qs": "6.13.0", "range-parser": "~1.2.1", @@ -6613,6 +6615,10 @@ }, "engines": { "node": ">= 0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/express/node_modules/debug": { @@ -7475,6 +7481,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/google-palette": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/google-palette/-/google-palette-1.1.1.tgz", + "integrity": "sha512-yZiM5oLl8lCZzf06IMOGdDkxqvCMd9HNFcCiOMqWgGGiGzC22vWBVhKJNvykXXbeC0NAElNH97jA/y0bq6TCrA==", + "license": "Apache-2.0" + }, "node_modules/gopd": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", @@ -10949,9 +10961,9 @@ "dev": true }, "node_modules/nanoid": { - "version": "3.3.7", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.7.tgz", - "integrity": "sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g==", + "version": "3.3.8", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.8.tgz", + "integrity": "sha512-WNLf5Sd8oZxOm+TzppcYk8gVOgP+l58xNy58D0nbUnOxOWRWvlcCV4kUF7ltmI6PsrLl/BgKEyS4mqsGChFN0w==", "dev": true, "funding": [ { @@ -10959,6 +10971,7 @@ "url": "https://github.com/sponsors/ai" } ], + "license": "MIT", "bin": { "nanoid": "bin/nanoid.cjs" }, @@ -11560,10 +11573,11 @@ "dev": true }, "node_modules/path-to-regexp": { - "version": "0.1.10", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.10.tgz", - "integrity": "sha512-7lf7qcQidTku0Gu3YDPc8DJ1q7OOucfa/BSsIwjuh56VU7katFvuM8hULfkwB3Fns/rsVF7PwPKVw1sl5KQS9w==", - "dev": true + "version": "0.1.12", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.12.tgz", + "integrity": "sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==", + "dev": true, + "license": "MIT" }, "node_modules/path-type": { "version": "4.0.0", diff --git a/mcc/package.json b/mcc/package.json index 7853ca022..bef057dc1 100644 --- a/mcc/package.json +++ b/mcc/package.json @@ -23,7 +23,8 @@ "react-dom": "^17.0.2", "react-tooltip": "^5.28.0", "tsv": "^0.2.0", - "uuid": "^10.0.0" + "uuid": "^10.0.0", + "google-palette": "^1.1.1" }, "devDependencies": { "@labkey/build": "^7.7.1", diff --git a/mcc/src/client/GeneticsPlot/GeneticsPlot.tsx b/mcc/src/client/GeneticsPlot/GeneticsPlot.tsx index 083627a1e..031af83d2 100644 --- a/mcc/src/client/GeneticsPlot/GeneticsPlot.tsx +++ b/mcc/src/client/GeneticsPlot/GeneticsPlot.tsx @@ -108,10 +108,34 @@ export function GeneticsPlot() { <>
- Population structure analysis using PCA is a helpful way to summarize the genetic relationships among animals in the MCC. The PCA results can be thought of as a simple type of genetic clustering - animals with more similar principal component loadings are more genetically similar. A more precise description of the relationship between two animals is provided by kinship coefficients – these are quantitative measures of relatedness that can be calculated by comparing two genomes, and interpreted using genealogical language, such as ‘parent-child’, ‘uncle-nephew’, ‘first cousins’, etc. -
-
- Whole genome sequencing was performed on each animal and genotypes were called with GATK haplotype caller. Principal components analysis was performed with GCTA (https://yanglab.westlake.edu.cn/software/gcta/#PCA) and kinship coefficients were calculated with KING (https://www.kingrelatedness.com/). Analyses were performed by Ric del Rosario (Broad Institute). + Over the past few years, the MCC team has been working on extracting, sequencing and analyzing DNA from + marmosets across the participating breeding centers. While we have deposited the raw sequence data for + 578 marmosets on NCBI's Sequence Read Archive (SRA), we are excited to report that the MCC portal now + houses a call set with single nucleotide variants and short indels for over 800 individuals. +

+ The MCC genomic database is extensive, with each individual being genotype at millions of variants + across the genome. One way to summarize a large dataset can be done using Principal Component Analysis + (PCA). PCA is a technique used across disciplines (from astronomy to genomics) that reduces the + information in a multi-dimensional dataset to (fewer) principal components (PC) that retain overall + trends and patterns in the original data. Biologically, this could mean merging together two variants + that are always inherited together into just one PC, making the data easier to analyze while maintaining + its most important patterns. See the **Visualization with PCA** tab below. +

+ Although PCA is useful for broad-scale comparisons, it is not very useful when trying to distinguish + whether two individuals are siblings or first-cousins, for instance. For that, we have better statistics + that can describe the genetic relatedness between two individuals. We estimated genetic relatedness for + all pairs of individuals for which we have whole-genome data, and made these available under the + **Kinship** tab. There you will find the inferred relationships between pairs of individuals as well as + the calculated kinship coefficient, which is a quantitative measure of genetic relatedness + (see here for more details). +

+ It is possible to explore the full MCC database of variants with a graphical interface by accessing the + **Genome Browser** tab. There you can, for example, visualize all the variants present in your gene of + interest by typing it's name in the search bar. +

+ The genetic analyses described here were performed by Karina Ray (ONPRC), Murillo Rodrigues (ONPRC), and + Ric del Rosario (Broad Institute). Please contact us at mcc@ohsu.edu with any + questions.

diff --git a/mcc/src/client/GeneticsPlot/ScatterChart.tsx b/mcc/src/client/GeneticsPlot/ScatterChart.tsx index 080c89674..3ffd23e34 100644 --- a/mcc/src/client/GeneticsPlot/ScatterChart.tsx +++ b/mcc/src/client/GeneticsPlot/ScatterChart.tsx @@ -5,23 +5,14 @@ import { LineElement, Tooltip, Legend -} from 'chart.js'; +} from 'chart.js' +import palette from 'google-palette' import { Scatter } from 'react-chartjs-2'; import React, { useEffect, useRef, useState } from 'react'; ChartJS.register(LinearScale, PointElement, LineElement, Tooltip, Legend); -const CHART_COLORS = [ - 'rgb(255, 99, 132)', //red - 'rgb(255, 159, 64)', //orange - 'rgb(255, 205, 86)', //yellow - 'rgb(75, 192, 192)', //green - 'rgb(54, 162, 235)', //blue - 'rgb(153, 102, 255)', //purple - 'rgb(201, 203, 207)' //grey -] - export default function ScatterChart(props: {data: any}) { const { data } = props; @@ -39,11 +30,13 @@ export default function ScatterChart(props: {data: any}) { }); const dataByColony = [] - const uniqueColonies = [...new Set(collectedData.map(x => x.colony))] + const uniqueColonies = [...new Set(collectedData.map(x => String(x.colony)))] + const colors = palette(['Set1', 'qualitative'], uniqueColonies.length); + uniqueColonies.forEach((colonyName : string, idx) => { dataByColony.push({ label: colonyName, - backgroundColor: CHART_COLORS[idx], + backgroundColor: '#' + colors[idx], data: collectedData.filter(x => x.colony == colonyName) }) }) diff --git a/primeseq/resources/external/comb-p/Dockerfile b/primeseq/resources/external/comb-p/Dockerfile deleted file mode 100644 index 47efb6ca6..000000000 --- a/primeseq/resources/external/comb-p/Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -from continuumio/anaconda - -RUN apt-get update --fix-missing && apt-get install -y unzip && apt-get clean - -# install comb-p -RUN mkdir -p /comb-p -RUN cd /comb-p -RUN wget https://github.com/brentp/combined-pvalues/archive/v0.32.zip -RUN unzip v0.32.zip -RUN cd /combined-pvalues-0.32 && python setup.py install diff --git a/primeseq/resources/external/comb-p/combpWrapper.sh b/primeseq/resources/external/comb-p/combpWrapper.sh deleted file mode 100644 index cfce67e4e..000000000 --- a/primeseq/resources/external/comb-p/combpWrapper.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -set -e -set -u -set -x - -DATA_DIR=$1 -INPUT_FILE=$2 -OUT_DIR=$3 -OUT_PREFIX=$4 -DIST=$5 -STEP=$6 -SEED=$7 -DOCKER=/opt/acc/sbin/exadocker - -# NOTE: switch to using dockerhub for the build -# https://github.com/bbimber/combpdocker -#docker build -q -t combp $1 -#docker pull bbimber/combpdocker - -sudo $DOCKER run --rm=true -v "${DATA_DIR}:/data" -v "${OUT_DIR}:/outDir" bbimber/combpdocker comb-p pipeline -c 5 --dist $DIST --step $STEP --seed $SEED -p /outDir/${OUT_PREFIX} /data/${INPUT_FILE} diff --git a/primeseq/src/org/labkey/primeseq/PrimeseqModule.java b/primeseq/src/org/labkey/primeseq/PrimeseqModule.java index a50f4f915..af4095cf0 100644 --- a/primeseq/src/org/labkey/primeseq/PrimeseqModule.java +++ b/primeseq/src/org/labkey/primeseq/PrimeseqModule.java @@ -72,8 +72,6 @@ protected void init() @Override protected void doStartupAfterSpringConfig(ModuleContext moduleContext) { - SequencePipelineService.get().registerResourceSettings(new ExacloudResourceSettings()); - SystemMaintenance.addTask(new ClusterMaintenanceTask()); ClusterService.get().registerResourceAllocator(new BlastPipelineJobResourceAllocator.Factory()); @@ -109,6 +107,8 @@ public PipelineStartup() } else { + SequencePipelineService.get().registerResourceSettings(new ExacloudResourceSettings()); + SequencePipelineService.get().registerPipelineStep(new BismarkWrapper.Provider()); SequencePipelineService.get().registerPipelineStep(new BismarkWrapper.MethylationExtractorProvider()); diff --git a/primeseq/src/org/labkey/primeseq/analysis/MethylationRateComparisonHandler.java b/primeseq/src/org/labkey/primeseq/analysis/MethylationRateComparisonHandler.java index 633a08273..e7f1e0fd1 100644 --- a/primeseq/src/org/labkey/primeseq/analysis/MethylationRateComparisonHandler.java +++ b/primeseq/src/org/labkey/primeseq/analysis/MethylationRateComparisonHandler.java @@ -29,18 +29,19 @@ import org.labkey.api.security.User; import org.labkey.api.sequenceanalysis.SequenceOutputFile; import org.labkey.api.sequenceanalysis.pipeline.PipelineContext; +import org.labkey.api.sequenceanalysis.pipeline.PipelineOutputTracker; import org.labkey.api.sequenceanalysis.pipeline.SequenceAnalysisJobSupport; import org.labkey.api.sequenceanalysis.pipeline.SequenceOutputHandler; import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService; import org.labkey.api.sequenceanalysis.run.AbstractCommandWrapper; import org.labkey.api.sequenceanalysis.run.CommandWrapper; +import org.labkey.api.sequenceanalysis.run.DockerWrapper; import org.labkey.api.util.FileType; import org.labkey.api.util.FileUtil; import org.labkey.api.util.Pair; import org.labkey.api.view.ActionURL; import org.labkey.api.writer.PrintWriters; import org.labkey.primeseq.PrimeseqModule; -import org.labkey.primeseq.pipeline.CombpRunner; import java.io.BufferedReader; import java.io.File; @@ -487,8 +488,7 @@ public void processFilesRemote(List inputFiles, JobContext c Double seed = ctx.getParams().optDouble("seed", 0.05); Integer step = ctx.getParams().optInt("step", 100); - CombpRunner combp = new CombpRunner(ctx.getLogger()); - File outBed = combp.runCompP(finalOut, ctx.getOutputDir(), dist, seed, step); + File outBed = runCompP(finalOut, ctx.getOutputDir(), dist, seed, step, ctx, ctx.getFileManager()); SequenceOutputFile so2 = new SequenceOutputFile(); so2.setName("Comb-p: " + ctx.getJob().getDescription()); so2.setDescription("Comb-p: " + jobDescription); @@ -726,4 +726,36 @@ else if (i2.compareTo(il) > 0) } } } + + private File runCompP(File inputBed, File outputDir, int dist, double seed, int stepSize, PipelineContext ctx, PipelineOutputTracker tracker) throws PipelineJobException + { + // See: https://github.com/bbimber/combpdocker + DockerWrapper wrapper = new DockerWrapper("bbimber/combpdocker", ctx.getLogger(), ctx); + File outputPrefix = new File(outputDir, FileUtil.getBaseName(inputBed) + ".combp"); + + List args = new ArrayList<>(); + args.add("comb-p"); + args.add("pipeline"); + args.add("-c"); + args.add("5"); + args.add("--dist"); + args.add(String.valueOf(dist)); + args.add("--step"); + args.add(String.valueOf(stepSize)); + args.add("--seed"); + args.add(String.valueOf(seed)); + args.add("--p"); + args.add(outputPrefix.getPath()); + args.add(inputBed.getPath()); + + wrapper.executeWithDocker(args, ctx.getWorkingDirectory(), tracker, Arrays.asList(inputBed)); + + File outputBed = new File(outputPrefix.getPath() + ".regions.bed"); + if (!outputBed.exists()) + { + throw new PipelineJobException("Unable to find expected output: " + outputBed.getPath()); + } + + return outputBed; + } } diff --git a/primeseq/src/org/labkey/primeseq/pipeline/CombpRunner.java b/primeseq/src/org/labkey/primeseq/pipeline/CombpRunner.java deleted file mode 100644 index 1deb4d0f0..000000000 --- a/primeseq/src/org/labkey/primeseq/pipeline/CombpRunner.java +++ /dev/null @@ -1,66 +0,0 @@ -package org.labkey.primeseq.pipeline; - -import org.apache.logging.log4j.Logger; -import org.labkey.api.module.Module; -import org.labkey.api.module.ModuleLoader; -import org.labkey.api.pipeline.PipelineJobException; -import org.labkey.api.resource.FileResource; -import org.labkey.api.resource.Resource; -import org.labkey.api.sequenceanalysis.run.AbstractCommandWrapper; -import org.labkey.api.util.FileUtil; -import org.labkey.primeseq.PrimeseqModule; - -import java.io.File; -import java.util.ArrayList; -import java.util.List; - -/** - * Created by bimber on 11/3/2016. - */ -public class CombpRunner extends AbstractCommandWrapper -{ - public CombpRunner(Logger log) - { - super(log); - } - - public File runCompP(File inputBed, File outputDir, int dist, double seed, int stepSize) throws PipelineJobException - { - List args = new ArrayList<>(); - File scriptFile = getScriptFile(); - args.add("bash"); - args.add(scriptFile.getPath()); - args.add(inputBed.getParentFile().getPath()); - args.add(inputBed.getName()); - File outputPrefix = new File(outputDir, FileUtil.getBaseName(inputBed) + ".combp"); - args.add(outputPrefix.getParentFile().getPath()); - args.add(outputPrefix.getName()); - args.add(String.valueOf(dist)); - args.add(String.valueOf(stepSize)); - args.add(String.valueOf(seed)); - - execute(args); - File outputBed = new File(outputPrefix.getPath() + ".regions.bed"); - if (!outputBed.exists()) - { - throw new PipelineJobException("Unable to find expected output: " + outputBed.getPath()); - } - - return outputBed; - } - - private File getScriptFile() throws PipelineJobException - { - String path = "/external/comb-p/combpWrapper.sh"; - Module module = ModuleLoader.getInstance().getModule(PrimeseqModule.NAME); - Resource script = module.getModuleResource(path); - if (script == null || !script.exists()) - throw new PipelineJobException("Unable to find file: " + path + " in module: " + PrimeseqModule.NAME); - - File f = ((FileResource) script).getFile(); - if (!f.exists()) - throw new PipelineJobException("Unable to find file: " + f.getPath()); - - return f; - } -} diff --git a/primeseq/src/org/labkey/primeseq/pipeline/ExacloudResourceSettings.java b/primeseq/src/org/labkey/primeseq/pipeline/ExacloudResourceSettings.java index e7eaf6c75..629724225 100644 --- a/primeseq/src/org/labkey/primeseq/pipeline/ExacloudResourceSettings.java +++ b/primeseq/src/org/labkey/primeseq/pipeline/ExacloudResourceSettings.java @@ -1,5 +1,6 @@ package org.labkey.primeseq.pipeline; +import org.jetbrains.annotations.Nullable; import org.json.JSONObject; import org.labkey.api.data.Container; import org.labkey.api.module.ModuleLoader; @@ -9,6 +10,7 @@ import org.labkey.api.sequenceanalysis.pipeline.ToolParameterDescriptor; import org.labkey.primeseq.PrimeseqModule; +import java.io.File; import java.util.Arrays; import java.util.Collection; import java.util.HashSet; @@ -55,15 +57,12 @@ public Collection getDockerVolumes(Container c) Set volumes = new HashSet<>(); volumes.add("/home/groups/prime-seq"); volumes.add("/home/exacloud/gscratch"); + volumes.add("/mnt/scratch"); PipeRoot pr = PipelineService.get().findPipelineRoot(c); if (pr != null && pr.getRootPath().exists()) { - if (pr.getRootPath().getPath().startsWith("/home/groups/")) - { - String folderName = pr.getRootPath().getPath().replaceAll("^/home/groups/", "").split("/")[0]; - volumes.add("/home/groups/" + folderName); - } + volumes.add(convertHomeGroups(pr.getRootPath()).getPath()); } if (c.isWorkbook()) @@ -71,14 +70,34 @@ public Collection getDockerVolumes(Container c) PipeRoot pr2 = PipelineService.get().findPipelineRoot(c.getParent()); if (pr2 != null && pr2.getRootPath().exists()) { - if (pr2.getRootPath().getPath().startsWith("/home/groups/")) - { - String folderName = pr2.getRootPath().getPath().replaceAll("^/home/groups/", "").split("/")[0]; - volumes.add("/home/groups/" + folderName); - } + volumes.add(convertHomeGroups(pr2.getRootPath()).getPath()); } } return volumes; } + + private File convertHomeGroups(File input) + { + input = input.isDirectory() ? input : input.getParentFile(); + if (input.getPath().startsWith("/home/groups/")) + { + String folderName = input.getPath().replaceAll("^/home/groups/", "").split("/")[0]; + return new File("/home/groups/", folderName); + } + + return input; + } + + @Override + public @Nullable File inferDockerVolume(File input) + { + input = input.isDirectory() ? input : input.getParentFile(); + if (input.getPath().startsWith("/home/exacloud/gscratch")) + { + return new File("/home/exacloud/gscratch"); + } + + return convertHomeGroups(input); + } } diff --git a/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java b/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java index 382fe9551..be4ca8fcf 100644 --- a/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java +++ b/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java @@ -595,7 +595,7 @@ private void possiblyAddQOS(PipelineJob job, RemoteExecutionEngine engine, List< private String getPartition(PipelineJob job) { - return needsGPUs(job) ? "gpu" : "exacloud"; + return needsGPUs(job) ? "gpu" : "batch"; } private Long getFileSize(PipelineJob job)