This project has received funding from the European Union s Horizon 2020 research and innovation programme under grant agreement No

Transcription

1 This project has received funding from the European Union s Horizon 2020 research and innovation programme under grant agreement No

2

3 This project has received funding from the European Union s Horizon 2020 research and innovation programme under grant agreement No

4 This project has received funding from the European Union s Horizon 2020 research and innovation programme under grant agreement No

5

6

7

8 Configuration file Input File List Output File List Pipeline Output file metadata {tool_params} Input file metadata {meta1}, {meta2} {meta1}, {meta2}, {meta3} Output metadata file location

9

10 Configuration file Input File List Output File List {tool_params} Input file metadata {meta1}, {meta2}, {meta3} Pipeline Tool 1 Input Files Output Files Tool 2 Input Files Output Files Output file metadata {meta1}, {meta2} Output metadata file location

11

12 1. 2.

13 Configuration file Input File List Output File List {tool_params} Input file metadata {meta1}, {meta2}, {meta3} Pipeline Tool 1 Input Files Output Files Tool 2 Input Files Output Files Output file metadata {meta1}, {meta2} Output metadata file location

14

15 [ ] { } "compressed": 0, "_id": "unique_file_id", "data_type": "text", "file_type": "bam", "file_path": "/home/training/code/macs2.human.drr _aln_filtered.bam", "sources": [], "taxon_id": 9606, "meta_data": { "visible": true, "validated": 1, "assembly": "hg38" }

16 { } input_files": [ {...},... ], "arguments": [ {...},... ], "output_files": [ {...},... ]

17 { } input_files": [ { "required": true, "allow_multiple": false, "name": "bam", "value": "unique_file_id" } ],...,...,

18 { }..., "arguments": [ { "name": "execution", "value": "/home/training/code/run00" }, { "name": "project", "value": "my_project_id" }, { "name": "macs_nomodel_param", "value": true } ],...

19 { }...,..., "output_files": [ { "required": false, "allow_multiple": false, "name": "narrow_peak", "file": { "file_type": "BED", "meta_data": { "visible": true, "tool": "process_chipseq", "description": "Output" }, "data_type": "data_chip_seq", "compressed": "null" } },... ]

20 output_files = { "narrow_peak": "/absolute/path/for/working_dir", or "narrow_peak": None, }

21

22 This project has received funding from the European Union s Horizon 2020 research and innovation programme under grant agreement No

23 Step 1 Step 2

24

25 1. 2. git clone cd mg-process-macs2 3. git checkout template 4. git mv mg_process_test mg_process_macs2 git commit -am Initial change to the repo name

26

27 1. 2. git clone --depth 1 -b master rm -rf mg-process-test/.git mv mg-process-test my-pipeline cd my-pipeline mv mg_process_test my_pipeline

28 git init git add. git commit -m 'Initial commit' git remote add origin git remote -v git push origin master

29

30 Multiscale Genomics (MuG) Copyright EMBL-European Bioinformatics Institute This product includes software developed at: - EMBL-European Bioinformatics Institute Multiscale Genomics (MuG) Copyright EMBL-European Bioinformatics Institute Copyright University of Nottingham This product includes software developed at: - EMBL-European Bioinformatics Institute - University of Nottingham

31

32 pyenv virtualenv mg-process-macs2 pyenv activate mg-process-macs2 pip install -e. pip install -r requirements.txt

33 This project has received funding from the European Union s Horizon 2020 research and innovation programme under grant agreement No

34 Step 1 Step 2

35

36 """.. See the NOTICE file distributed with this work for additional information regarding copyright ownership. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """

37 if name == " main ": # Set up the command line parameters PARSER = argparse.argumentparser(description="index the genome file") PARSER.add_argument("--config", help="configuration file") PARSER.add_argument("--in_metadata", help="location of input metadata file") PARSER.add_argument("--out_metadata", help="location of output metadata file") PARSER.add_argument("--local", action="store_const", const=true, default=false) # Get the matching parameters from the command line ARGS = PARSER.parse_args() CONFIG = ARGS.config IN_METADATA = ARGS.in_metadata OUT_METADATA = ARGS.out_metadata LOCAL = ARGS.local if LOCAL: import sys sys._run_from_cmdl = True # pylint: disable=protected-access RESULTS = main_json(config, IN_METADATA, OUT_METADATA)

38 def main_json(config, in_metadata, out_metadata): """ Main function This function launches the app using configuration written in two json files: config.json and input_metadata.json. """ # 1. Instantiate and launch the App logger.info("1. Instantiate and launch the App") from apps.jsonapp import JSONApp app = JSONApp() result = app.launch(process_macs2, config, in_metadata, out_metadata) # 2. The App has finished logger.info("2. Execution finished; see " + out_metadata) return result config in_metadata out_metadata config_test.json file generated by the VRE input_test.json file generated by the VRE describing each of the files Location of the output metadata file describing the output files

39

40 self.configuration = { "execution": "/absolute/path/for/working_dir", "project", "my_project_id", "description": None, "macs_nomodel_param": True }

41 from future import print_function import argparse from basic_modules.workflow import Workflow from utils import logger from mg_process_macs2.tool.macs2 import Macs2 # class process_macs2(workflow): # pylint: disable=invalid-name,too-few-public-methods """ Functions for demonstrating the pipeline set up. """ configuration = {} def init (self, configuration=none): """ Initialise the tool with its configuration. """ logger.info("processing Test")... def run(self, input_files, metadata, output_files): """ Main run function for processing a test file. """ # Initialise the test tool macs2_handle = Macs2(self.configuration) macs2_files, macs2_meta = macs2_handle.run(input_files, metadata, output_files) return (macs2_files, macs2_meta)

42 configuration = {} def init (self, configuration=none): """ Initialise the tool with its configuration. Parameters configuration : dict a dictionary containing parameters that define how the operation should be carried out, which are specific to each Tool. """ logger.info("processing Tool") if configuration is None: configuration = {} self.configuration.update(configuration)

43

44

45 This project has received funding from the European Union s Horizon 2020 research and innovation programme under grant agreement No

46

47 from future import print_function from utils import logger from basic_modules.metadata import Metadata from basic_modules.tool import Tool # class Macs2(Tool): """ Tool for peak calling for ChIP-seq data """ def init (self, configuration=none): """ Init function """... def macs2_peak_calling(self, opt1, opt2): """ Tool Description """... def run(self, input_files, input_metadata, output_files): """ Main run() function description """... return (output_files_created, output_metadata) #

48 Used to populate the self.configuration variable with the key-value pairs of parameters defined in the config.json file. def init (self, configuration=none): """ Init function """ logger.info("macs2 Peak Caller") Tool. init (self) if configuration is None: configuration = {} self.configuration.update(configuration) To initialise a tool from a pipeline it can be done by: from mg_process_macs2.tool import macs2... macs2_handle = macs2(self.configuration) macs2.run( {key-value pairs of locations for input files} {key-value pairs of metadata for files} {key-value pairs of locations for output files} )

49

50 def run(self, input_files, input_metadata, output_files): """ Main run() function description """ with open(input_files[ test_input ], r ) as f_in: x = int(f_in.readline()) with open( output_file.txt, w ): f_out.write(str(x+1)) output_metadata = { test_output : Metadata( output_files[ test_output ], data_type="data_chip_seq", file_type="bed", file_path=os.path.join(self.configuration[ execution ], output_file.txt ), sources=sources, taxon_id=input_metadata[ test_input ].taxon_id, meta_data={ "assembly": input_metadata[ test_input ].meta_data["assembly"], "tool": "macs2", "bed_type": output_bed_types[result_file], "parameters": command_params } ) } return (output_files, output_metadata)

51

52 This project has received funding from the European Union s Horizon 2020 research and innovation programme under grant agreement No

53

54 >>> from utils import logger >>> logger.info( Hello World! ) :56:10 INFO: Hello World! >>> logger.progress("macs2", task_id=0, total=1) :08:43 PROGRESS: MACS (0/1) >>> logger.progress("macs2", task_id=1, total=1) :08:43 PROGRESS: MACS (1/1) >>> logger.progress("macs", status="running") :09:14 PROGRESS: MACS - RUNNING >>> logger.progress("macs", status="done") :09:14 PROGRESS: MACS - DONE

55 This project has received funding from the European Union s Horizon 2020 research and innovation programme under grant agreement No

56 Step 1 Step 2

57 if name == " main ": # Set up the command line parameters PARSER = argparse.argumentparser(description="index the genome file") PARSER.add_argument("--config", help="configuration file") PARSER.add_argument("--in_metadata", help="location of input metadata file") PARSER.add_argument("--out_metadata", help="location of output metadata file") PARSER.add_argument("--local", action="store_const", const=true, default=false) # Get the matching parameters from the command line ARGS = PARSER.parse_args() CONFIG = ARGS.config IN_METADATA = ARGS.in_metadata OUT_METADATA = ARGS.out_metadata LOCAL = ARGS.local if LOCAL: import sys sys._run_from_cmdl = True # pylint: disable=protected-access RESULTS = main_json(config, IN_METADATA, OUT_METADATA)

58 1. 2. { } input_files": [ {"name": "input1", "value": "unique_file_id1"}, {...},... ], "arguments": [ {"name": "arg_name", "value": "foo"}, {...},... ], "output_files": [ {meta}, {...}... ] [ ] { }, { }, {...} "_id": "unique_file_id1", {meta} "_id": "unique_file_id2", {meta}

59 1. 2.

60 1. 2.

61 1. 2. Define - inputs (format, data type...) - arguments (type,...) - output files (format, data type...) VALIDATION Define... - Local test execution directory - Local test input data - Metadata for test input data tar -xzvf test.tar.gz cd test/ sh test_operation.sh

62 1. 2. { input_files : [ ], input_files_combinations : [ ], arguments: [ ], output_files[ ] } VALIDATION tar -xzvf test.tar.gz cd test/ sh test_operation.sh

63 #!/bin/bash ### ### Testing in a local installation the VRE server CMD ### * Automatically created by MuGVRE * ### # Local installation - EDIT IF REQUIREDD WORKING_DIR=/home/pmes/tests/run000 TOOL_EXECUTABLE=/main/mg-tool/executable.py TOOL_LIBRARY=/home/pmes/.pyenv/libs/python/site-packages # Test input files CWD="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" TEST_DATA_DIR=$CWD/json/0_CallpeaksonChIPSeqreads # Running macs2 tool... if -d [ $WORKING_DIR ]; then rm -r $WORKING_DIR/; else mkdir -p $WORKING_DIR; fi cd $WORKING_DIR $TOOL_EXECUTABLE --config $TEST_DATA_DIR/config.json --in_metadata $TEST_DATA_DIR/in_metadata.json --out_metadata $WORKING_DIR/out_metadata.json > $WORKING_DIR/tool.log runcompss -d --summary --base_log_dir=$working_dir --lang=python --pythonpath=$tool_library $TOOL_EXECUTABLE --config $TEST_DATA_DIR/config.json --in_metadata $TEST_DATA_DIR/in_metadata.json --out_metadata $WORKING_DIR/out_metadata.json > $WORKING_DIR/tool.log

64 1. 2. a. b. i. ii. c. i. ii. i. ii.

65 Copy/Paste the JSON defining I/O to save time. but make sure to set _id according to your tool

66 a. b.

67 This project has received funding from the European Union s Horizon 2020 research and innovation programme under grant agreement No

68 pip install macs2

69 1. process_test.py process_macs2.py run() 4. python process_macs2.py --config config.json --in_metadata input.json --out_metadata output.json cd ~/code/macs2_vre_json sh test_single_peakcalling.sh

70 ~/code/drr _filtered.bam macs2 callpeak --nomodel -t ~/code/drr _filtered.bam -n name --outdir.

71 1. mg_process_macs2/tool/testtool.py mg_process_macs2/tool/macs2.py test_writer() a. name b. bam_file c. name + _peaks.narrowpeak d. name + _summits.bed e. name + _peaks.broadpeak f. name + _peaks.gappedpeak 4. 5.

72 python process_macs2.py --config config.json --in_metadata input.json --out_metadata output.json cd ~/code/macs2_vre_json sh test_single_peakcalling.sh

73 # Save your work to the repo: git add process_macs2.py mg_process_macs2/tool/macs2.py git commit -m Created macs2 tool and pipeline # Answer: git checkout basic

74

75 1. run() macs2_nomodel_param self.configuration True --nomodel macs2() 2. macs2() 3. self.configuration

76 cd ~/code/macs2_vre_json sh test_single_peakcalling.sh

77 basic_arg # Save your work git commit -am Added arguments to the tool # Answers git checkout basic_arg

78