#!/bin/bash # ============================================================================================================================================================= # == SLURM parameters == # ============================================================================================================================================================= # Decide how much computer power you need # The programs used in this script can only be executed on 1 NODE # You can use a max of 24 task per node. But the lower the number of tasks requested the shorter the queue when submitting a job #SBATCH --nodes=1 #SBATCH --ntasks-per-node=4 #SBATCH --mem=8GB #SBATCH --time=24:00:00 #SBATCH --job-name=fastp #SBATCH --output=slurm-fastp-%j.log # ============================================================================================================================================================= # == Load modules == # ============================================================================================================================================================= adaptors=/scratch/p127804/PROJECTS/Anne/2025_EndSeq/adaptors_anne.fna fastq_dir=/scratch/hb-molgen/NGSdata/20250702_024/Anne trimmed_dir=/scratch/p127804/PROJECTS/Douwe/LisaFries_2025_024/fq_trimmed mkdir -p $trimmed_dir function my_fastp_SE { fq_trimmed=$trimmed_dir/$1.trimmed_tail.fastq.gz /userapps/hb-molgen/fastp/fastp -i $fastq_dir/$2 -o $fq_trimmed --trim_tail1 20 --html $trimmed_dir/$1.html --json $trimmed_dir/$1.json } my_fastp_SE AJ024_001 AJ024_001_S19_L001_R1_001.fastq.gz my_fastp_SE AJ024_002 AJ024_002_S20_L001_R1_001.fastq.gz my_fastp_SE AJ024_003 AJ024_003_S21_L001_R1_001.fastq.gz my_fastp_SE AJ024_004 AJ024_004_S22_L001_R1_001.fastq.gz my_fastp_SE AJ024_005 AJ024_005_S23_L001_R1_001.fastq.gz my_fastp_SE AJ024_006 AJ024_006_S24_L001_R1_001.fastq.gz my_fastp_SE AJ024_007 AJ024_007_S25_L001_R1_001.fastq.gz my_fastp_SE AJ024_008 AJ024_008_S26_L001_R1_001.fastq.gz my_fastp_SE AJ024_009 AJ024_009_S27_L001_R1_001.fastq.gz my_fastp_SE AJ024_010 AJ024_010_S28_L001_R1_001.fastq.gz my_fastp_SE AJ024_011 AJ024_011_S29_L001_R1_001.fastq.gz my_fastp_SE AJ024_012 AJ024_012_S30_L001_R1_001.fastq.gz my_fastp_SE AJ024_013 AJ024_013_S31_L001_R1_001.fastq.gz my_fastp_SE AJ024_014 AJ024_014_S32_L001_R1_001.fastq.gz my_fastp_SE AJ024_015 AJ024_015_S33_L001_R1_001.fastq.gz my_fastp_SE AJ024_016 AJ024_016_S34_L001_R1_001.fastq.gz my_fastp_SE CT024_001 CT024_001_S35_L001_R1_001.fastq.gz my_fastp_SE CT024_002 CT024_002_S36_L001_R1_001.fastq.gz my_fastp_SE CT024_003 CT024_003_S37_L001_R1_001.fastq.gz my_fastp_SE CT024_004 CT024_004_S38_L001_R1_001.fastq.gz my_fastp_SE DS024_001 DS024_001_S1_L001_R1_001.fastq.gz my_fastp_SE DS024_002 DS024_002_S2_L001_R1_001.fastq.gz my_fastp_SE DS024_003 DS024_003_S3_L001_R1_001.fastq.gz my_fastp_SE DS024_004 DS024_004_S4_L001_R1_001.fastq.gz my_fastp_SE DS024_005 DS024_005_S5_L001_R1_001.fastq.gz my_fastp_SE DS024_006 DS024_006_S6_L001_R1_001.fastq.gz my_fastp_SE DS024_007 DS024_007_S7_L001_R1_001.fastq.gz my_fastp_SE DS024_008 DS024_008_S8_L001_R1_001.fastq.gz my_fastp_SE DS024_009 DS024_009_S9_L001_R1_001.fastq.gz my_fastp_SE DS024_010 DS024_010_S10_L001_R1_001.fastq.gz my_fastp_SE DS024_011 DS024_011_S11_L001_R1_001.fastq.gz my_fastp_SE DS024_012 DS024_012_S12_L001_R1_001.fastq.gz my_fastp_SE DS024_013 DS024_013_S13_L001_R1_001.fastq.gz my_fastp_SE DS024_014 DS024_014_S14_L001_R1_001.fastq.gz my_fastp_SE DS024_015 DS024_015_S15_L001_R1_001.fastq.gz my_fastp_SE DS024_016 DS024_016_S16_L001_R1_001.fastq.gz my_fastp_SE DS024_017 DS024_017_S17_L001_R1_001.fastq.gz my_fastp_SE DS024_018 DS024_018_S18_L001_R1_001.fastq.gz # Number of reads with polyT #fq=/scratch/p127804/PROJECTS/Anne/2025_EndSeq/fastq/AJ023_01_S365_L001_R1_001.fastq.gz #zcat $fq | awk 'NR % 4 == 2 && /TTTTTTTTTT/' | wc -l # # usage: /userapps/hb-molgen/fastp/fastp [options] ... # options: # -i, --in1 read1 input file name (string [=]) # -o, --out1 read1 output file name (string [=]) # -I, --in2 read2 input file name (string [=]) # -O, --out2 read2 output file name (string [=]) # --unpaired1 for PE input, if read1 passed QC but read2 not, it will be written to unpaired1. Default is to discard it. (string [=]) # --unpaired2 for PE input, if read2 passed QC but read1 not, it will be written to unpaired2. If --unpaired2 is same as --unpaired1 (default mode), both unpaired reads will be written to this same file. (string [=]) # --overlapped_out for each read pair, output the overlapped region if it has no any mismatched base. (string [=]) # --failed_out specify the file to store reads that cannot pass the filters. (string [=]) # -m, --merge for paired-end input, merge each pair of reads into a single read if they are overlapped. The merged reads will be written to the file given by --merged_out, the unmerged reads will be written to the files specified by --out1 and --out2. The merging mode is disabled by default. # --merged_out in the merging mode, specify the file name to store merged output, or specify --stdout to stream the merged output (string [=]) # --include_unmerged in the merging mode, write the unmerged or unpaired reads to the file specified by --merge. Disabled by default. # -6, --phred64 indicate the input is using phred64 scoring (it'll be converted to phred33, so the output will still be phred33) # -z, --compression compression level for gzip output (1 ~ 9). 1 is fastest, 9 is smallest, default is 4. (int [=4]) # --stdin input from STDIN. If the STDIN is interleaved paired-end FASTQ, please also add --interleaved_in. # --stdout stream passing-filters reads to STDOUT. This option will result in interleaved FASTQ output for paired-end output. Disabled by default. # --interleaved_in indicate that is an interleaved FASTQ which contains both read1 and read2. Disabled by default. # --reads_to_process specify how many reads/pairs to be processed. Default 0 means process all reads. (int [=0]) # --dont_overwrite don't overwrite existing files. Overwritting is allowed by default. # --fix_mgi_id the MGI FASTQ ID format is not compatible with many BAM operation tools, enable this option to fix it. # -V, --verbose output verbose log information (i.e. when every 1M reads are processed). # -A, --disable_adapter_trimming adapter trimming is enabled by default. If this option is specified, adapter trimming is disabled # -a, --adapter_sequence the adapter for read1. For SE data, if not specified, the adapter will be auto-detected. For PE data, this is used if R1/R2 are found not overlapped. (string [=auto]) # --adapter_sequence_r2 the adapter for read2 (PE data only). This is used if R1/R2 are found not overlapped. If not specified, it will be the same as (string [=auto]) # --adapter_fasta specify a FASTA file to trim both read1 and read2 (if PE) by all the sequences in this FASTA file (string [=]) # --detect_adapter_for_pe enable adapter detection for PE data to get ultra-clean data. It takes more time to find just a little bit more adapters. # --allow_gap_overlap_trimming allow up to one gap when trim adapters by overlap analysis for PE data. By default no gap is allowed. # -f, --trim_front1 trimming how many bases in front for read1, default is 0 (int [=0]) # -t, --trim_tail1 trimming how many bases in tail for read1, default is 0 (int [=0]) # -b, --max_len1 if read1 is longer than max_len1, then trim read1 at its tail to make it as long as max_len1. Default 0 means no limitation (int [=0]) # -F, --trim_front2 trimming how many bases in front for read2. If it's not specified, it will follow read1's settings (int [=0]) # -T, --trim_tail2 trimming how many bases in tail for read2. If it's not specified, it will follow read1's settings (int [=0]) # -B, --max_len2 if read2 is longer than max_len2, then trim read2 at its tail to make it as long as max_len2. Default 0 means no limitation. If it's not specified, it will follow read1's settings (int [=0]) # -D, --dedup enable deduplication to drop the duplicated reads/pairs # --dup_calc_accuracy accuracy level to calculate duplication (1~6), higher level uses more memory (1G, 2G, 4G, 8G, 16G, 24G). Default 1 for no-dedup mode, and 3 for dedup mode. (int [=0]) # --dont_eval_duplication don't evaluate duplication rate to save time and use less memory. # -g, --trim_poly_g force polyG tail trimming, by default trimming is automatically enabled for Illumina NextSeq/NovaSeq data # --poly_g_min_len the minimum length to detect polyG in the read tail. 10 by default. (int [=10]) # -G, --disable_trim_poly_g disable polyG tail trimming, by default trimming is automatically enabled for Illumina NextSeq/NovaSeq data # -x, --trim_poly_x enable polyX trimming in 3' ends. # --poly_x_min_len the minimum length to detect polyX in the read tail. 10 by default. (int [=10]) # -5, --cut_front move a sliding window from front (5') to tail, drop the bases in the window if its mean quality < threshold, stop otherwise. # -3, --cut_tail move a sliding window from tail (3') to front, drop the bases in the window if its mean quality < threshold, stop otherwise. # -r, --cut_right move a sliding window from front to tail, if meet one window with mean quality < threshold, drop the bases in the window and the right part, and then stop. # -W, --cut_window_size the window size option shared by cut_front, cut_tail or cut_sliding. Range: 1~1000, default: 4 (int [=4]) # -M, --cut_mean_quality the mean quality requirement option shared by cut_front, cut_tail or cut_sliding. Range: 1~36 default: 20 (Q20) (int [=20]) # --cut_front_window_size the window size option of cut_front, default to cut_window_size if not specified (int [=4]) # --cut_front_mean_quality the mean quality requirement option for cut_front, default to cut_mean_quality if not specified (int [=20]) # --cut_tail_window_size the window size option of cut_tail, default to cut_window_size if not specified (int [=4]) # --cut_tail_mean_quality the mean quality requirement option for cut_tail, default to cut_mean_quality if not specified (int [=20]) # --cut_right_window_size the window size option of cut_right, default to cut_window_size if not specified (int [=4]) # --cut_right_mean_quality the mean quality requirement option for cut_right, default to cut_mean_quality if not specified (int [=20]) # -Q, --disable_quality_filtering quality filtering is enabled by default. If this option is specified, quality filtering is disabled # -q, --qualified_quality_phred the quality value that a base is qualified. Default 15 means phred quality >=Q15 is qualified. (int [=15]) # -u, --unqualified_percent_limit how many percents of bases are allowed to be unqualified (0~100). Default 40 means 40% (int [=40]) # -n, --n_base_limit if one read's number of N base is >n_base_limit, then this read/pair is discarded. Default is 5 (int [=5]) # -e, --average_qual if one read's average quality score =1000), a sequential number prefix will be added to output name ( 0001.out.fq, 0002.out.fq...), disabled by default (long [=0]) # -d, --split_prefix_digits the digits for the sequential number padding (1~10), default is 4, so the filename will be padded as 0001.xxx, 0 to disable padding (int [=4]) # --cut_by_quality5 DEPRECATED, use --cut_front instead. # --cut_by_quality3 DEPRECATED, use --cut_tail instead. # --cut_by_quality_aggressive DEPRECATED, use --cut_right instead. # --discard_unmerged DEPRECATED, no effect now, see the introduction for merging. # -?, --help print this message #