1Pipeline.sh

[TOC]

# 1EasyMetagenome Pipeline (1易宏基因组流程)

    # Authors(作者): Yong-Xin Liu(刘永鑫), Defeng Bai(白德凤), Tong Chen(陈同) et al.
    # Version(版本): 1.24, 2025/11/22
    # Operation System(操作系统): Linux Ubuntu 22.04+ / CentOS 7.7+ 
    # Homepage(主页): https://github.com/YongxinLiu/EasyMetagenome
    # Cititon(引文): Bai, et al. 2025. EasyMetagenome: A User‐Friendly and Flexible Pipeline for Shotgun Metagenomic Analysis in Microbiome Research. iMeta 4: e70001. https://doi.org/10.1002/imt2.70001

# 1. Data preprocessing(数据预处理)

## 1.1 Preparing(准备工作)

    # 1. First-time use, please refer to the `0Install.sh` to install the software and database (approximately 1-3 days, only once).
    # 2. Copy the EasyMetagenome workflow `1Pipeline.sh` to the project folder, e.g., in this case, `meta`.
    # 3. Prepare the sequencing data (seq/*.fq.gz) and sample metadata (result/metadata.txt) in the project folder.
    # 1.  首次使用请参照`0Install.sh`脚本，安装软件和数据库(大约1-3天，仅一次)
    # 2.  易宏基因组(EasyMetagenome)流程`1Pipeline.sh`复制到项目文件夹，如本次为meta
    # 3.  项目文件夹准备测序数据(seq/*.fq.gz)和样本元数据(result/metadata.txt)

    # **Software, database and work directory settings(数据库、软件和工作目录设置)**
    # **The follwoing paragraph must run before(分析前必须运行)**
    # Conda directory(软件安装目录), e.g. /anaconda3 , `conda env list` view software list,
    soft=~/miniconda3
    # database(db, 数据库位置), e.g. /db or ~/db
    db=~/db
    # work directory(wd, 工作目录)，e.g. meta
    wd=~/meta
    # Create and enter the working directory 创建并进入工作目录
    mkdir -p $wd && cd $wd
    # Create sequences, temporary files, and a results directory 创建序列，临时文件和结果目录
    mkdir -p seq temp result
    # Add software/scripts to environment variables 添加软件/脚本至环境变量
    PATH=$soft/bin:$soft/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$db/EasyMicrobiome/linux:$db/EasyMicrobiome/script
    echo $PATH

    # **Metadata and sequence files (元数据和序列文件) **

    # Metadata (元数据)
    # Edit metadata in Excel then save txt format in result, demo in result or download (编写元数据metadata.txt并保存至result目录，此处下载演示)
    wget -c http://www.imeta.science/github/EasyMetagenome/result/metadata.txt
    mv metadata.txt result/metadata.txt
    # Format check: ^I is tab, $ is linux new line, ^M$ is windows new line, ^M is Mac new line
    # 格式预览，^I为制表符，$为Linux换行，^M$为Windows回车，^M为Mac换行符
    cat -A result/metadata.txt
    # Format windows ^M$ to linux $, delete ^M in regulation expression is '\r'
    # 转换Windows回车为Linux换行，即删除^M (正则表达式为\r)
    sed -i 's/\r//' result/metadata.txt
    cat -A result/metadata.txt

    # Sequencing data (序列文件)
    # Using filezilla upload to seq directory (用户使用filezilla上传测序文件至seq目录)
    # This test uses theChina National Center for Bioinformation's GSA network for downloading data. Backup links are also provided via iMeta and BaiduNetDisk.
    # 本次测试国家生信息中心GSA网络下载，同时提供iMeta、百度网盘备用站点数据下载链接
    
    # Site 1. GSA download links and batch rename by metadata (站点1. 国家生信息中心下载链接，按实验设计编号批量下载并改名)
    # GSA: https://ngdc.cncb.ac.cn/gsa/ search CRA032890, find link test one example C3
    wget -c ftp://download.big.ac.cn/gsa5/CRA032890/1/CRR2274865/CRR2274865_r1.fq.gz -O seq/C3_1.fq.gz
    wget -c ftp://download.big.ac.cn/gsa5/CRA032890/1/CRR2274865/CRR2274865_r2.fq.gz -O seq/C3_2.fq.gz
    awk 'BEGIN{OFS=FS="\t"}{system("wget -c ftp://download.big.ac.cn/gsa5/"$8"/"$9"/"$9"_r1.fq.gz -O seq/"$1"_1.fq.gz")}' \
        <(tail -n+2 result/metadata.txt)
    awk 'BEGIN{OFS=FS="\t"}{system("wget -c ftp://download.big.ac.cn/gsa5/"$8"/"$9"/"$9"_r2.fq.gz -O seq/"$1"_2.fq.gz")}' \
        <(tail -n+2 result/metadata.txt)

    # Site 2. iMeta Science download link （站点2. iMeta下载链接）
    cd seq
    awk '{system("wget -c http://www.imeta.science/github/EasyMetagenome/seq/"$1"_1.fq.gz")}' <(tail -n+2 ../result/metadata.txt)
    awk '{system("wget -c http://www.imeta.science/github/EasyMetagenome/seq/"$1"_2.fq.gz")}' <(tail -n+2 ../result/metadata.txt)
    cd ..

    # Site 3. BaiduNetDisk (站点3. 百度网盘) in /db/meta/seq directory from https://pan.baidu.com/s/1Ikd_47HHODOqC3Rcx6eJ6Q?pwd=0315

    # ls show file info, -l (l: list) show detail, -sh s: size, h: human readable
    # ls查看文件大小等信息，-l 列出详细信息 (l: list)，-sh 显示人类可读方式文件大小 (s: size; h: human readable)
    ls -lsh seq/*.fq.gz
    # Statistic basic info of sequences 统计序列信息
    time seqkit stat seq/*.fq.gz > result/seqkit.txt
    cat result/seqkit.txt

    # **Sequence file format check (序列文件格式检查)**
    # Use zless/zcat to view compressible files and check the sequence quality format 
    # (quality values in uppercase are in standard Phred33 format, lowercase in Phred64 format);
    # check if the pair-end sequence IDs have duplicate names, and if so, rename them. 
    # Refer to **Appendix – Quality Control kneaddata: End-to-End Mismatch After Host Removal; Sequence Renaming**.
    # zless/zcat查看可压缩文件，检查序列质量格式(质量值大写字母为标准Phred33格式，小写字母为Phred64)；
    # 检查双端序列ID是否重名，如重名需要改名。参考**附录 —— 质控kneaddata，去宿主后双端不匹配；序列改名**。

    # Set a sample name be variable i, which can be reused multiple times, reducing the number of modifications required.
    # 设置某个样本名为变量i，后面可多次重用，减少修改次数
    i=C1
    # zless is used to view compressed files; spacebar to turn pages, q to exit; head specifies the number of lines to display.
    # zless查看压缩文件，空格翻页，q退出; head指定显示行数
    zless seq/${i}_1.fq.gz | head -n4

    # **Summary of Working Directory and File Structure**
    # **工作目录和文件结构总结**
    # ├── pipeline.sh
    # ├── result
    # │   └── metadata.txt
    # ├── seq
    # │   ├── C1_1.fq.gz
    # │   ├── ...
    # │   └── Y1_2.fq.gz
    # └── temp

    # * `1pipeline.sh` is the analysis workflow code;
    # * The `seq` directory contains 6 samples of short-reads paired-end 150 bp sequencing and 12 sequence files;
    # * `temp` is a temporary folder that stores intermediate analysis files. It can be deleted entirely after analysis to save space;
    # * `result` contains important node files and formatted analysis results figures. `metadata.txt` is also located here.
    # *   1pipeline.sh是分析流程代码；
    # *   seq目录中有2个样本Illumina双端测序，4个序列文件；
    # *   temp是临时文件夹，存储分析中间文件，结束可全部删除节约空间
    # *   result是重要节点文件和整理化的分析结果图表，实验设计metadata.txt也在此

## 1.2 Fastp Quality Control(质量控制)

    # Create a directory to record software versions and citations
    # 创建目录，记录软件版本和引文
    mkdir -p temp/qc result/qc
    fastp
    
    # Single sample quality control (单样本质控)
    i=`tail -n+2 result/metadata.txt|cut -f1|head -n1`
    fastp -i seq/${i}_1.fq.gz  -I seq/${i}_2.fq.gz \
      -o temp/qc/${i}_1.fastq -O temp/qc/${i}_2.fastq
    
    # Multiple samples are processed in parallel; this step requires 5 times the space of the original data.
    # 多样本并行，此步占用原始数据5x空间
    # -j 2: indicates processing 2 samples simultaneously; for 6 samples, j2, 2minmutes 
    # -j 2: 表示同时处理2个样本；6个样本j2, 2分钟(minutes, m)
    time tail -n+2 result/metadata.txt|cut -f1|rush -j 2 \
      "fastp -i seq/{1}_1.fq.gz -I seq/{1}_2.fq.gz \
        -j temp/qc/{1}_fastp.json -h temp/qc/{1}_fastp.html \
        -o temp/qc/{1}_1.fastq  -O temp/qc/{1}_2.fastq \
        > temp/qc/{1}.log 2>&1"
    
    # Summary of Quality Control Results (质控后结果汇总)
    echo -e "SampleID\tRaw\tClean" > temp/fastp
    for i in `tail -n+2 result/metadata.txt|cut -f1`;do
        echo -e -n "$i\t" >> temp/fastp
        grep 'total reads' temp/qc/${i}.log|uniq|cut -f2 -d ':'|tr '\n' '\t' >> temp/fastp
        echo "" >> temp/fastp
        done
    sed -i 's/ //g;s/\t$//' temp/fastp
    # Sort by metadata (按metadata排序)
    awk 'BEGIN{FS=OFS="\t"}NR==FNR{a[$1]=$0}NR>FNR{print a[$1]}' temp/fastp result/metadata.txt \
      > result/qc/fastp.txt
    cat result/qc/fastp.txt
    
## 1.3 KneadData Host removal (去宿主)

    # The kneaddata relies on bowtie2 to align with the host sequence, and then filters out non-host sequences for downstream analysis.
    # kneaddata是流程主要依赖bowtie2比对宿主，然后筛选非宿主序列用于下游分析。

    # Create directories, activate the environment, and record versions.
    # 创建目录、启动环境、记录版本
    mkdir -p temp/hr
    conda activate kneaddata
    kneaddata --version # v0.12.3

    # Single-sample host removal (单样本去宿主)
    i=`tail -n+2 result/metadata.txt|cut -f1 | head -n1`
    time kneaddata -i1 temp/qc/${i}_1.fastq -i2 temp/qc/${i}_2.fastq \
        -o temp/hr \
        --bypass-trim --bypass-trf --reorder \
        --bowtie2-options '--very-sensitive --dovetail' \
        -db ${db}/kneaddata/human/hg_39 \
        --remove-intermediate-output -v -t 3

    # Multi-sample host removal, this step occupies 5 times the space of the original data, 3m
    # 多样本去宿主,此步占用原始数据5x空间,5m
    time tail -n+3 result/metadata.txt | cut -f1 | rush -j 2 \
      "kneaddata \
        -i1 temp/qc/{1}_1.fastq -i2 temp/qc/{1}_2.fastq \
        -o temp/hr/ \
        --bypass-trim --bypass-trf --reorder \
        --bowtie2-options '--very-sensitive --dovetail' \
        -db ${db}/kneaddata/human/hg_39 \
        --remove-intermediate-output -v -t 3"

    # To check the size, * matches any number of characters, and ? matches any single character.
    # 查看大小，*匹配任意多个字符，?匹配任意一个字符
    ls -shtr temp/hr/*_paired_?.fastq

    # Simplified renaming (简化改名)
    # Ubuntu System renaming (Ubuntu系统改名)
    rename 's/_1_kneaddata_paired//' temp/hr/*.fastq
    # CentOS System renaming (CentOS系统改名)
    rename '_1_kneaddata_paired' '' temp/hr/*.fastq

    # Summary of Quality Control Results (质控结果汇总)
    kneaddata_read_count_table --input temp/hr \
      --output temp/kneaddata.txt
    # Filter key results column (筛选重点结果列)
    cut -f 1,2,5,6 temp/kneaddata.txt | sed 's/_1_kneaddata//' > result/qc/sum.txt
    # View table alignment (对齐方式查看表格)
    csvtk -t pretty result/qc/sum.txt

    # Verify if the IDs match in pair-end reads (校验ID是否配对)
    paste <(head -n40 temp/hr/`tail -n+2 result/metadata.txt|cut -f1|head -n1`_1.fastq|grep @)    <(head -n40 temp/hr/`tail -n+2 result/metadata.txt|cut -f1|head -n1`_2.fastq|grep @)

    # Large file cleanup: samples with high host content can save >90% of space.
    # 大文件清理，高宿主含量样本可节约>90%空间
    # Using the absolute path of a command ensures that it is a parameterless command. Administrators can use aliases to define commands with parameters, which can affect the operation results.
    # 使用命令的绝对路径确保使用无参数的命令，管理员用alias自定义命令含参数，影响操作结果
    /bin/rm -rf temp/hr/*contam* temp/hr/*unmatched* temp/hr/reformatted* temp/hr/_temp*
    ls -l temp/hr/
    # After confirming the host removal results, the intermediate files after quality control can be deleted
    # 确认去宿主结果后，可以删除质控后中间文件
    rm temp/qc/*.fastq
    # Human data should be published as dehosted files to reduce the risk of privacy leaks.
    # 人类数据建议发布去宿主后的文件，减少隐私泄露风险
    

# Read-based HUMAnN4/Kraken2 (二、基于读长分析)

## 2.1 HUMAnN4分析

    # HUMAnN 4: https://docs.google.com/document/d/1rCx5JkuO7wCKWrL8_-UJx_FkopJAfcDFtZktgPspak0/edit?pli=1&tab=t.0

### HUMAnN4 taxonomic and functional annotation (物种和功能分析)

    # Prepare input file (准备输入文件)
    # HUMAnN requires a file containing paired-end sequences to be merged as input. 
    # A for loop is used to merge paired-end sequences in batches according to the experimental design sample names. 
    # Note the asterisk (*) and question mark (?), which represent multiple and a single character, respectively. 
    # Of course, you must also pay attention to the fact that each line of code ends with a backslash (\\).
    # HUMAnN要求双端序列合并的文件作为输入，for循环根据实验设计样本名批量双端序列合并。
    # 注意星号(\*)和问号(?)，分别代表多个和单个字符。当然大家更不能溜号，行分割的代码行末有一个\\

    mkdir -p temp/concat
    # Merge pair-end files into a single file (双端合并为单个文件)
    for i in `tail -n+2 result/metadata.txt|cut -f1`;do 
      cat temp/hr/${i}_?.fastq \
      > temp/concat/${i}.fq; done &
    # Check sample quantity and size (查看样品数量和大小)
    ls -shl temp/concat/*.fq
    # The data is too large and the computation time is long. You can use head to truncate the single-end analysis to a 20M sequence, i.e., 3G, with 80M lines. See Appendix: HUANN2 Reduce Input File Speedup.
    # 数据太大，计算时间长，可用head对单端分析截取20M序列，即3G，行数为80M行，详见附录：HUMAnN2减少输入文件加速

    # HUMAnN4 analysis tutorial and test (教学和测试)
    # *   Taxonomic annotation call MetaPhlAn4 (物种组成调用)
    # *   Input(输入)：temp/concat/*.fq Merged sequences (合并的序列)
    # *   Output(输出)：temp/humann4/
    #     *   Y1_pathabundance.tsv
    #     *   Y1_pathcoverage.tsv
    #     *   Y1_genefamilies.tsv
    
    # Start the humann4 environment and check the database configuration (启动环境并检查数据库配置)
    conda activate humann4
    mkdir -p temp/humann4
    humann --version # v4.0.0.alpha.1
    humann_config

    # Single-sample test (单样本测试): Y1 657M, 8p, 32min
    i=`tail -n+2 result/metadata.txt|cut -f1 | head -n1`
    time humann --input temp/concat/${i}.fq --threads 8 \
      --metaphlan-options "--input_type fastq --bowtie2db ${db}/metaphlan4 --index mpa_vOct22_CHOCOPhlAnSGB_202403 --offline -t rel_ab_w_read_stats --nproc 8" \
      --output temp/humann4
      
    # Multi-sample parallel computing, test data with 6 samples in dual parallel: 2h, recommended 16p, 3h/6G;
    # 多样本并行计算，测试数据6个样本双并行：2h，推荐16p，3h/6G；
    # -n+3 start from second samples, --threads set 8/16/32 to accelerate
    time tail -n+3 result/metadata.txt | cut -f1 | rush -j 2 \
      "humann --input temp/concat/{1}.fq --threads 8 \
        --metaphlan-options '--input_type fastq --bowtie2db ${db}/metaphlan4 --index mpa_vOct22_CHOCOPhlAnSGB_202403 --offline -t rel_ab_w_read_stats --nproc 8'  \
        --output temp/humann4/ "

    # (Optional) Run MetaPhlAn4 separately (可选)单独运行MetaPhlAn4
    metaphlan -v # MetaPhlAn version 4.1.1 (11 Mar 2024)
    mkdir -p temp/metaphlan4
    i=`tail -n+2 result/metadata.txt|cut -f1 | head -n1`
    # taxonomy classification 8p, 4min
    time metaphlan --input_type fastq temp/hr/${i}_1.fastq \
      temp/metaphlan4/${i}.txt --nproc 8 --bowtie2db ${db}/metaphlan4 --index mpa_vOct22_CHOCOPhlAnSGB_202403 --offline

### Taxonomic composition table (物种组成表)

    # Output(输出)：
    #   result/metaphlan4/taxonomy.tsv
    #   result/metaphlan4/taxonomy.spf spf for STAMP（用于STAMP分析）
    
    # Sample Merge, correct name, preview (样品合并、修正ID、预览) | sed '2i #metaphlan4'
    mkdir -p result/metaphlan4
    merge_metaphlan_tables.py temp/humann4/*_metaphlan_profile.tsv | sed 's/_1_metaphlan//g' | tail -n+2 | sed '1 s/clade_name/ID/' | sed '2i #metaphlan4' \
      > result/metaphlan4/taxonomy.tsv
    csvtk -t stat result/metaphlan4/taxonomy.tsv
    head -n5 result/metaphlan4/taxonomy.tsv

    # Format to stamp spf(转换为STAMP输入spf格式)
    # duplicate rows redandency by sort & uniq 
    metaphlan_to_stamp.pl result/metaphlan4/taxonomy.tsv \
      |sort -r | uniq > result/metaphlan4/taxonomy.spf
    # stat and view, 14 columns, 274 rows
    csvtk -t stat result/metaphlan4/taxonomy.spf
    head result/metaphlan4/taxonomy.spf
    # STAMP not support unclassified, filtered left 203 row
    grep -v 'unclassified' result/metaphlan4/taxonomy.spf > result/metaphlan4/taxonomy2.spf
    csvtk -t stat result/metaphlan4/taxonomy2.spf
    head result/metaphlan4/taxonomy2.spf
    # Download metadata.txt & taxonomy2.spf, and open in STAMP software 结果文件下载可用STAMP分析

## 2.4 Functional composition analysis (功能组成分析)

    # 整合后的输出：
    # *   result/metaphlan4/taxonomy.tsv 物种丰度表
    # *   result/metaphlan4/taxonomy.spf 物种丰度表（用于stamp分析）
    # *   result/humann3/pathabundance_relab_unstratified.tsv 通路丰度表
    # *   result/humann3/pathabundance_relab_stratified.tsv 通路物种组成丰度表
    # *   stratified(每个菌对此功能通路组成的贡献)和unstratified(功能组成)

    # Samples merging table (样本合并为功能组成表)
    mkdir -p result/humann4
    humann_join_tables --input temp/humann4 \
      --file_name pathabundance \
      --output result/humann4/path.tsv
    # format sampleID, stat and view, 样本名统一、统计和预览
    sed -i 's/_Abundance//g' result/humann4/path.tsv
    csvtk -t stat result/humann4/path.tsv
    head -n5 result/humann4/path.tsv

    # Normolization to relative abundance relab(1) or per million cpm(1,000,000) (标准化为相对丰度relab或百万比cpm)
    humann_renorm_table \
      --input result/humann4/path.tsv \
      --units relab \
      --output result/humann4/path_relab.tsv
    head -n5 result/humann4/path_relab.tsv

    # Stratify into function related species (stratified) and function only (unstratified) 分层为功能包括物种组成和仅功能组成
    humann_split_stratified_table \
      --input result/humann4/path_relab.tsv \
      --output result/humann4/ 

### Difference comparison and bar chart (差异比较和柱状图)

    # * Input: Pathway abundance table result/humann4/path.tsv and experimental design result/metadata.txt
    # * Intermediate: Pathway abundance table file containing grouping information result/humann4/path.pcl
    # * Output: result/humann4/associate.txt
    # *   输入数据：通路丰度表格 result/humann4/path.tsv和实验设计 result/metadata.txt
    # *   中间数据：包含分组信息的通路丰度表格文件 result/humann4/path.pcl
    # *   输出结果：result/humann4/associate.txt

    # Add grouping to pathway abundance (在通路丰度中添加分组)
    # List of samples ID (提取样品列表)
    head -n1 result/humann4/path.tsv | sed 's/# Pathway HUMAnN v4.0.0.alpha.1/SampleID/' | tr '\t' '\n' > temp/header
    # The sample corresponds to group; in this example group is the 3rd column ($3) 样本对应分组，本示例分组为第3列($3)
    awk 'BEGIN{FS=OFS="\t"}NR==FNR{a[$1]=$3}NR>FNR{print a[$1]}' result/metadata.txt temp/header | tr '\n' '\t'|sed 's/\t$/\n/' > temp/group
    # replace grouping
    cat <(head -n1 result/humann4/path.tsv) temp/group <(tail -n+2 result/humann4/path.tsv) \
      > result/humann4/path.pcl
    head -n5 result/humann4/path.pcl
    tail -n5 result/humann4/path.pcl

    # For intergroup comparisons, small sample sizes may no significant differences
    # 组间比较，样本量少无差异
    # Backup demo data from HMP (hmp_pathabund.pcl replace to path.pcl)
    # wget -c http://www.imeta.science/github/EasyMetagenome/result/humann2/hmp_pathabund.pcl
    # cp -f hmp_pathabund.pcl result/humann4/path.pcl
    # Set input file 设置输入文件名
    pcl=result/humann4/path.pcl
    # Count the number of rows and columns in the table 7x4789 (统计表格行、列数量)
    csvtk -t stat ${pcl}
    head -n3 ${pcl} | cut -f 1-5
    # For the KW test with grouping, note the grouping column names in the second column 按分组KW检验，注意第二列的分组列名
    humann_associate --input ${pcl} \
        --focal-metadatum Group --focal-type categorical \
        --last-metadatum Group --fdr 0.2 \
        --output result/humann4/associate.txt
    # 4 colume include ID, mean, P-value and Q-value, 322 rows
    csvtk -t stat result/humann4/associate.txt
    head -n5 result/humann4/associate.txt

    # barplot show pathway taxonomic composition:  L-lysine fermentation to acetate and butanoate 
    # 柱状图展示通路的物种组成，如：L-赖氨酸发酵生成乙酸和丁酸
    # Set significant pathway ID from associate.txt，--sort sum metadata 按丰度和分组排序 
    # eg. P163-PWY (P 0.03, Q 0.09)/ 1CMET2-PWY (P 0.12, Q 0.18)
    path=P163-PWY
    grep $path result/humann4/associate.txt
    humann_barplot \
        --input ${pcl} --focal-feature ${path} \
        --focal-metadata Group --last-metadata Group \
        --output result/humann4/barplot_${path}.pdf --sort sum metadata 
    
### KEGG annotation (注释)

    # Support GO, PFAM, eggNOG, level4ec, KEGG, etc. Detail see `humann_regroup_table -h`。

    # regroup gene family into KO(uniref90_ko), options eggNOG(uniref90_eggnog) or EC(uniref90_level4ec)
    for i in `tail -n+2 result/metadata.txt|cut -f1`;do
      humann_regroup_table \
        -i temp/humann4/${i}_2_genefamilies.tsv \
        -g uniref90_ko \
        -o temp/humann4/${i}_ko.tsv
    done
    # Merge sample and correct name (合并并修正样本名)
    humann_join_tables \
      --input temp/humann4/ \
      --file_name ko \
      --output result/humann4/ko.tsv
    sed -i '1s/_Abundance-RPKs//g' result/humann4/ko.tsv
    tail result/humann4/ko.tsv

    # Similar to pathabundance, it can perform operations such as normalization, stratified, and barplot.
    # 与pathabundance类似，可进行标准化renorm、分层stratified、柱状图barplot等操作

    # Stratify into function related species (stratified) and function only (unstratified)
    # 分层为功能包括物种组成和仅功能组成
    humann_split_stratified_table \
      --input result/humann4/ko.tsv \
      --output result/humann4/ 
    wc -l result/humann4/ko*

    # KO to level 1/2/3, 9, 53 and 332 respectively (KO合并为更高层级L3/L2/L1)
    summarizeAbundance.py \
      -i result/humann4/ko_unstratified.tsv \
      -m ${db}/EasyMicrobiome/kegg/KO1-4.txt \
      -c 2,3,4 -s ',+,+,' -n raw --dropkeycolumn \
      -o result/humann4/KEGG
    wc -l result/humann4/KEGG*
    
## 2.2 GraPhlAn taxonomic and phylogenetic trees 高颜值物种或进化树

    # Use export2graphlan to create plotting files
    # 使用export2graphlan制作绘图文件
    conda activate graphlan
    # Detail parameters in PPT or run `export2graphlan.py --help`
    export2graphlan.py --skip_rows 1,2 -i result/metaphlan4/taxonomy.tsv \
      --tree temp/merged_abundance.tree.txt \
      --annotation temp/merged_abundance.annot.txt \
      --most_abundant 100 --abundance_threshold 20 --least_biomarkers 10 \
      --annotations 3,4 --external_annotations 7
    # graphlan annotation
    graphlan_annotate.py --annot temp/merged_abundance.annot.txt \
      temp/merged_abundance.tree.txt  temp/merged_abundance.xml
    # output PDF figure, annoat and legend
    graphlan.py temp/merged_abundance.xml result/metaphlan4/graphlan.pdf --external_legends 


## 2.3 LEfSe Differential analysis (差异分析物种)

    # Set input & output director for different versions (可调目录尝试不同版本)
    # For example, 12 sample examples, replace result to result12 (例如12个样本示例)
    # wget -c http://www.imeta.science/db/EasyMetagenome/result12.zip
    # unzip result12.zip
    result=result

    # Prepare the input file, and change the sample name to the group name (this can be done manually).
    # 准备输入文件，修改样本品为组名(可手动修改)
    # Extract the sample rows and replace them with one row for each sample, then change the ID to SampleID.
    # 提取样本行替换为每个样本一行，修改ID为SampleID
    sed -i 's/\r//g' $result/metaphlan4/taxonomy.tsv
    head -n1 $result/metaphlan4/taxonomy.tsv|tr '\t' '\n'|sed '1 s/ID/SampleID/' > temp/sampleid
    head -n3 temp/sampleid
    # 提取SampleID对应的分组Group(假设为metadata.txt中第二列$2)，替换换行\n为制表符\t，再把行末制表符\t替换回换行
    awk 'BEGIN{OFS=FS="\t"}NR==FNR{a[$1]=$3}NR>FNR{print a[$1]}' \
      $result/metadata.txt temp/sampleid|tr '\n' '\t'|sed 's/\t$/\n/' >temp/groupid
    cat temp/groupid
    sed 's/SampleID/subject_id/' temp/sampleid | tr '\n' '\t'|sed 's/\t$/\n/' > temp/sampleid2
    # 合并分组和数据(替换表头)
    cat temp/groupid temp/sampleid2 <(tail -n+2 $result/metaphlan4/taxonomy.tsv) | grep -v '#' > $result/metaphlan4/lefse.txt
    head -n3 $result/metaphlan4/lefse.txt

    # Method 1. ImageGP 2 https://www.bic.ac.cn/BIC/#/analysis?page=b%27MzY%3D%27&tool_type=tool
    # 方法1. 推荐 https://www.bic.ac.cn/BIC/ 中LEfSe分析
    # Error delete the subject_id line (在线出错，请删除subject_id行)

    # Method 2. LEfSe command line analysis
    # 方法2. LEfSe命令行分析
    # Example：https://github.com/SegataLab/lefse/blob/master/example/bioconda-lefse_run.sh
    conda activate lefse
    result=result
    lefse_run.py -h # LEfSe 1.1.01
    # LEfSe format (转换lefse格式)，s 2  correction for dependent comparison
    lefse_format_input.py $result/metaphlan4/lefse.txt \
      temp/input.in -c 1 -s 2 -o 1000000
    # LEfSe run
    lefse_run.py temp/input.in temp/input.res
    # Plot cladogram (绘制物种树注释差异)
    lefse_plot_cladogram.py temp/input.res \
      $result/metaphlan4/lefse_cladogram.pdf --format pdf
    # Plot features barplot (绘制所有差异特征的柱状图)
    lefse_plot_res.py temp/input.res \
      $result/metaphlan4/lefse_res.pdf --format pdf
    # Draw a bar chart of a single feature (绘制单个特征柱状图)
    # View differences features sorting by abundance (按丰度排序查看显著差异特征)
    grep -v '-' temp/input.res | sort -k3,3n
    # Select a feature to draw, such as Actinobacteria (指定特征绘图，如放线菌)
    lefse_plot_features.py -f one --format pdf \
      --feature_name "k__Bacteria.p__Actinobacteria" \
      temp/input.in temp/input.res \
      $result/metaphlan4/lefse_Actinobacteria.pdf
    # (Optional) Batch plot all difference feature bar charts (可选)批量绘制所有差异特征柱状图
    lefse_plot_features.py -f diff \
      --archive none --format pdf \
      temp/input.in temp/input.res \
      $result/metaphlan4/lefse_

## 2.4 Kraken2+Bracken taxonomic classification and abundance estimation (物种注释和丰度估计)

    # Kraken2 can quickly perform species annotation and quantification at the read level, 
    # and can also perform sequence species annotation at the contig, gene, and metagenomic assembly (MAG/bin) levels.
    # Kraken2可以快速完成读长(read)层面的物种注释和定量，还可以进行重叠群(contig)、基因(gene)、宏基因组组装基因组(MAG/bin)层面的序列物种注释。

    # Start the Kraken2 working environment(启动kraken2工作环境)
    conda activate kraken2
    # Record software version(记录软件版本)
    kraken2 --version # 2.1.6
    mkdir -p temp/kraken2

### Kraken2 taxonomic classification (物种注释)

    # Input(输入): temp/hr/{1}_?.fastq, {1} representative sample name (代表样本名)
    # Database(数据库): -db ${db}/kraken2/pluspf16g/
    # Output(输出结果): temp/kraken2/{1}_report and {1}_output
    # Feature table(物种丰度表): result/kraken2/taxonomy_count.txt 

    # Select a database based on server memory size or specific analytical needs
    # 根据服务器内存大小或具体分析需求选择数据库
    # pluspf16g for small memory / pluspf(100G) for human/animial / pluspfp(214G) for plant/soil
    type=pluspf16g
    # test first sample, 2min
    i=`tail -n+2 result/metadata.txt|cut -f1 | head -n1`
    time kraken2 --db ${db}/kraken2/${type}/ \
      --paired temp/hr/${i}_?.fastq \
      --threads 2 --use-names --report-zero-counts \
      --report temp/kraken2/${i}.report \
      --output temp/kraken2/${i}.output
      # output classified and unclassified reads
      # --unclassified-out "temp/kraken2/${i}.unclassified_reads#.fasta" \
      # --classified-out "temp/kraken2/${i}.classified_reads#.fasta" \
    # Batch processing of multiple samples to generate reports consumes a lot of memory but is fast; therefore, multi-task parallelism is not recommended.
    # 多样本批处理生成report，内存消耗大但速度快，不建议用多任务并行
    for i in `tail -n+3 result/metadata.txt | cut -f1`;do
      kraken2 --db ${db}/kraken2/${type} \
      --paired temp/hr/${i}_?.fastq \
      --threads 2 --use-names --report-zero-counts \
      --report temp/kraken2/${i}.report \
      --output temp/kraken2/${i}.output; done
      
    # Use Krakentools to convert the report to MPA format.
    # 使用krakentools转换report为mpa格式
    for i in `tail -n+2 result/metadata.txt | cut -f1`;do
      kreport2mpa.py -r temp/kraken2/${i}.report \
        --display-header -o temp/kraken2/${i}.mpa; done

    # Display by percentage (optional) 按照百分比展示(可选)
    for i in `tail -n+2 result/metadata.txt | cut -f1`;do
      kreport2mpa.py -r temp/kraken2/${i}.report \
        --percentages --display-header -o temp/kraken2/${i}.p.mpa; done
  
    # Merged samples into a table (合并样本为表格)
    mkdir -p result/kraken2
    # Same row number, sort ensure consistent (结果行数相同sort确保排序一致)
    tail -n+2 result/metadata.txt | cut -f1 | rush -j 1 \
      'tail -n+2 temp/kraken2/{1}.mpa | LC_ALL=C sort | cut -f 2 | sed "1 s/^/{1}\n/" \
      > temp/kraken2/{1}_count '
    # sample1 as header(提取第一样本品行名为表行名)
    tail -n+2 temp/kraken2/`tail -n 1 result/metadata.txt | cut -f 1`.mpa | LC_ALL=C sort | cut -f 1 | \
      sed "1 s/^/Taxonomy\n/" > temp/kraken2/0header_count
    head -n3 temp/kraken2/0header_count
    # paste merge into table (合并样本为表格)
    ls temp/kraken2/*count
    paste temp/kraken2/*count > result/kraken2/tax_count.mpa
    # Check table and statistics (检查表格及统计)
    head -n 5 result/kraken2/tax_count.mpa
    csvtk -t stat result/kraken2/tax_count.mpa

### Bracken abundance estimation (丰度估计)

    # Parameter description (参数简介):
    # -d database, -i input Kraken2 report file
    # -r read length, usually 150, -o outputs the re-estimated abundance
    # -l is the taxonomic level, domain (D), phylum (P), class (C), order (O), family (F), genus (G), and species (S) levels
    # -t is the threshold, defaults to 0, larger values result in higher reliability
    # -d为数据库，-i为输入kraken2报告文件
    # r是读长，通常为150，o输出重新估计的值
    # l为分类级，可选域D、门P、纲C、目O、科F、属G、种S级别丰度估计
    # t是阈值，默认为0，越大越可靠

    # Re-estimate the abundance of each sample in a loop. 
    # Please modify the tax and recalculate P and S once each.
    # 循环重新估计每个样品的丰度，请修改tax分别重新计算P和S各1次
    mkdir -p temp/bracken
    # read length (测序数据长度)、
    readLen=150
    # Only those present in 20% of the samples are retained (20%样本中存在才保留)
    prop=0.2
    # Classification is set into D, P, C, O, F, G, S, with commonly used categories being Kingdom (D), Phylum (P), Genus (G), and Species (S).
    # 设置分类级D,P,C,O,F,G,S，常用界D门P和属G种S
    for tax in P G S;do
    # tax=P
    for i in `tail -n+2 result/metadata.txt | cut -f1`;do
        # i=C1
        bracken -d ${db}/kraken2/${type}/ \
          -i temp/kraken2/${i}.report \
          -r ${readLen} -l ${tax} -t 0 \
          -o temp/bracken/${i}.brk \
          -w temp/bracken/${i}.report; done
    # bracken结果合并成表: 需按表头排序，提取第6列reads count，并添加样本名
    bash ${db}/EasyMicrobiome/script/bracken_integration.sh -t ${tax} \
      -i temp/bracken/*.brk \
      -o result/kraken2/bracken.${tax}.txt
    # 需要确认行数一致才能按以下方法合并      
    # wc -l temp/bracken/*.report
    # bracken结果合并成表: 需按表头排序，提取第6列reads count，并添加样本名
    # tail -n+2 result/metadata.txt | cut -f1 | rush -j 1 \
    #   'tail -n+2 temp/bracken/{1}.brk | LC_ALL=C sort | cut -f6 | sed "1 s/^/{1}\n/" \
    #   > temp/bracken/{1}.count'
    # 提取第一样本品行名为表行名
    # h=`tail -n1 result/metadata.txt|cut -f1`
    # tail -n+2 temp/bracken/${h}.brk | LC_ALL=C sort | cut -f1 | \
    #   sed "1 s/^/Taxonomy\n/" > temp/bracken/0header.count
    # 检查文件数，为n+1
    # ls temp/bracken/*count | wc
    # paste合并样本为表格，并删除非零行
    # paste temp/bracken/*count > result/kraken2/bracken.${tax}.txt
    # 统计行列，默认去除表头
    csvtk -t stat result/kraken2/bracken.${tax}.txt
    # 按频率过滤，-r可标准化，-e过滤(microbiome_helper)
    Rscript ${db}/EasyMicrobiome/script/filter_feature_table.R \
      -i result/kraken2/bracken.${tax}.txt \
      -p ${prop} \
      -o result/kraken2/bracken.${tax}.${prop}
    # head result/kraken2/bracken.${tax}.${prop}
    done
    csvtk -t stat result/kraken2/bracken.?.txt result/kraken2/bracken.?.$prop

    # Personalized results filtering, e.g. Chordata  (个性化结果筛选，如脊索动物)
    # Phylum level removal of chordate (humans) 门水平去除脊索动物(人)
    grep 'Chordata' result/kraken2/bracken.P.${prop}
    grep -v 'Chordata' result/kraken2/bracken.P.${prop} > result/kraken2/bracken.P.${prop}-H

    # Remove host contamination by species name, humans as an example: Homo sapiens (按物种名手动去除宿主污染，以人为例)
    grep 'Homo sapiens' result/kraken2/bracken.S.${prop}
    grep -v 'Homo sapiens' result/kraken2/bracken.S.${prop} \
      > result/kraken2/bracken.S.${prop}-H

    # Clean big annotation files for each sequence (清理每条序列的注释大文件)

    /bin/rm -rf temp/kraken2/*.output

### Diversity and Visualization (多样性和可视化)

    # Alpha diversity：Berger Parker’s (BP), Simpson’s (Si), inverse Simpson’s (ISi), Shannon’s (Sh)
    echo -e "SampleID\tBerger Parker\tSimpson\tinverse Simpson\tShannon" > result/kraken2/alpha.txt
    for i in `tail -n+2 result/metadata.txt|cut -f1`;do
        echo -e -n "$i\t" >> result/kraken2/alpha.txt
        for a in BP Si ISi Sh;do
            alpha_diversity.py -f temp/bracken/${i}.brk -a $a | cut -f 2 -d ':' | tr '\n' '\t' >> result/kraken2/alpha.txt
        done
        echo "" >> result/kraken2/alpha.txt
    done
    cat result/kraken2/alpha.txt

    # Beta diversity
    beta_diversity.py -i temp/bracken/*.brk --type bracken > result/kraken2/beta.txt
    cat result/kraken2/beta.txt

    # Krona plot
    for i in `tail -n+2 result/metadata.txt|cut -f1`;do
        kreport2krona.py -r temp/bracken/${i}.report -o temp/bracken/${i}.krona --no-intermediate-ranks
        ktImportText temp/bracken/${i}.krona -o result/kraken2/krona.${i}.html
    done

    # Pavian Sankey diagram (桑基图)：https://fbreitwieser.shinyapps.io/pavian/ 
    # Online visualization: `Upload sample set` (temp/bracken/*.report), supports multiple samples; `Sample` view results; `Configure Sankey` to configure graph styles; `Save Network` to download the graph webpage.
    # 在线可视化:，左侧菜单，Upload sample set (temp/bracken/*.report)，支持多样本同时上传；Sample查看结果，Configure Sankey配置图样式，Save Network下载图网页

    # For diversity/composition visualization, see 3StatPlot.sh (多样性分析/物种组成可视化见3StatPlot.sh)


# 3. Assemble-based (三、组装分析流程)

##  3.1 Assemble (组装)

    # Start working environment (启动工作环境)
    conda activate megahit
    
### MEGAHIT Assembly (组装)

    # Delete directory for rerun (删除旧文件夹才能重新运行)
    # /bin/rm -rf temp/megahit
    # Assembly，demo 3p 80m, TB may n days (TB数据要几天)
    megahit -v # MEGAHIT v1.2.9
    /usr/bin/time -v megahit -t 3 \
        -1 `tail -n+2 result/metadata.txt|cut -f1|sed 's/^/temp\/hr\//;s/$/_1.fastq/'|tr '\n' ','|sed 's/,$//'` \
        -2 `tail -n+2 result/metadata.txt|cut -f1|sed 's/^/temp\/hr\//;s/$/_2.fastq/'|tr '\n' ','|sed 's/,$//'` \
        -o temp/megahit 
    # Stat contig size, 160M, eg. 18sample 100G data 10h contigs 1.8G
    # If too many contigs, filter by length to reduce data and improve gene integrity. See appendix megahit
    # 如果contigs太多，可以按长度筛选，降低数据量，提高基因完整度，详见附录megahit
    seqkit stat temp/megahit/final.contigs.fa
    # Preview first 6 rows and 60 characters each line (预览重叠群最前6行，前60列字符)
    head -n6 temp/megahit/final.contigs.fa | cut -c1-60
    # Delete temporary files (删除临时文件)
    /bin/rm -rf temp/megahit/intermediate_contigs

### metaSPAdes fine assembly (option) 精细组装(可选) 

    # More time and memory consume (精细但使用内存和时间更多)
    mkdir -p temp/metaspades
    /usr/bin/time -v -o metaspades.py.log metaspades.py -t 3 -m 100 \
      `tail -n+2 result/metadata.txt|cut -f1|sed 's/^/temp\/hr\//;s/$/_1.fastq/'|sed 's/^/-1 /'| tr '\n' ' '` \
      `tail -n+2 result/metadata.txt|cut -f1|sed 's/^/temp\/hr\//;s/$/_2.fastq/'|sed 's/^/-2 /'| tr '\n' ' '` \
      -o temp/metaspades
    # View calculate time (Elapsed time, 2.5h) and Memory consume (Maximum resident set size, 22G)
    cat metaspades.py.log
    # 219M，contigs bigger than megahit
    ls -sh temp/metaspades/contigs.fasta
    seqkit stat temp/metaspades/contigs.fasta
    # Select VIP result, Delete temporary files (删除临时文件)
    mv temp/metaspades/contigs.fasta temp/metaspades.fa
    /bin/rm -rf temp/metaspades/

    # Note: metaSPAdes supports hybrid assembly of second and third generation systems (see appendix). 
    # Additionally, there are OPERA-MS assembly solutions for second and third generation systems.
    # 注：metaSPAdes支持二、三代混合组装，见附录，此外还有OPERA-MS组装二、三代方案

### QUAST assembly evaluation (组装评估)

    mkdir -p result/megahit

    # QUAST evaluation, generating reports in various formats such as tsv/txt text, html webpage, and PDF.
    # QUAST评估，生成report文本tsv/txt、网页html、PDF等格式报告
    quast.py temp/megahit/final.contigs.fa \
      -o result/megahit/quast -t 2

    # (opt) megahit versus metaspades
    quast.py --label "megahit,metapasdes" \
        temp/megahit/final.contigs.fa \
        temp/metaspades.fa \
        -o result/quast

    # (opt)metaquast access (更全面评估，需下载相关数据库受网速影响可能时间很长或失败)
    # metaquast based on silva, and top 50 species genome, 18min
    time metaquast.py temp/megahit/final.contigs.fa \
      -o result/megahit/metaquast

## 3.2 Gene prediction, cluster & quantitfy(基因预测、去冗余和定量)

### metaProdigal Gene prediction (基因预测)

    # Input file: Assembled sequences temp/megahit/final.contigs.fa
    # Output file: Gene sequence predicted by prodigal temp/prodigal/gene.fa
    # For large gene sequences, refer to the appendix for prodigal gene file splitting and parallel computation.
    # 输入文件：组装的序列 result/megahit/final.contigs.fa
    # 输出文件：prodigal预测的基因序列 temp/prodigal/gene.fa
    # 基因大，可参考附录prodigal拆分基因文件，并行计算

    mkdir -p temp/prodigal
    # prodigal meta mode, 8m, > and 2>&1 record gene.log, ~1G 1h
    time prodigal -i temp/megahit/final.contigs.fa \
        -d temp/prodigal/gene.fa \
        -o temp/prodigal/gene.gff \
        -p meta -f gff > temp/prodigal/gene.log 2>&1 
    # Check if the log has completed (查看日志是否运行完成)
    tail temp/prodigal/gene.log
    # Count of genes (基因数量) 261K, 6G18s3M
    seqkit stat temp/prodigal/gene.fa 
    # Count complete genes, if the data volume is large, select complete gene, 73K
    # 统计完整基因数量，数据量大可只用完整基因部分
    grep -c 'partial=00' temp/prodigal/gene.fa 
    # Select complete gene (提取完整基因)
    seqkit grep -n -r -p "partial=00" temp/prodigal/gene.fa > temp/prodigal/full_length.fa
    seqkit stat temp/prodigal/full_length.fa
    
    # Opt. Single sample assemble bacth run (可选单样本组装可批量运行)
    for i in `tail -n+2 result/metadata.txt | cut -f1`;do
    	mkdir -p temp/prodigal/${i}/
    	prodigal -i result/megahit/${i}/final.contigs.fa \
            -d temp/prodigal/${i}/gene.fa \
            -o temp/prodigal/${i}/gene.gff \
            -p meta -f gff > temp/prodigal/${i}/gene.log 2>&1 
    done

### cd-hit cluster & redundancy (基因聚类/去冗余)

    # Input file: gene sequences predicted by prodigal temp/prodigal/gene.fa
    # Output files: Deredundant gene and protein sequences: result/NR/nucleotide.fa, result/NR/protein.fa
    # 输入文件：prodigal预测的基因序列 temp/prodigal/gene.fa
    # 输出文件：去冗余后的基因和蛋白序列：result/NR/nucleotide.fa, result/NR/protein.fa

    mkdir -p result/NR
    # aS coverage, c similarity, G local alignment, g optimal solution, T multithreading, M memory 0 unlimited
    # aS覆盖度，c相似度，G局部比对，g最优解，T多线程，M内存0不限制
    # 2M 384p 8m，3M384p15m，2千万需要2000h，多线程可加速
    time cd-hit-est -i temp/prodigal/gene.fa \
        -o result/NR/nucleotide.fa \
        -aS 0.9 -c 0.95 -G 0 -g 0 -T 0 -M 0
    # Counting the number of non-redundant genes, the number of genes in a single assembly does not decrease significantly, such as 3M-2M. Multiple batch assemblies show high redundancy.
    # 统计非冗余基因数量，单次拼接结果数量下降不大，如3M-2M，多批拼接冗余度高
    grep -c '>' result/NR/nucleotide.fa
    # Translate nucleic acids into corresponding protein sequences, and use `--trim` to remove trailing asterisks.
    # 翻译核酸为对应蛋白序列, --trim去除结尾的*
    seqkit translate --trim result/NR/nucleotide.fa \
        > result/NR/protein.fa 
    # Redundancy removal for both batches of data was accelerated using cd-hit-est-2d, see appendix.
    # 两批数据去冗余使用cd-hit-est-2d加速，见附录
    
    # batch samples (样本批量运行)
    for i in `tail -n+2 result/metadata.txt | cut -f1`;do
    	mkdir -p result/NR/${i}
        cd-hit-est -i temp/prodigal/${i}/gene.fa \
            -o result/NR/${i}/nucleotide.fa \
            -aS 0.9 -c 0.95 -G 0 -g 0 -T 0 -M 0
        grep -c '>' result/NR/${i}/nucleotide.fa
        seqkit translate --trim result/NR/${i}/nucleotide.fa \
            > result/NR/${i}/protein.fa 
    done

### salmon quantitfy (基因定量)

    # Input file: Redundancy-free gene sequence: result/NR/nucleotide.fa
    # Output file: Salmon quantification: result/salmon/gene.count, gene.TPM
    # 输入文件：去冗余后的基因序列：result/NR/nucleotide.fa
    # 输出文件：Salmon定量：result/salmon/gene.count, gene.TPM

    mkdir -p temp/salmon
    salmon -v # 1.10.3

    # build index, -t sequence, -i index (建索引, -t序列, -i 索引, 2m)
    time salmon index -t result/NR/nucleotide.fa \
      -p 3 -i temp/salmon/index 

    # Quantitative: -l automatic library type selection, p threads, --meta metagenomics
    # 定量，l文库类型自动选择，p线程，--meta宏基因组, 10s
    i=C1
    time salmon quant -i temp/salmon/index -l A -p 3 --meta \
        -1 temp/hr/${i}_1.fastq -2 temp/hr/${i}_2.fastq \
        -o temp/salmon/${i}.quant
    # parallel, 1m, 18 samples 30m
    time tail -n+2 result/metadata.txt | cut -f1 | rush -j 2 \
      "salmon quant -i temp/salmon/index -l A -p 3 --meta \
        -1 temp/hr/{1}_1.fastq -2 temp/hr/{1}_2.fastq \
        -o temp/salmon/{1}.quant"

    # Merge (合并)
    mkdir -p result/salmon
    salmon quantmerge --quants temp/salmon/*.quant \
        -o result/salmon/gene.TPM
    salmon quantmerge --quants temp/salmon/*.quant \
        --column NumReads -o result/salmon/gene.count
    sed -i '1 s/.quant//g' result/salmon/gene.*
    # Preview (预览结果)
    head -n3 result/salmon/gene.*

## 3.3 Functional gene annotation (功能基因注释)

    # input：result/NR/protein.fa
    # COG: result/eggnog/cogtab.count
    #      result/eggnog/cogtab.count.spf (for STAMP)
    # KO table：result/eggnog/kotab.count
    # CAZy Carbohydrate：result/dbcan3/cazytab.count
    # Expanded to other databases (可拓展其它数据库)

### eggNOG gene annotation(COG/KEGG/CAZy基因注释)

    # eggNOG: https://github.com/eggnogdb/eggnog-mapper
    # Run and record the software version (运行并记录软件版本)
    conda activate eggnog
    emapper.py --version
    mkdir -p temp/eggnog
    # emapper-2.1.7 / Expected eggNOG DB version: 5.0.2 / diamond version 2.0.15

    # run emapper, 3p 100m, default diamond 1e-3; 2M,32p,1.5h
    time emapper.py --data_dir ${db}/eggnog \
      -i result/NR/protein.fa --cpu 3 -m diamond --override \
      -o temp/eggnog/output

    # Format the results and display the table headers (格式化结果并显示表头)
    grep -v '^##' temp/eggnog/output.emapper.annotations | sed '1 s/^#//' \
      > temp/eggnog/output
    csvtk -t headers -v temp/eggnog/output

    # create COG/KO/CAZy abundance table
    mkdir -p result/eggnog
    # Show help (显示帮助)
    summarizeAbundance.py -h
    # Summary, the 7 columns COG_category are alphabetically separated, and the 12 columns KEGG_ko and 19 columns CAZy are comma-separated. The original values are summed.
    # 汇总，7列COG_category按字母分隔，12列KEGG_ko和19列CAZy按逗号分隔，原始值累加
    summarizeAbundance.py \
      -i result/salmon/gene.TPM \
      -m temp/eggnog/output --dropkeycolumn \
      -c '7,12,19' -s '*+,+,' -n raw \
      -o result/eggnog/eggnog
    sed -i 's#^ko:##' result/eggnog/eggnog.KEGG_ko.raw.txt
    sed -i '/^-/d' result/eggnog/eggnog*
    head -n3 result/eggnog/eggnog*
    # eggnog.CAZy.raw.txt  eggnog.COG_category.raw.txt  eggnog.KEGG_ko.raw.txt

    # format to STAMP spf format
    awk 'BEGIN{FS=OFS="\t"} NR==FNR{a[$1]=$2} NR>FNR{print a[$1],$0}' \
      ${db}/EasyMicrobiome/kegg/KO_description.txt \
      result/eggnog/eggnog.KEGG_ko.raw.txt | \
      sed 's/^\t/Unannotated\t/' \
      > result/eggnog/eggnog.KEGG_ko.TPM.spf
    head -n 5 result/eggnog/eggnog.KEGG_ko.TPM.spf
    # KO to level 1/2/3
    summarizeAbundance.py \
      -i result/eggnog/eggnog.KEGG_ko.raw.txt \
      -m ${db}/EasyMicrobiome/kegg/KO1-4.txt \
      -c 2,3,4 -s ',+,+,' -n raw --dropkeycolumn \
      -o result/eggnog/KEGG
    head -n3 result/eggnog/KEGG*
    
    # CAZy
    awk 'BEGIN{FS=OFS="\t"} NR==FNR{a[$1]=$2} NR>FNR{print a[$1],$0}' \
       ${db}/EasyMicrobiome/dbcan2/CAZy_description.txt result/eggnog/eggnog.CAZy.raw.txt | \
      sed 's/^\t/Unannotated\t/' > result/eggnog/eggnog.CAZy.TPM.spf
    head -n 3 result/eggnog/eggnog.CAZy.TPM.spf
    
    # COG
    awk 'BEGIN{FS=OFS="\t"} NR==FNR{a[$1]=$2"\t"$3} NR>FNR{print a[$1],$0}' \
      ${db}/EasyMicrobiome/eggnog/COG.anno result/eggnog/eggnog.COG_category.raw.txt > \
      result/eggnog/eggnog.COG_category.TPM.spf
    head -n 3 result/eggnog/eggnog.COG_category.TPM.spf

### CARD耐药基因

    # CARD: https://card.mcmaster.ca/ 
    # GitHub: https://github.com/arpcard/rgi
    # Result：protein.json, upload to CARD; protein.txt, annotation list
    mkdir -p result/card
    conda activate rgi
    rgi main -v # 6.0.5
    # Simplified Protein ID (简化蛋白ID)
    cut -f 1 -d ' ' result/NR/protein.fa > temp/protein.fa
    grep '>' result/NR/protein.fa | head -n 3
    grep '>' temp/protein.fa | head -n 3
    
    # Protein-level annotation ARG
    # rgi load -i $db/card/card.json --card_annotation $db/card/card.fasta
    time rgi main -i temp/protein.fa -t protein \
      -n 9 -a DIAMOND --include_loose --clean \
      -o result/card/protein
    head -n3 result/card/protein.txt
    # WARNING baeR ---> hsp.bits: 140.6 <class 'float'> ? <class 'str'>  Exception : <class 'KeyError'> -> '2885' -> Model(2885) missing in database. Please generate new database.
    # Software and database have bugs that do not affect the results (新版软件与数据库bug，不影响主体结果)
    
    # (Optional) Gene-level annotation ARG (可选)基因层面注释ARG 
    cut -f 1 -d ' ' result/NR/nucleotide.fa > temp/nucleotide.fa
    grep '>' temp/nucleotide.fa | head -n3
    rgi main -i temp/nucleotide.fa -t contig \
      -n 9 -a DIAMOND --include_loose --clean \
      -o result/card/nucleotide
    head -n3 result/card/nucleotide.txt
    
    # (Optional) Overlap Contigs Level Annotation (ARG) (可选)重叠群层面注释ARG
    cut -f 1 -d ' ' temp/megahit/final.contigs.fa > temp/contigs.fa
    grep '>' temp/contigs.fa | head -n3
    time rgi main -i temp/contigs.fa -t contig \
      -n 9 -a DIAMOND --include_loose --clean \
      -o result/card/contigs
    head result/card/contigs.txt

## 3.4 Kraken2 gene taxonomic annotation (基因物种注释)

    # Generate report in default taxid output
    conda activate kraken2
    # 16g 60.69%, pf 73.02%, pfp 74.20%
    kraken2 --db ${db}/kraken2/pluspf16g \
      result/NR/nucleotide.fa \
      --threads 3 \
      --report temp/NRgene.report \
      --output temp/NRgene.output
    # Genes & taxid list
    grep '^C' temp/NRgene.output | cut -f 2,3 | sed '1 i Name\ttaxid' \
      > temp/NRgene.taxid
    # Add taxonomy
    awk 'BEGIN{FS=OFS="\t"} NR==FNR{a[$1]=$0} NR>FNR{print $1,a[$2]}' \
      $db/EasyMicrobiome/kraken2/taxonomy.txt \
      temp/NRgene.taxid > result/NR/nucleotide.tax
    conda activate eggnog 
    summarizeAbundance.py \
      -i result/salmon/gene.TPM \
      -m result/NR/nucleotide.tax  --dropkeycolumn \
      -c '2,3,4,5,6,7,8,9' -s ',+,+,+,+,+,+,+,' -n raw \
      -o result/NR/tax
    wc -l result/NR/tax*|sort -n

# 4. Binning (四、分箱挖掘单菌基因组)

## MetwWRAP binning (分箱)

    # GitHub: https://github.com/bxlab/metaWRAP
    # For mining single-bacterial genomes, a single sample size of 6GB+ is recommended, and for complex samples such as soil, a data size of 30GB+ is recommended. 
    # The demonstration data consists of 6 samples (approximately 1GB), which is insufficient to obtain single-bacterial genomes. Therefore, official sequencing data is used for demonstration and explanation.
    # 挖掘单菌基因组，推荐单样本6GB+，复杂样本如土壤推荐数据量30GB+
    # 演示数据6个样品~1G，无法获得单菌基因组，这里使用官方测序数据演示讲解

    cd $wd
    conda activate metawrap
    metawrap -v # 1.3.2
    mkdir -p temp/bin temp/bin_refine temp/drep_in result/bin

    # Input: quaility control & host removal sequences, *_1.fastq和*_2.fastq,  
    #        assemble contigs, temp/megahit/final.contigs.fa
    # Output: Binning result: temp/bin
    #     Stat：temp/bin_refine/metawrap_50_10_bins.stats

### All samples mix Binning (多样本混合分箱)

    # Using maxbin2, metabat2，3p41m；32p 18sample 16-19h
    metawrap binning -o temp/bin \
      -t 3 -a temp/megahit/final.contigs.fa \
      --metabat2 --maxbin2 \
      temp/hr/*.fastq
    #  --concoct > /dev/null 2>&1 # increase 3~10 fold calculating, /dev/null omit Warning

    ### Bin refinement (分箱提纯), 8p1h
    metawrap bin_refinement \
      -o temp/bin_refine \
      -A temp/bin/metabat2_bins/ \
      -B temp/bin/maxbin2_bins/ \
      -c 50 -x 10 -t 8
    # -C temp/bin/concoct_bins/ \
    # Bin count 22
    tail -n+2 temp/bin_refine/metawrap_50_10_bins.stats|wc -l
    # Plot in temp/bin_refine/figures/

    # Together: all bins in one directory (所有分箱至同一目录)
    # Mix binning link and rename (混合组装分箱链接和重命名)
    ln -s `pwd`/temp/bin_refine/metawrap_50_10_bins/bin.* temp/drep_in/
    ls -l temp/drep_in/
    # CentOS rename
    rename 'bin.' 'Mx_All_' temp/drep_in/bin.*
    # Ubuntu rename
    rename s/bin./Mx_All_/ temp/drep_in/bin.*
    ls temp/drep_in/Mx*

### Single sample binning (单样本分箱)

    # When multiple samples cannot be completed due to hardware or computation time limitations, 
    # single-sample assembly and binning are required. Multiple samples offer greater information abundance, 
    # resulting in more binned products and making it easier to reduce contamination.
    # 多样本受硬件、计算时间限制无法完成时，需要单样本组装、分箱。多样本信息丰度，分箱结果更多，更容易降低污染。

    # Configure global threads, number of parallel tasks, and conditions for filtering bins.
    # 设置全局线程、并行任务数和筛选分箱的条件
    # p: threads (线程数), j: job (任务数), c: complete (完整度), x: contaminate (污染率)
    p=16
    j=3
    c=50
    x=10
    
    # Assemble(组装), 10min; 18s6h
    time tail -n+2 result/metadata.txt|cut -f1|rush -j ${j} \
      "metawrap assembly -m 200 -t ${p} --megahit \
        -1 temp/hr/{}_1.fastq -2 temp/hr/{}_2.fastq \
        -o temp/megahit/{}"

    # Binning (分箱), 3m
    time tail -n+2 result/metadata.txt|cut -f1|rush -j ${j} \
      "metawrap binning \
        -o temp/bin/{} -t ${p} \
        -a temp/megahit/{}/final_assembly.fasta \
        --metabat2 --maxbin2 \
        temp/hr/{}_*.fastq" # --concoct > /dev/null 2>&1

    # Bin refinement (分箱提纯), 42m
    time tail -n+2 result/metadata.txt|cut -f1|rush -j ${j} \
      "metawrap bin_refinement \
      -o temp/bin_refine/{} -t ${p} \
      -A temp/bin/{}/metabat2_bins/ \
      -B temp/bin/{}/maxbin2_bins/ \
      -c ${c} -x ${x}"
      # -C temp/bin/{}/concoct_bins/ \
    # 分别为1,2,2个
    tail -n+2 result/metadata.txt|cut -f1|rush -j 1 \
      "tail -n+2 temp/bin_refine/{}/metawrap_50_10_bins.stats|wc -l "

    # Together, link and rename (单样品分箱链接和重命名)
    for i in `tail -n+2 result/metadata.txt|cut -f1`;do
       ln -s `pwd`/temp/bin_refine/${i}/metawrap_50_10_bins/bin.* temp/drep_in/
       rename 'bin.' "Sg_${i}_" temp/drep_in/bin.*
       rename "s/bin./Sg_${i}_/" temp/drep_in/bin.*
    done
    # Delete fail link (删除空白中无效链接)
    # /bin/rm -f temp/drep_in/*\*
    # Bin source, 22 mix, 22 single sample
    ls temp/drep_in/|cut -f 1 -d '_'|uniq -c
    # Bin source, each sample
    ls temp/drep_in/|cut -f 2 -d '_'|cut -f 1 -d '.' |uniq -c


### (Opt) Subgroup binning (可选)分组分箱

    # For samples >30 or data volume >300G, completing hybrid assembly and binning on a 1TB memory fat node may result in insufficient memory and take >1 week or even 1 month. It is necessary to divide the research into small groups based on similar conditions and locations, and each group should write a metadata.txt file.
    # 样本>30或数据量>300G在1TB内存胖结点上完成混合组装和分箱可能内存不足、且时间>1周甚至1月，需要对研究相近条件、地点进行分小组，且每组编写一个metadata??.txt。

    # Group: Young/Centenarians
    g=Young
    grep -P "Group|${g}" result/metadata.txt > result/meta${g}.txt
    cat result/meta${g}.txt

    # Assemble, 9m, <30 samples or <300G data, ~12h
    metawrap assembly -m 600 -t 32 --megahit \
      -1 `tail -n+2 result/meta${g}.txt|cut -f1|sed 's/^/temp\/hr\//;s/$/_1.fastq/'|tr '\n' ','|sed 's/,$//'` \
      -2 `tail -n+2 result/meta${g}.txt|cut -f1|sed 's/^/temp\/hr\//;s/$/_2.fastq/'|tr '\n' ','|sed 's/,$//'` \
      -o temp/megahit_${g}

    # Prepare data directory
    mkdir -p temp/${g}/
    for i in `tail -n+2 result/meta${g}.txt|cut -f1`;do
        ln -s `pwd`/temp/hr/${i}*.fastq temp/${g}/; done
    # Bin (分箱), 4m
    metawrap binning -o temp/bin/${g} \
      -t 32 -a temp/megahit_${g}/final_assembly.fasta \
      --metabat2 --maxbin2 \
      temp/${g}/*.fastq

    # Bin refinement (分箱提纯), 30m
    metawrap bin_refinement \
      -o temp/bin_refine/${g} \
      -A temp/bin/${g}/metabat2_bins/ \
      -B temp/bin/${g}/maxbin2_bins/ \
      -c 50 -x 10 -t 32
    wc -l temp/bin_refine/${g}/metawrap_50_10_bins.stats

    # Together, link and rename (单样品分箱链接和重命名)
    ln -s `pwd`/temp/bin_refine/${g}/metawrap_50_10_bins/bin.* temp/drep_in/
    rename "s/bin./Gp_${g}_/" temp/drep_in/bin.* # Ubuntu
    rename 'bin.' "Gp_${g}_" temp/drep_in/bin.* # CentOS

    # Stat each type and sample
    echo -e "Type\tCount" > result/bin/count.txt
    ls temp/drep_in/|cut -f 1 -d '_'|uniq -c|awk '{print $2"\t"$1}' >> result/bin/count.txt
    ls temp/drep_in/|cut -f 1-2 -d '_'|uniq -c|awk '{print $2"\t"$1}' >> result/bin/count.txt
    cat result/bin/count.txt

### Option. CheckM2 Reassessment (可选. 重新评估)

    conda activate checkm2
    mkdir -p temp/checkm2 result/checkm2
    # 22 genomes, 2m
    time checkm2 predict --input temp/drep95/dereplicated_genomes/* \
      --output-directory temp/checkm2 --threads 8
    ln temp/checkm2/quality_report.tsv result/checkm2/
    less result/checkm2/quality_report.tsv 

## 4.2 dRep genome redundance (基因组去冗余)

    cd $wd
    conda activate drep
    mkdir -p temp/drep95

    # dereplicate by species：10 min; 44 genome, 22 from mix samples, 22 from signle samle
    time dRep dereplicate temp/drep95/ \
      -g temp/drep_in/*.fa  \
      -sa 0.95 -nc 0.30 -comp 50 -con 10 -p 8
    # log in temp/drep95/log/cmd_logs, -d show more detail
    ls temp/drep95/dereplicated_genomes/|cut -f 1 -d '_'|sort|uniq -c
    # ls temp/drep95/dereplicated_genomes/|sed 's/.fa//' > temp/drep95/data_tables/id
    # format_drep2cluster.pl -i temp/drep95/data_tables/Cdb.csv -d temp/drep95/data_tables/id \
    #   -o temp/drep95/data_tables/Cdb.list -h header num

    # main result in temp/drep95
    # 1. Redundancy genome catalogue (非冗余基因组集)：temp/drep95/dereplicated_genomes/*.fa
    # 2. Table(聚类表): temp/drep95/data_tables/Cdb.csv
    # 3. Plot(聚类和质量图)：temp/drep95/figures/*clustering*

    # (Option) drep in 99% strain level, 7m (可选)按株水平99%去冗余，20-30min
    mkdir -p temp/drep99
    time dRep dereplicate temp/drep99/ \
      -g temp/drep_in/*.fa \
      -sa 0.99 -nc 0.30 -comp 50 -con 10 -p 16
    # species level 26, strain level 29
    ls -l temp/drep99/dereplicated_genomes/ | grep '.fa' | wc -l

## 4.3 CoverM quantify in genome (基因组定量)

    conda activate coverm
    mkdir -p temp/coverm
    
    # Single test, Y1 30s
    i=`tail -n+2 result/metadata.txt|cut -f1 | head -n1`
    time coverm genome --coupled temp/hr/${i}_1.fastq temp/hr/${i}_2.fastq \
      --genome-fasta-directory temp/drep95/dereplicated_genomes/ -x fa \
      -o temp/coverm/${i}.txt
    cat temp/coverm/${i}.txt
    
    # Parallel, 4min；注：尝试拆分2步，节省建索引时间
    tail -n+3 result/metadata.txt|cut -f1|rush -j 2 \
      "coverm genome --coupled temp/hr/{}_1.fastq temp/hr/{}_2.fastq -t 3 \
      --genome-fasta-directory temp/drep95/dereplicated_genomes/ -x fa \
      -o temp/coverm/{}.txt > temp/coverm/{}.log "

    # Merge to table (结果合并)
    mkdir -p result/coverm
    conda activate humann4
    sed -i 's/_1.fastq Relative Abundance (%)//' temp/coverm/*.txt
    humann_join_tables --input temp/coverm --file_name txt --output result/coverm/abundance.tsv
    csvtk -t stat result/coverm/abundance.tsv

    # Group mean (按组求均值，需要metadata中有3列且每个组有多个样本)
    Rscript ${db}/EasyMicrobiome/script/otu_mean.R --input result/coverm/abundance.tsv \
      --metadata result/metadata.txt \
      --group Group --thre 0 \
      --scale TRUE --zoom 100 --all TRUE --type mean \
      --output result/coverm/group_mean.txt
    # https://www.bic.ac.cn/ImageGP/ 直接选择热图可视化

## 4.4 GTDB taxonomic classifications of prokaryote (原核基因组注释和进化树)

    conda activate gtdbtk
    export GTDBTK_DATA_PATH="${db}/gtdb"
    gtdbtk -v # 2.5.2
    
    # Classify, 8p, 22 genomes, 40m
    mkdir -p temp/gtdb_classify
    time gtdbtk classify_wf \
        --genome_dir temp/drep95/dereplicated_genomes \
        --out_dir temp/gtdb_classify \
        --extension fa --skip_ani_screen \
        --prefix tax \
        --cpus 8
    # less -S view, press q quit; 26 bac short for Bacterial, 0 ar short for Archaea
    less -S temp/gtdb_classify/tax.bac120.summary.tsv
    less -S temp/gtdb_classify/tax.ar53.summary.tsv

    # (Optional) Annotations for all MAG species (可选)所有MAG物种注释
    mkdir -p temp/gtdb_all
    # 10000 genome，32p，100min
    time gtdbtk classify_wf --genome_dir temp/drep_in/ \
        --out_dir temp/gtdb_all --extension fa --skip_ani_screen \
        --prefix tax --cpus 32
    
    # multi-sequence alignment and phylogenetic tree (多序列对齐结果建树)
    mkdir -p temp/gtdb_infer
    gtdbtk infer --msa_file temp/gtdb_classify/align/tax.bac120.user_msa.fasta.gz \
        --out_dir temp/gtdb_infer --prefix tax --cpus 3
    # tree `tax.unrooted.tree` using iTOL online visualization

    # Tree annotation: gtdb-tk taxonomy (tax.bac120.summary.tsv) and drep genome info(Widb.csv)
    mkdir -p result/itol
    # Taxonomy table (分类学表)
    tail -n+2 temp/gtdb_classify/tax.bac120.summary.tsv|cut -f 1-2|sed 's/;/\t/g'|sed '1 s/^/ID\tDomain\tPhylum\tClass\tOrder\tFamily\tGenus\tSpecies\n/' > result/itol/tax.txt
    head result/itol/tax.txt
    # Genome info (基因组评估信息)
    sed 's/,/\t/g;s/.fa//' temp/drep95/data_tables/Widb.csv|cut -f 1-7,11|sed '1 s/genome/ID/' > result/itol/genome.txt
    head result/itol/genome.txt
    # Merge (整合注释文件)
    awk 'BEGIN{OFS=FS="\t"} NR==FNR{a[$1]=$0} NR>FNR{print $0,a[$1]}' result/itol/genome.txt result/itol/tax.txt|cut -f 1-8,10- > result/itol/annotation.txt
    head result/itol/annotation.txt
    # Add relative abundance of each sample/group (添加各样本/组相对丰度)
    awk 'BEGIN{OFS=FS="\t"} NR==FNR{a[$1]=$0} NR>FNR{print $0,a[$1]}' <(sed '1 s/Genome/ID/' result/coverm/abundance.tsv) result/itol/annotation.txt|cut -f 1-15,17- > result/itol/annotation2.txt
    head result/itol/annotation2.txt    


# 5 (Optional) Bacterial genome (可选)单菌基因组

## 5.1 Fastp quality control (质量控制)

    # Need genome sequencing data to start, 30s/per sample
    mkdir -p temp/qc/ 
    time tail -n+2 result/metadata.txt | cut -f1 | rush -j 2 \
      "time fastp -i seq/{1}_1.fq.gz -I seq/{1}_2.fq.gz \
        -j temp/qc/{1}_fastp.json -h temp/qc/{1}_fastp.html \
        -o temp/qc/{1}_1.fastq -O temp/qc/{1}_2.fastq \
        > temp/qc/{1}.log 2>&1"

## 5.2 metaspades assembly (组装)

    conda activate megahit
    spades.py -v # v3.15.4
    mkdir -p temp/spades result/spades
    # 127 genoms, 1m17s
    time tail -n+2 result/metadata.txt|cut -f1|rush -j 3 \
    	"spades.py --pe1-1 temp/qc/{1}_1.fastq \
    	  --pe1-2 temp/qc/{1}_2.fastq \
    	  -t 16 --isolate --cov-cutoff auto \
    	  -o temp/spades/{1}" 
	
    # Filter sequences > 1k and summarize/statistically analyze them (筛选>1k的序列并汇总、统计)
    time tail -n+2 result/metadata.txt|cut -f1|rush -j 3 \
	  "seqkit seq -m 1000 temp/spades/{1}/contigs.fasta \
	    > temp/spades/{1}.fa"
	  seqkit stat temp/spades/*.fa | sed 's/temp\/spades\///;s/.fa//' > result/spades/stat1k.txt

## 5.3 checkm quality assessment (质量评估)

    # checkm评估质量
    conda activate drep
    checkm # CheckM v1.1.2
  	mkdir -p temp/checkm result/checkm
  	# 127 genoms, 1m17s
  	time checkm lineage_wf -t 8 -x fa temp/spades/ temp/checkm
  	# format checkm jason to tab
  	checkmJason2tsv.R -i temp/checkm/storage/bin_stats_ext.tsv \
  	  -o temp/checkm/bin_stats.txt
      csvtk -t  pretty temp/checkm/bin_stats.txt | less
	
    # (可选)checkm2评估(测试中...)
    conda activate checkm2
  	mkdir -p temp/checkm2
  	time checkm2 predict --threads 8 --input temp/spades/ --output-directory temp/checkm2
	
    # 筛选污染和高质量基因组 >5% contamination and high quailty
  	awk '$5<90 || $10>5' temp/checkm/bin_stats.txt | csvtk -t cut -f 1,5,10,4,9,2 > temp/checkm/contamination5.txt
  	tail -n+2 temp/checkm/contamination5.txt|wc -l 
  	# 筛选高质量用于下游分析 <5% high-quality for down-stream analysis
  	awk '$5>=90 && $10<=5' temp/checkm/bin_stats.txt | csvtk -t cut -f 1,5,10,4,9,2 | sed '1 i ID\tCompleteness\tContamination\tGC\tN50\tsize' > result/checkm/Comp90Cont5.txt
  	tail -n+2 result/checkm/Comp90Cont5.txt|wc -l 
  	# 链接高质量基因组至新目录，单菌完整度通常>99%
  	mkdir -p temp/drep_in/
  	for n in `tail -n+2 result/checkm/Comp90Cont5.txt|cut -f 1`;do
  	  ln temp/spades/${n}.fa temp/drep_in/
  	done


## 5.4 metawarp binning for mixture (混菌分箱)

    # 分箱和提纯binning & refinement
    conda activate metawrap
    mkdir -p temp/binning temp/bin
    time tail -n+2 temp/checkm/contamination5.txt|cut -f1|rush -j 3 \
      "metawrap binning \
        -o temp/binning/{} -t 8 \
        -a temp/spades/{}/contigs.fasta \
        --metabat2 --maxbin2 \
        temp/qc/{}_*.fastq" 
    time tail -n+2 temp/checkm/contamination5.txt|cut -f1|rush -j 15 \
      "metawrap bin_refinement \
      -o temp/bin/{} -t 8 \
      -A temp/binning/{}/metabat2_bins/ \
      -B temp/binning/{}/maxbin2_bins/ \
      -c 50 -x 10"

    # 分箱结果汇总
  	echo -n -e "" > temp/bin/metawrap.stat
  	for m in `tail -n+2 temp/checkm/contamination5.txt|cut -f1`;do
  	  echo ${m} >> temp/bin/metawrap.stat
  	  cut -f1-4,6-7 temp/bin/${m}/metawrap_50_10_bins.stats >> temp/bin/metawrap.stat
  	done
  	# 分箱后的按b1,b2,b3重命名共培养，单菌也可能减少污染
  	for m in `tail -n+2 temp/checkm/contamination5.txt|cut -f1`;do
          c=1
      	for n in `tail -n+2 temp/bin/$m/metawrap_50_10_bins.stats|cut -f 1`;do
      	  cp temp/bin/$m/metawrap_50_10_bins/${n}.fa temp/drep_in/${m}b${c}.fa
      	  ((c++))
  	done
  	done

    # 分箱前后统计比较
    # 如107个测序分箱为352个基因组，共418个基因组
  	tail -n+2 temp/checkm/contamination5.txt|wc -l
  	ls temp/drep_in/*b?.fa | wc -l
  	ls temp/drep_in/*.fa | wc -l
  	# 重建新ID列表，A代表所有，B代表Bin分箱过的单菌
  	ls temp/drep_in/*.fa|cut -f 3 -d '/'|sed 's/.fa//'|sed '1 i ID'|less -S>result/metadataA.txt
  	ls temp/drep_in/*b?.fa|cut -f 3 -d '/'|sed 's/.fa//'|sed '1 i ID'|less -S>result/metadataB.txt

    # 可视化混菌中覆盖度分布，以第一污染菌为例
    mkdir -p temp/cov
    for i in `tail -n+2 temp/checkm/contamination5.txt|cut -f1`;do
    grep '>' temp/drep_in/${i}*|cut -f 3 -d '/'|sed 's/.fa:>NODE//'|cut -f 1,2,4,6 -d '_'|sed 's/_/\t/g'|sed '1i Genome\tContig\tLength\tvalue' > temp/cov/${i}
    sp_scatterplot2.sh -f temp/cov/${i} -X Contig -Y value -c Genome -s Length -O `tail -n+2 temp/cov/${i}|cut -f2|uniq|awk '{print "\""$1"\""}'|tr "\n" ","|sed 's/,$//'` -w 40 -u 12.5
    done


## 5.5 drep redudancy (基因组去冗余)

  	mkdir -p temp/drep95/ temp/drep99/
  	conda activate drep
  	ls temp/drep_in/*.fa|wc -l
  	# 相似度sa 0.99995 去重复, 0.99 株水平, 0.95 种水平
  	dRep dereplicate \
  	  -g temp/drep_in/*.fa \
  	  -sa 0.99 -nc 0.3 -p 16 -comp 50 -con 10 \
  	  temp/drep99
  	ls temp/drep99/dereplicated_genomes/|wc -l
  	dRep dereplicate \
  	  -g temp/drep_in/*.fa \
  	  -sa 0.95 -nc 0.3 -p 16 -comp 50 -con 10 \
  	  temp/drep95
  	# 统计使用基因组数量丢弃stat total used genomes no discard
  	grep 'passed checkM' temp/drep95/log/logger.log|sed 's/[ ][ ]*/ /g'|cut -f 4 -d ' '
  	# 去冗余后数量，418变为49种
  	ls temp/drep95/dereplicated_genomes/|wc -l
  	# 唯一和重复的基因组unique and duplicate genome
  	csvtk cut -f 11 temp/drep95/data_tables/Widb.csv | sort | uniq -c
  	# 整理种列表
  	echo "SampleID" > result/metadataS.txt
  	ls temp/drep95/dereplicated_genomes/|sed 's/\.fa//' >> result/metadataS.txt
  	# 基因组信息genomeInfo.csv 
  	sed 's/,/\t/g;s/.fa//' temp/drep95/data_tables/genomeInfo.csv |sed '1 s/genome/ID/' > result/gtdb_all/genome.txt

    # 非冗余菌定量
    conda activate coverm
    mkdir -p temp/coverm result/coverm
    # (可选)单样本测试, 3min
    i=X001
    time coverm genome --coupled temp/qc/${i}_1.fastq temp/qc/${i}_2.fastq \
      --genome-fasta-directory temp/drep95/dereplicated_genomes/ -x fa \
      -o temp/coverm/${i}.txt -t 32
    cat temp/coverm/${i}.txt
    # 并行计算, 173样本4min
    tail -n+2 result/metadata.txt|cut -f1|rush -j 4 \
      "coverm genome --coupled temp/qc/{}_1.fastq temp/qc/{}_2.fastq \
      --genome-fasta-directory temp/drep95/dereplicated_genomes/ -x fa \
      -o temp/coverm/{}.txt -t 32"
    # 结果合并
    conda activate humann3
    sed -i 's/_1.fastq Relative Abundance (%)//' temp/coverm/*.txt
    humann_join_tables --input temp/coverm \
      --file_name txt \
      --output result/coverm/abundance.tsv    

## 5.6 GTDB taxonomy classification (物种注释)

  	conda activate gtdbtk
  	# 所有基因组注释，400g, 1h, 1T
  	mkdir -p temp/gtdb_all result/gtdb_all
  	memusg -t gtdbtk classify_wf \
  	  --genome_dir temp/drep_in/ \
  	  --out_dir temp/gtdb_all/ \
        --extension fa --skip_ani_screen \
        --prefix tax \
        --cpus 16
      
  	# 95%聚类种基因组注释，40g, 1h, 500G
  	mkdir -p temp/gtdb_95 result/gtdb_95
  	# Taxonomy classify 
  	gtdbtk classify_wf \
  	  --genome_dir temp/drep95/dereplicated_genomes/ \
  	  --out_dir temp/gtdb_95 \
        --extension fa --skip_ani_screen \
        --prefix tax \
        --cpus 8
  	# Phylogenetic tree infer
  	gtdbtk infer \
  	  --msa_file temp/gtdb_95/align/tax.bac120.user_msa.fasta.gz \
  	  --out_dir temp/gtdb_95 \
  	  --cpus 8 --prefix g >> temp/gtdb_95/infer.log 2>&1
  	ln `pwd`/temp/gtdb_95/infer/intermediate_results/g.unrooted.tree result/gtdb_95/
  
  	# 细菌format to standard 7 levels taxonomy 
  	tail -n+2 temp/gtdb_95/classify/tax.bac120.summary.tsv|cut -f 1-2|sed 's/;/\t/g'|sed '1 s/^/ID\tKingdom\tPhylum\tClass\tOrder\tFamily\tGenus\tSpecies\n/' > result/gtdb_95/tax.bac.txt
  	# 古菌(可选)
  	tail -n+2 temp/gtdb_95/classify/tax.ar122.summary.tsv|cut -f 1-2|sed 's/;/\t/g'|sed '1 s/^/ID\tKingdom\tPhylum\tClass\tOrder\tFamily\tGenus\tSpecies\n/' > result/gtdb_95/tax.ar.txt
  	cat result/gtdb_95/tax.bac.txt <(tail -n+2 result/gtdb_95/tax.ar.txt) > result/gtdb_95/tax.txt
  	
  	# Widb.csv 非冗余基因组信息
  	sed 's/,/\t/g;s/.fa//' temp/drep95/data_tables/Widb.csv|cut -f 1-7,11|sed '1 s/genome/ID/' > result/gtdb_95/genome.txt
  	# 整合物种注释和基因组信息 Integrated taxonomy and genomic info 
  	awk 'BEGIN{OFS=FS="\t"} NR==FNR{a[$1]=$0} NR>FNR{print $0,a[$1]}' result/gtdb_95/genome.txt result/gtdb_95/tax.txt|cut -f 1-8,10- > result/gtdb_95/annotation.txt
  	# csvtk -t headers -v result/gtdb_95/annotation.txt
	
  	# 制作itol files
  	cd result/gtdb_95
  	table2itol.R -D plan1 -a -c double -i ID -l Genus -t %s -w 0.5 annotation.txt
  	table2itol.R -D plan2 -a -d -c none -b Phylum -i ID -l Genus -t %s -w 0.5 annotation.txt
  	table2itol.R -D plan3 -c keep -i ID -t %s annotation.txt
  	table2itol.R -D plan4 -a -c factor -i ID -l Genus -t %s -w 0 annotation.txt
  	# Stat each level
  	echo -e 'Taxonomy\tKnown\tNew' > tax.stat
  	for i in `seq 2 8`;do
  	  head -n1 tax.txt|cut -f ${i}|tr '\n' '\t' >> tax.stat
  	  tail -n+2 tax.txt|cut -f ${i}|grep -v '__$'|sort|uniq -c|wc -l|tr '\n' '\t' >> tax.stat
  	  tail -n+2 tax.txt|cut -f ${i}|grep '__$'|wc -l >> tax.stat; done
  	cat tax.stat
  	tail -n+2 tax.txt|cut -f3|sort|uniq -c|awk '{print $2"\t"$1}'|sort -k2,2nr > count.phylum
  	cat count.phylum
  	cd ../..

## 5.7 Functional annotation by eggnog/dbcan/arg/antismash

    ## 基因注释
  	mkdir -p temp/prodigal
  	conda activate eggnog
      prodigal -v # V2.6.3
      # 50g, 31s, 4m
      time tail -n+2 result/metadataS.txt|cut -f1|rush -j 10 \
  	"prodigal \
  	  -i temp/drep95/dereplicated_genomes/{1}.fa \
  	  -o temp/prodigal/{1}.gff  \
  	  -a temp/prodigal/{1}.faa \
  	  -d temp/prodigal/{1}.ffn \
  	  -p single -f gff" 
  	seqkit stat temp/prodigal/*.ffn | sed 's/temp\/prodigal\///;s/\.ffn//;s/[[:blank:]]\{1,\}/\t/g' | cut -f 1,3-  \
  	  > result/prodigal.txt

    # 碳水化合物注释
    mkdir -p temp/dbcan3 result/dbcan3
    time tail -n+2 result/metadataS.txt|cut -f1|rush -j 9 \
  	"diamond blastp \
  	  --db ${db}/dbcan3/CAZyDB \
  	  --query temp/prodigal/{1}.faa \
  	  --outfmt 6 --threads 8 --quiet --log \
  	  --evalue 1e-102 --max-target-seqs 1 --sensitive \
  	  --block-size 6 --index-chunks 1 \
  	  --out temp/dbcan3/{1}_diamond.f6"
  	wc -l temp/dbcan3/*.f6|head -n-1|awk '{print $2"\t"$1}'|cut -f3 -d '/'|sed 's/_diamond.f6//'|sed '1 i ID\tCAZy'|less -S > result/dbcan3/gene.count
  	# format blast2genelist
  	for i in `tail -n+2 result/metadataS.txt|cut -f1`;do
  	format_dbcan3list.pl \
  	  -i temp/dbcan3/${i}_diamond.f6 \
  	  -o temp/dbcan3/${i}.list
  	done
  	# CAZy type count
  	for i in `tail -n+2 result/metadataS.txt|cut -f1`;do
  	  tail -n+2 temp/dbcan3/${i}.list|cut -f2|sort|uniq -c|awk '{print $2"\t"$1}'|sed "1 i CAZy\t${i}"|less -S > temp/dbcan3/${i}_CAZy.tsv
  	done
  	# merge2table
  	conda activate humann3
  	humann_join_tables \
  	  --input temp/dbcan3/ --file_name CAZy \
  	  --output result/dbcan3/cazy.txt
  	csvtk -t stat result/dbcan3/cazy.txt
  	# merge to level1
  	paste <(cut -f1 result/dbcan3/cazy.txt) <(cut -f1 result/dbcan3/cazy.txt|tr '0-9' ' '|sed 's/ //g') | sed '1 s/\tCAZy/\tLevel1/' >  result/dbcan3/cazy.L1
  	summarizeAbundance.py \
  	  -i result/dbcan3/cazy.txt \
  	  -m result/dbcan3/cazy.L1 \
  	  -c 2 -s ',' -n raw  --dropkeycolumn \
  	  -o result/dbcan3/sum
  	# 基因相似度
  	echo -e 'Name\tCAZy\tIdentity\tGenome' > result/dbcan3/identity.txt
  	for i in `tail -n+2 result/metadataS.txt|cut -f1`;do
  	  csvtk -t replace -f 2 -p "\d+" -r "" temp/dbcan3/${i}.list | uniq | tail -n+2 | sed "s/$/\t${i}/" >> result/dbcan3/identity.txt
  	done
  	csvtk -t stat result/dbcan3/identity.txt
  	sp_boxplot.sh -f result/dbcan3/identity.txt -m T -F CAZy -d Identity

    # 耐药基因
  	mkdir -p temp/card result/card
    conda activate rgi6
    # load database 加载数据库
    rgi load -i ${db}/card/card.json \
    	--card_annotation ${db}/card/card.fasta --local
    	# Annotation 蛋白注释
    	# 默认为0, --include_loose 可极大增加结果，519/4657=11.14%;  --exclude_nudge结果不变，但jason为空
    time for i in `tail -n+2 result/metadataS.txt|cut -f1`;do
    	# i=X004b2
    	cut -f 1 -d ' ' temp/prodigal/${i}.faa | sed 's/\*//' > temp/prodigal/protein_${i}.fa
    	rgi main \
    	--input_sequence temp/prodigal/protein_${i}.fa \
    	--output_file temp/card/${i} \
    	--input_type protein --clean \
    	--num_threads 8 --alignment_tool DIAMOND > temp/log 2>&1
    done

    mkdir -p temp/ARG
    for FILE in temp/card/*.txt; do
    # 获取文件名的基本部分
    BASENAME=$(basename "$FILE")
    OUTPUT="temp/ARG/${BASENAME}"

    # 提取第1列，第17列和第28列，并写入对应的输出文件
    awk -F'\t' -v OFS='\t' 'BEGIN {print "ORF_ID"} NR>1 {print $1}' "$FILE" > "$OUTPUT"
    done

    # 处理每个输出文件
    for file in temp/ARG/*.txt; do
        # 去掉回车符并更新文件
        awk -v fname="${file%.txt}" 'BEGIN {FS=OFS="\t"} {gsub(/\r/, ""); if (NR==1) print $0, fname; else print $1, $2, 1}' "$file" > "${file%.txt}_updated.txt"
        echo "$file 处理完成"
    done

    mkdir -p temp/ARG_updated
    mv temp/ARG/*updated.txt temp/ARG_updated
    # merge2table
    conda activate humann3
    humann_join_tables \
      --input temp/ARG_updated/ --file_name Mx_All \
      --output temp/ARG_updated/ARG.tsv
    sed -i 's/\t\t/\t1\t/g; s/\t$/\t1/' temp/ARG_updated/ARG.tsv
    
    mkdir -p temp/ARG2
    # 获取ORF_ID对应的基因和抗生素注释
    for FILE in temp/card/*.txt; do
        # 获取文件名的基本部分
        BASENAME=$(basename "$FILE")
        OUTPUT="temp/ARG2/${BASENAME}"
        # 提取第17列和第28列，并写入对应的输出文件
        awk -F'\t' -v OFS='\t' 'BEGIN {print "ORF_ID", "AMR_Gene_Family", "Antibiotic"} NR>1 {print $1, $17, $28}' "$FILE" > "$OUTPUT"
    done
    
    for FILE in temp/card/*.txt; do
        # 获取文件名的基本部分
        BASENAME=$(basename "$FILE")
        OUTPUT="temp/ARG2/${BASENAME}"
        # 提取第17列和第28列，并写入对应的输出文件
        awk -F'\t' -v OFS='\t' 'NR>1 {print $1, $17, $28}' "$FILE" > "$OUTPUT"
    done
    cat temp/ARG2/*.txt > temp/ARG2/annotation.txt
    echo -e "ORF_ID\tAMR_Gene_Family\tAntibiotic\n$(cat temp/ARG2/annotation.txt)" > temp/ARG2/annotation.txt
    
    #排序ORF_ID列，合并ARG2/annotation.txt和ARG/updated/ARG.txt,保存为ARG_final.tsv
    sed -n '1p' temp/ARG_updated/ARG.tsv > temp/ARG_updated/sorted_ARG.tsv
    sed -n '1!p' temp/ARG_updated/ARG.tsv | sort -k1,1 >> temp/ARG_updated/sorted_ARG.tsv
    sed -n '1p' temp/ARG2/annotation.txt > temp/ARG2/sorted_annotation.txt
    sed -n '1!p' temp/ARG2/annotation.txt | sort -k1,1 >> temp/ARG2/sorted_annotation.txt
    join -t $'\t' -1 1 -2 1 temp/ARG_updated/sorted_ARG.tsv temp/ARG2/sorted_annotation.txt > temp/ARG2/ARG_final.tsv


# 6. (Optional) pangenome (可选)泛基因组

利用宏基因组测序数据进行泛基因组分析，首先需要确定要分析泛基因组的对象，比如Alistipes属或者Alistipes putredinis菌种，这里的流程以Alistipes putredinis为例进行泛基因组数据的挖掘分析

## 6.1 Download genome from NCBI (下载基因组)

    # 从NCBI上获取assembly_summary_refseq.txt文件
    mkdir -p temp/metapangenome
    cd temp/metapangenome
    wget -c https://ftp.ncbi.nlm.nih.gov/genomes/refseq/assembly_summary_refseq.txt

    assemblyFile="assembly_summary_refseq.txt"
    taxa="Alistipes_putredinis"
    taxDir="Alistipes_putredinis"

    mkdir ncbi_genomes
    mkdir ncbi_genomes/$taxDir

    for taxon in $taxa; do
    taxonSafe=$(echo $taxon | tr ' ' '_')
    taxonGrep=$(echo $taxon | tr '_' ' ')
    # 基因组ftp地址
    ftpDir=$(grep "$taxonGrep" $assemblyFile | awk -F"\t" '{print $20}')
    cd ncbi_genomes/$taxDir

    for dirPath in $ftpDir; do
    # 获取ID
    genomeID=$(echo "$dirPath" | sed 's/^.*\///g')
    # 获取基因组链接
    ftpPath="$dirPath/${genomeID}_genomic.fna.gz"
    # 重命名
    friendlyName=$(grep "$genomeID" ../../$assemblyFile | awk -F"\t" '{print $8}' | awk '{print $1 FS $2}' | tr -d '_,.-' | sed -E 's/^(.)[A-z0-9]* ([A-z0-9]{2}).*$/\1\2/')
    deflineName=$(grep "$genomeID" ../../$assemblyFile | awk -F"\t" '{print $8 " " $9}' | sed 's/strain=//' | tr ' -' '_' | tr -d '/=(),.')
    genomeID="${friendlyName}_$genomeID"
    # 下载并解压
    wget $ftpPath -O $genomeID.fa.gz
    gunzip $genomeID.fa.gz
    # 序列信息重命名
    awk -v defline="$deflineName" '/^>/{print ">" defline "_ctg" (++i)}!/^>/' $genomeID.fa >> temp.txt
    mv temp.txt $genomeID.fa
    done
    done

    # 加入自有数据得到的对应细菌基因组
    # 非冗余基因组集位置：temp/drep95/dereplicated_genomes/*.fa
    # 非冗余基因组分类信息位置：temp/gtdb_classify
    # 需要根据分类信息找到分析物种或类群的基因组，然后与NCBI下载的基因组文件合并为1个文件
    cp temp/drep95/dereplicated_genomes/Mx_All_55.fa ./ncbi_genomes/Alistipes_putredinis/Mx_All_55.fa
    cp ./ncbi_genomes/Alistipes_putredinis/Mx_All_55.fa ./ncbi_genomes/Alistipes_putredinis/Mx_All_55.fna

    # 由于每一个基因组文件里面包含多条序列，需要将多条序列合并为1条
    # Directory containing the files
    input_dir="ncbi_genomes/Alistipes_putredinis/"

    # 循环处理每一个fna文件
    for file in "$input_dir"/*.fna; do
      filename=$(basename "$file")
      output_file="${file%.fna}_merged_sequence.fna"
      awk '/^>/{if (seqlen){print seq} if (NR==1){print} seq=""; seqlen=0; next} {seq=seq $0; seqlen+=length($0)} END{print seq}' "$file" > "$output_file"
    done

    # 合并NCBI下载多个基因组文件并修改格式
    cat "$input_dir"/*merged_sequence.fna > $species-isolates.fa

    ## 获取fa文件的head信息
    conda activate anvio-8
    grep '^>' Alistipes-putredinis-isolates.fa > headers01.txt
    ## 在开始时清理 contig 文件中的 fasta序列的head信息，以确保下游操作顺利进行
    species="Alistipes-putredinis"
    anvi-script-reformat-fasta -o $species-isolates-CLEAN.fa -l 1000 \
                  --simplify-names \
                  -r $species-isolates-contigIDs.txt $species-isolates.fa
    mv $species-isolates-CLEAN.fa $species-isolates.fa

## 6.2 CONTIGS library 建立序列库

    # 建立Alistipes-putredinis-isolates-CONTIGS库
    anvi-gen-contigs-database -f $species-isolates.fa -n Alistipes \
                  -o $species-isolates-CONTIGS.db \
                  --split-length 1500000 \
                  --num-threads 24
    ## 从 hmm 集合中获取细菌单拷贝基因 16S 信息
    anvi-run-hmms -c $species-isolates-CONTIGS.db \
                  --num-threads 24
    
    ## 安装ncbi-cogs用于功能注释
    anvi-setup-ncbi-cogs -T 24 --just-do-it --reset
    
    ## COGs功能注释
    anvi-run-ncbi-cogs -c $species-isolates-CONTIGS.db --search-with blastp -T 24
    
    ## 导出注释得到的功能数据表
    # 如果没有进行COGs功能注释，可跳过该步骤
    touch ncbi-cogs-results-fmt-$species-isolates01.tsv
    anvi-import-functions -c $species-isolates-CONTIGS.db \
                  -i ncbi-cogs-results-fmt-$species-isolates01.tsv
    
    ## 建议映射             
    bowtie2-build $species-isolates.fa $species-isolates
    
    ## 使用 bowtie2和默认参数（“--sensitive”），将宏基因组匹配到 contigs 数据库中的contigs上
    prefix="$species-isolates"
    ## 单样本处理示例，其它样本相同
    bowtie2 -x $prefix -1 /temp/hr2/FC1_1.fastq -2 \
                  /temp/hr2/FC1_2.fastq --no-unal \
                  --threads 24 | samtools view -b - | samtools sort -@ 12 - > bt_mapped_Alistipes-putredinis-isolates/FC1.bam
    ## 排序和建立索引
    cd bt_mapped_Alistipes-putredinis-isolates
    samtools index FC1.bam
    cd ..

    ## 批量运行
    # 设置 Bowtie2 索引的前缀
    prefix="$species-isolates"
    # 包含 .fastq 文件的目录
    input_dir="../hr2"
    # BAM 文件的输出目录
    output_dir="bt_mapped_$species-isolates"
    # 确保输出目录存在
    mkdir -p $output_dir
    
    # 循环遍历输入目录中的所有 *_1.fastq 文件
    for fastq1 in "$input_dir"/*_1.fastq; do
      # 通过删除“_1.fastq”后缀获取样本名称
      sample_name=$(basename "$fastq1" _1.fastq)
      # 定义相应的_2.fastq文件
      fastq2="$input_dir/${sample_name}_2.fastq"
      # 定义输出 BAM 文件
      bam_output="${output_dir}/${sample_name}.bam"
      # 检查对应的_2.fastq文件是否存在
      if [[ -f "$fastq2" ]]; then
        # 运行 Bowtie2 和 SAMtools
        bowtie2 -x $prefix -1 $fastq1 -2 $fastq2 --no-unal --threads 24 | \
        samtools view -b - | \
        samtools sort -@ 12 - > $bam_output
        # 索引 BAM 文件
        samtools index $bam_output
      else
        # 如果不存在相应的 _2.fastq 文件，则警告
        echo "Warning: Missing corresponding _2.fastq file for $fastq1"
      fi
    done

    ## 使用 anvi’o 分析映射结果
    # 单样本运行
    anvi-profile -i bt_mapped_$species-isolates/FC1.bam -c $prefix-CONTIGS.db \
                  -W -M 2500 -T 12 --write-buffer-size 500 -o profs_mapped_$species-isolates/FC1/
    
    # 批量运行
    # 定义输入 BAM 目录
    bam_dir="bt_mapped_$species-isolates/"
    # 定义分析结果的输出目录
    output_dir="profs_mapped_$species-isolates"
    mkdir -p $output_dir
    # 循环遍历输入目录中的所有 BAM 文件
    for bam_file in $bam_dir/*.bam; do
      # 从 BAM 文件中提取样本名称（例如 FC1、FE1 等）
      sample_name=$(basename "$bam_file" .bam)
      # 为当前样本创建输出目录
      sample_output_dir="$output_dir/$sample_name"
      mkdir -p "$sample_output_dir"
      echo "Profiling sample $sample_name..."
      # 运行 anvi-profile 命令
      anvi-profile -i "$bam_file" -c "${prefix}-CONTIGS.db" -W -M 2500 -T 12 --write-buffer-size 500 -o "$sample_output_dir"
    done


## 6.3 pan-genome analysis (泛基因组分析)

    ## 生成合并的 anvi’o 配置文件数据库
    anvi-merge profs_mapped_$species-isolates/*/PROFILE.db \
               -o $species-isolates-MERGED \
               -c $species-isolates-CONTIGS.db
    ## 得到集合文件
    for split_name in `sqlite3 $species-isolates-CONTIGS.db 'select split from splits_basic_info;'`
    do
        GENOME=`echo $split_name | awk 'BEGIN{FS="-"}{print $1}'`
        echo -e "$split_name\t$GENOME"
    done > $species-GENOME-COLLECTION.txt
    
    anvi-import-collection $species-GENOME-COLLECTION.txt \
                           -c $species-isolates-CONTIGS.db \
                           -p $species-isolates-MERGED/PROFILE.db \
                           -C Genomes
                           
    anvi-summarize -c $species-isolates-CONTIGS.db \
                   -p $species-isolates-MERGED/PROFILE.db \
                   -C Genomes \
                   --init-gene-coverages \
                   -o $species-isolates-SUMMARY
               
    ## 生成internal-genomes-table.txt文件
    taxonPrefix=$species-isolates
    collection="Genomes"
    echo -e "name\tbin_id\tcollection_id\tprofile_db_path\tcontigs_db_path" > $taxonPrefix-internal-genomes-table.txt
    for bin in $(ls $taxonPrefix-SUMMARY/bin_by_bin); do
    echo -e "$bin\t$bin\t$collection\t$taxonPrefix-MERGED/PROFILE.db\t$taxonPrefix-CONTIGS.db" >> $taxonPrefix-internal-genomes-table.txt
    done

    ## 泛基因组计算
    anvi-gen-genomes-storage -i $species-isolates-internal-genomes-table.txt \
                             -o $species-PAN-GENOMES.db
    anvi-pan-genome -g $species-PAN-GENOMES.db \
                    --use-ncbi-blast \
                    --minbit 0.5 \
                    --mcl-inflation 10 \
                    --project-name $species-PAN \
                    --num-threads 24
    anvi-meta-pan-genome -i $species-isolates-internal-genomes-table.txt \
                    -g $species-PAN-GENOMES.db -p $species-PAN/*PAN.db

    ## 泛基因组绘图
    ## 此处的I为ip地址，例如这里是服务器的ip地址，需要根据自己实际情况修改；P为未被占用的端口，一般为8080或8082
    ## 直接在浏览器中打开结果给出的服务器网址，点击draw可得到相应的图，可在交互界面上得到基因组相关信息并对图进行调整
    anvi-display-pan -g $species-PAN-GENOMES.db \
                     -p $species-PAN/$species-PAN-PAN.db \
                     -I 192.168.60.214 \
    				 -P 8082
    
    # 每个样本中Alistipes putredinis的基因组信息                
    anvi-interactive -p $species-isolates-MERGED/PROFILE.db \
                     -c $species-isolates-CONTIGS.db \
                     -I 192.168.60.214 \
                     -P 8082

# Appendix: Common Analysis Problems and Supplementary Code (附录：常见分析问题和补充代码)

## Calculation Time Statistics Table (计算时间统计表)

    # 在60核(p)及以上服务器，单样本3个并行推荐16p，混合组装分箱推荐32p。
    # 小数据：2个7.5M样本，在72个核、512GB服务器上测试。
    # 大数据：20个10G样本，在192个核、2TB大内存服务器上测试。
    # | 步骤 | 数据小(6Gx3) | 数据(10Gx20) | 备注 |
    # | --- | --- | --- | --- |
    # | fastqc | 33s | 15m |  |
    # | seqkit | 2s | 15m |  |
    # | fastp | 2s | 40m |  |
    # | kneaddata | 25s | 5h |  |
    # | humann | 34m | 30h |  |
    # | megahit | 39s | 15h |  |
    # | binning | 6h | 16h | -concoct |
    # | binrefine | 2h | 3h |  |
    # | coverm | 4m | 30m |  |
    # 注：m为分，h为小时，d为天


## Supplementary scripts (补充代码)

    #**for循环批量处理样本列表**
    # 基于样本元数据提取样本列表命令解析
    # 去掉表头
    tail -n+2 result/metadata.txt
    # 提取第一列样本名
    tail -n+2 result/metadata.txt|cut -f1
    # 循环处理样本
    for i in `tail -n+2 result/metadata.txt|cut -f1`;do echo "Processing "$i; done
    # ` 反引号为键盘左上角Esc键下面的按键，一般在数字1的左边，代表运行命令返回结果
    

## KneadData质控去宿主

    # **双端序列质控后是否配对的检查**
    # 双端序列质控后序列数量不一致是肯定出错了。但即使序列数量一致，也可能序列不对。在运行metawrap分箱时会报错。可以kneaddata运行时添加--reorder来尝试解决。以下提供了检查双端序列ID是否配对的比较代码
    # 文件
    i=C1
    seqkit seq -n -i temp/qc/${i}_1_kneaddata_paired_1.fastq|cut -f 1 -d '/' | head > temp/header_${i}_1
    seqkit seq -n -i temp/qc/${i}_1_kneaddata_paired_2.fastq|cut -f 1 -d '/' | head > temp/header_${i}_2
    cmp temp/header_${i}_?

    # 如果序列双端名称一致，且单样本质控结果异常时使用，适合旧版本：新版全kneaddata 1.12已经将此功能添加至流程，以下代码运行返倒引起错误
    #序列改名，解决NCBI SRA数据双端ID重名问题，详见[《MPB：随机宏基因组测序数据质量控制和去宿主的分析流程和常见问题》](https://mp.weixin.qq.com/s/ovL4TwalqZvwx5qWb5fsYA)。
    # 以0.12.0为例，序列中间不一至存在:1和:2会无结果，
    @A01909:80:HFT7YDSX5:2:1101:1289:1000 1:N:0:ATGAATAT+TTAAGTGG
    @A01909:80:HFT7YDSX5:2:1101:1289:1000 2:N:0:ATGAATAT+TTAAGTGG
    # 以0.12.0为例，最终的结果为无空格，结果/1和/2
    @A01909:80:HFT7YDSX5:2:1101:1289:1000.1:N:0:ATGAATAT+TTAAGTGG/1
    @A01909:80:HFT7YDSX5:2:1101:1289:1000.1:N:0:ATGAATAT+TTAAGTGG/2
    # 以单个样本修改
    i=A01
    sed -i '1~4 s/ 1:/.1:/;1~4 s/$/\/1/' temp/qc/${i}_1.fastq
    sed -i '1~4 s/ 2:/.1:/;1~4 s/$/\/2/' temp/qc/${i}_2.fastq
    # 批量修改
    sed -i '1~4 s/ 1:/.1:/;1~4 s/$/\/1/' temp/qc/${i}_1.fastq
    sed -i '1~4 s/ 1:/.1:/;1~4 s/$/\/1/' temp/qc/${i}_2.fastq

    # **Perl环境不匹配**
    # 报错'perl binaries are mismatched'的解决
    e=~/miniconda3/envs/meta
    PERL5LIB=${e}/lib/5.26.2:${e}/lib/5.26.2/x86_64-linux-thread-multi

    ## **Java环境错误**
    # 出现错误 Unrecognized option: -d64，为版本不匹配——重装Java运行环境解决：
    conda install -c cyclus java-jdk
    # 若出现错误 Error message returned from Trimmomatic :
    # Error: Invalid or corrupt jarfile \~/miniconda3/envs/kneaddata/share/trimmomatic/trimmomatic；找不到程序，修改配置文件指定脚本名称
    sed -i 's/trimmomatic\*/trimmomatic.jar/' ~/miniconda3/envs/kneaddata/lib/python3.10/site-packages/kneaddata/config.py


    # **Python环境不匹配-找不到包module**
    # ModuleNotFoundError: No module named 'importlib.metadata'
    # 找不到包，一般是环境变量错误，先确定是否正常启动conda环境，没有重复启动 conda activate kneaddata。已启动检测环境变量
    echo $PATH
    # /public/software/env01/bin:/public/home/liuyongxin/miniconda3/envs/kneaddata/bin:/public/home/liuyongxin/miniconda3/condabin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin
    # 确认conda环境是否为第一个路径，此处kneaddata路径前还有更高优先级的目录在前，重设PATH变量，即删除当前conda环境前的所有路径
    PATH=/public/home/liuyongxin/miniconda3/envs/kneaddata/bin:/public/home/liuyongxin/miniconda3/condabin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin

## HUMANN物种功能定量

    # **metaphlan_to_stamp.pl**
    # 如果出现下面的错误：
    # bash: /db/EasyMicrobiome/script/metaphlan_to_stamp.pl: /usr/bin/perl^M: 解释器错误: 没有那个文件或目录
    # sed -i 's/\r//' ${db}/EasyMicrobiome/script/*.pl  


    # **HUMAnN2减少输入文件加速**
    # HUMAnN2是计算非常耗时的步骤，如果上百个10G+的样本，有时需要几周至几月的分析。以下介绍两种快速完成分析，而且结果变化不大的方法。替换下面for循环为原文中的“双端合并为单个文件”部分代码
    # 方法1. 软件分析不考虑双端信息，只用一端可获得相近结果，且速度提高1倍。链接质控结果左端高质量至合并目录
    for i in `tail -n+2 result/metadata.txt|cut -f1`;do 
      ln -sf `pwd`/temp/hr/${i}_1.fastq temp/concat/${i}.fq
    done

    # 方法2. 控制标准样比对时间。测序数据量通常为6~50G，同一样本分析时间可达10h~100h，严重浪费时间而浪费硬盘空间。
    # 可用head对单端分析截取20M序列，即3G，则为80M行
    for i in `tail -n+2 result/metadata.txt|cut -f1`;do 
       head -n80000000 temp/qc/${i}_1_kneaddata_paired_1.fastq  > temp/concat/${i}.fq
    done

    # **metaphlan2-共有或特有物种网络图**
    awk 'BEGIN{OFS=FS="\t"}{if(FNR==1) {for(i=9;i<=NF;i++) a[i]=$i; print "Tax\tGroup"} \
       else {for(i=9;i<=NF;i++) if($i>0.05) print "Tax_"FNR, a[i];}}' \
       result/metaphlan2/taxonomy.spf > result/metaphlan2/taxonomy_highabundance.tsv
    awk 'BEGIN{OFS=FS="\t"}{if(FNR==1) {print "Tax\tGrpcombine";} else a[$1]=a[$1]==""?$2:a[$1]$2;}END{for(i in a) print i,a[i]}' \
       result/metaphlan2/taxonomy_highabundance.tsv > result/metaphlan2/taxonomy_group.tsv
    cut -f 2 result/metaphlan2/taxonomy_group.tsv | tail -n +2 | sort -u >group
    for i in `cat group`; do printf "#%02x%02x%02x\n" $((RANDOM%256)) $((RANDOM%256)) $((RANDOM%256)); done >colorcode
    paste group colorcode >group_colorcode
    awk 'BEGIN{OFS=FS="\t"}ARGIND==1{a[$1]=$2;}ARGIND==2{if(FNR==1) {print $0, "Grpcombinecolor"} else print $0,a[$2]}' \
       group_colorcode result/metaphlan2/taxonomy_group.tsv > result/metaphlan2/taxonomy_group2.tsv
    awk 'BEGIN{OFS=FS="\t"}{if(FNR==1) {print "Tax",$1,$2,$3,$4, $5, $6, $7, $8 } else print "Tax_"FNR, $1,$2,$3,$4, $5,$6, $7, $8}' \
       result/metaphlan2/taxonomy.spf > result/metaphlan2/taxonomy_anno.tsv

## LEfSe生物标志鉴定

    # **lefse-plot_cladogram.py：Unknown property axis_bgcolor**
    # 若出现错误 Unknown property axis\_bgcolor，则修改`lefse-plot_cladogram.py`里的`ax_bgcolor`替换成`facecolor`即可。
    # 查看脚本位置，然后使用RStudio或Vim修改
    type lefse-plot_cladogram.py

## Kraken2物种分类

    # **合并样本为表格combine_mpa.py**
    # krakentools中combine_mpa.py，需手动安装脚本，且结果还需调整样本名
    combine_mpa.py \
      -i `tail -n+2 result/metadata.txt|cut -f1|sed 's/^/temp\/kraken2\//;s/$/.mpa/'|tr '\n' ' '` \
      -o temp/kraken2/combined_mpa

    # **序列筛选/去宿主extract_kraken_reads.py**
    # 提取非植物33090和动物(人)33208序列、选择细菌2和古菌2157
    mkdir -p temp/kraken2_qc
    parallel -j 3 \
      "/db/script/extract_kraken_reads.py \
      -k temp/kraken2/{1}.output \
      -r temp/kraken2/{1}.report \
      -1 temp/qc/{1}_1_kneaddata_paired_1.fastq \
      -2 temp/qc/{1}_1_kneaddata_paired_2.fastq \
      -t 33090 33208 --include-children --exclude \
      --max 20000000 --fastq-output \
      -o temp/kraken2_qc/{1}_1.fq \
      -o2 temp/kraken2_qc/{1}_2.fq" \
      ::: `tail -n+2 result/metadata.txt|cut -f1`

## 2nd+3rd assemble (二三代混合组装)

    # **metaspades**
    # 3G数据，耗时3h
    i=SampleA
    time metaspades.py -t 48 -m 500 \
      -1 seq/${i}_1.fastq -2 seq/${i}L_2.fastq \
      --nanopore seq/${i}.fastq \
      -o temp/metaspades_${i}

    # **二三代混合组装OPERA-MS**
    # 结果卡在第9步polishing，可添加--no-polishing参数跳过此步；短序列只支持成对文件，多个文件需要cat合并
    perl ../OPERA-MS.pl \
        --short-read1 R1.fastq.gz \
        --short-read2 R2.fastq.gz \
        --long-read long_read.fastq \
        --no-ref-clustering \
        --num-processors 32 \
        --out-dir RESULTS

    # **二代组装+三代优化**
    perl ~/soft/OPERA-MS/OPERA-MS.pl \
        --contig-file temp/megahit/final.contigs.fa \
        --short-read1 R1.fastq.gz \
        --short-read2 R2.fastq.gz \
        --long-read long_read.fastq \
        --num-processors 32 \
        --no-ref-clustering \
        --no-strain-clustering \
        --no-polishing \
        --out-dir temp/opera

## prodigal gene prediction (基因预测)

    # **序列拆分并行预测基因**
    # (可选)以上注释大约1小时完成1M个基因的预测。加速可将contigs拆分，并行基因预测后再合并。
    # 拆分contigs，按1M条每个文件
    n=10000
    seqkit split result/megahit/final.contigs.fa -s $n
    # 生成拆分文件序列列表
    ls result/megahit/final.contigs.fa.split/final.contigs.part_*.fa|cut -f 2 -d '_'|cut -f 1 -d '.' \
      > temp/split.list
    # 9线程并行基因预测，此步只用单线程且读写强度不大
    time parallel -j 9 \
      "prodigal -i result/megahit/final.contigs.fa.split/final.contigs.part_{}.fa \
      -d temp/gene{}.fa  \
      -o temp/gene{}.gff -p meta -f gff \
      > temp/gene{}.log 2>&1 " \
      ::: `cat temp/split.list`
    # 合并预测基因和gff注释文件
    cat temp/gene*.fa > temp/prodigal/gene.fa
    cat temp/gene*.gff > temp/prodigal/gene.gff

## cd-hit gene redundancy (基因去冗余)

    # **两批基因合并cd-hit-est-2d**
    # cd-hit-est-2d 两批次构建非冗余基因集
    # A和B基因集，分别有M和N个非冗余基因，两批数据合并后用cd-hit-est去冗余，计算量是(M + N) X (M + N -1)
    # cd-hit-est-2d比较，只有M X N的计算量

    # 计算B中特有的基因
    cd-hit-est-2d -i A.fa -i2 B.fa -o B.uni.fa \
        -aS 0.9 -c 0.95 -G 0 -g 0 \
        -T 96 -M 0 -d 0
    # 合并为非冗余基因集
    cat A.fa B.uni.fa > NR.fa

    # **cd-hit合并多批基因salmon索引时提示ID重复**
    # [error] In FixFasta, two references with the same name but different sequences: k141_2390219_1. We require that all input records have a unique name up to the first whitespace (or user-provided separator) character.
    # 错误解决
    mv temp/NRgene/gene.fa temp/NRgene/gene.fa.bak
    # 15G,2m,4G
    seqkit rename temp/NRgene/gene.fa.bak -o temp/NRgene/gene.fa

## salmon gene quantify (基因定量)

    # **找不到库文件liblzma.so.0**
    # 
    # *   报错信息：error while loading shared libraries: liblzma.so.0
    # *   问题描述：直接运行salmon报告，显示找不到lib库，
    # *   解决方法：可使用程序完整路径解决问题，`alias salmon="${soft}/envs/metagenome_env/share/salmon/bin/salmon"`

## Gene annotation database (基因功能数据库)

    # **综合功能注释KEGG描述整理**
    # 脚本位于 /db/script 目录，<https://www.kegg.jp/kegg-bin/show_brite?ko00001.keg> 下载htext，即为最新输入文件 ko00001.keg
    kegg_ko00001_htext2tsv.pl -i ko00001.keg -o ko00001.tsv
    # 原核蛋白数据库(需付费购买)建索引
    conda activate eggnog
    diamond --version # 2.0.13
    cd genes/fasta
    # 3m, 2G
    memusg -t diamond makedb --in prokaryotes.pep.gz \
      --db prokaryotes.pep
    
    # ID转换为KO
    zless prokaryotes.dat.gz|cut -f1,2|sed "1i Kgene\tKO">prokaryotes.gene2KO

    # **抗生素抗性ResFam**
    # 数据库：<http://www.dantaslab.org/resfams>
    # 参考文献：<http://doi.org/10.1038/ismej.2014.106>
    mkdir -p temp/resfam result/resfam
    # 比对至抗生素数据库 1m
    time diamond blastp \
      --db ${db}/resfam/Resfams-proteins \
      --query result/NR/protein.fa \
      --threads 9 --outfmt 6 --sensitive \
      -e 1e-5 --max-target-seqs 1 --quiet \
      --out temp/resfam/gene_diamond.f6
    # 提取基因对应抗性基因列表
    cut -f 1,2 temp/resfam/gene_diamond.f6 | uniq | \
      sed '1 i Name\tResGeneID' > temp/resfam/gene_fam.list
    # 统计注释基因的比例, 488/19182=2.5%
    wc -l temp/resfam/gene_fam.list  result/salmon/gene.count 
    # 按列表累计丰度
    summarizeAbundance.py \
      -i result/salmon/gene.TPM \
      -m temp/resfam/gene_fam.list \
      -c 2 -s ',' -n raw \
      -o result/resfam/TPM
    # 结果中添加FAM注释，spf格式用于stamp分析
    awk 'BEGIN{FS=OFS="\t"} NR==FNR{a[$1]=$4"\t"$3"\t"$2} NR>FNR{print a[$1],$0}' \
      ${db}/resfam/Resfams-proteins_class.tsv  result/resfam/TPM.ResGeneID.raw.txt \
      > result/resfam/TPM.ResGeneID.raw.spf

## MetaWRAP binning (分箱)

    # **MetaWRAP分箱注释Bin classify & annotate**
    # Taxator-tk对每条contig物种注释，再估计bin整体的物种，11m (用时66 min)
    metawrap classify_bins -b temp/bin_refinement/metawrap_50_10_bins \
      -o temp/bin_classify -t 3 &
    # 注释结果见`temp/bin_classify/bin_taxonomy.tab`

    # export LD_LIBRARY_PATH=/conda2/envs/metagenome_env/lib/:${LD_LIBRARY_PATH}
     # 这是动态链接库找不到时的一个简单的应急策略
    #ln -s /conda2/envs/metagenome_env/lib/libssl.so.1.0.0 .
    #ln -s /conda2/envs/metagenome_env/lib/libcrypto.so.1.0.0 .

    # 基于prokka基因注释，4m
    metaWRAP annotate_bins -o temp/bin_annotate \
      -b temp/bin_refinement/metawrap_50_10_bins  -t 5
    # 每个bin基因注释的gff文件bin_funct_annotations, 
    # 核酸ffn文件bin_untranslated_genes，
    # 蛋白faa文件bin_translated_genes
    ls -sh temp/bin_annotate/prokka_out/bin.1/

    # **分箱定量Bin quantify**
    # 耗时3m，系统用时10m，此处可设置线程，但salmon仍调用全部资源
    metawrap quant_bins -b temp/bin_refinement/metawrap_50_10_bins -t 4 \
      -o temp/bin_quant -a temp/megahit/final.contigs.fa temp/qc/ERR*.fastq
    # 文件名字改变
    # 结果包括bin丰度热图`temp/bin_quant/bin_abundance_heatmap.png`
    # 原始数据位于`temp/bin_quant/bin_abundance_table.tab`
    ls -l temp/bin_quant/bin_abundance_heatmap.png

    # **GTDB的文件名不存在错**
    # ERROR: ['BMN5'] are not present in the input list of genome to process，但并无此菌，可能是名称 中存在"-"或"."，替换为i
    # 修改metadata
    sed 's/-/i/;s/\./i/' result/metadatab.txt > result/metadata.txt
    # 修改文件名
    awk 'BEGIN{OFS=FS="\t"}{system("mv temp/antismash/"$1".fna temp/antismash/"$2".fna")ll }' <(paste result/metadatab.txt result/metadata.txt|tail -n+2)