forte-wrappers/.github/workflows/main.yml at 0dc65f96158e62f83cfd15b37e9316433d1da078 · asyml/forte-wrappers · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
name: Python Build

on:
  push:
  pull_request:
    branches: [ main ]
  repository_dispatch:
    types: [ trigger-forte-wrappers ]

  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:

jobs:
  build:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: [ 3.6, 3.7 ]
        torch-version: [ 1.5.0 ]
        tensorflow-version: [ 1.15.0 ]

    steps:
      - uses: actions/checkout@v2
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python-version }}
      - name: Cache pip
        uses: actions/cache@v2
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
          restore-keys: |
            ${{ runner.os }}-pip-
            ${{ runner.os }}-
      - name: Install dependencies
        run: |
          python -m pip install --progress-bar off --upgrade pip
          pip install --progress-bar off Django django-guardian
          pip install --progress-bar off pylint==2.6.0 flake8==3.8.2 mypy==0.790 pytest==5.1.3 black==20.8b1
          pip install --progress-bar off coverage codecov ddt
      - name: Install deep learning frameworks
        run: |
          pip install --progress-bar off torch==${{ matrix.torch-version }}
          pip install --progress-bar off tensorflow==${{ matrix.tensorflow-version }}
      - name: Install Texar
        run: |
          git clone https://github.com/asyml/texar-pytorch.git
          cd texar-pytorch
          pip install --progress-bar off .
          cd ..
          # Remove them to avoid confusing pytest.
          rm -rf texar-pytorch
      - name: Lint with flake8
        run: |
          # stop the build if there are Python syntax errors or undefined names
          flake8 src/ --count --select=E9,F63,F7,F82 --show-source --statistics
          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
          flake8 src/ --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
      - name: Lint with pylint
        run: |
          pylint src/
      - name: Format check with Black
        run: |
          black --line-length 80 --check src/
      - name: Lint with mypy
        run: |
          for dep in allennlp elastic faiss gpt2 huggingface nltk spacy stanza tweepy vader
            do
              mypy "src/"$dep
            done

  pytest:
    needs: [ build ]
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: [ 3.6, 3.7 ]
        torch-version: [ 1.5.0 ]
        tensorflow-version: [ 1.15.0 ]
        test-details:
          - { dep: "nltk allennlp", testfile: tests/wrappers/allennlp_processors_test.py }
          - { dep: nltk, testfile: tests/wrappers/nltk_processors_test.py }
          - { dep: "vader nltk ", testfile: tests/wrappers/sentiment_processor_test.py }
          - { dep: spacy, testfile: tests/wrappers/spacy_processors_test.py }
          - { dep: stanza, testfile: tests/wrappers/stanfordnlp_processor_test.py }
          - { dep: "nltk gpt2", extra: "termcolor>=1.1.0", testfile: examples/gpt2_test.py }
          - { dep: elastic, testfile: tests/wrappers/elastic }
          - { dep: faiss, testfile: tests/wrappers/faiss_indexers_test.py }
          - { dep: "huggingface nltk", testfile: tests/wrappers/huggingface }
          - { dep: tweepy, testfile: tests/wrappers/twitter_processor_test.py }
    steps:
      - uses: actions/checkout@v2
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python-version }}
      - name: Cache pip
        uses: actions/cache@v2
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
          restore-keys: |
            ${{ runner.os }}-pip-
            ${{ runner.os }}-
      - name: Install dependencies
        run: |
          python -m pip install --progress-bar off --upgrade pip
          pip install --progress-bar off pytest==5.1.3
          pip install --progress-bar off coverage ddt
      - name: Install deep learning frameworks
        run: |
          pip install --progress-bar off torch==${{ matrix.torch-version }}
      - name: Install the project dependencies
        run: |
          # Install Wrappers
          for d in ${{ matrix.test-details.dep }}; do pip install "src/"$d; done
          # Install Extras
          if [ ! -z ${{ matrix.test-details.extra }} ]
          then
              pip install ${{ matrix.test-details.extra }}
          fi
      - name: Start elastic server if test elastic search
        if: ${{ matrix.test-details.dep == 'elastic' }}
        run: |
          wget -q https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.4.2-linux-x86_64.tar.gz
          tar -zxf elasticsearch-7.4.2-linux-x86_64.tar.gz
          rm -rf elasticsearch-7.4.2-linux-x86_64.tar.gz
          elasticsearch-7.4.2/bin/elasticsearch -d
          chmod +x tests/wait_elastic.sh
          tests/wait_elastic.sh
      - name: Test with pytest and run coverage
        run: |
          coverage run -m pytest ${{ matrix.test-details.testfile}}

  docs:
    needs: [ build, pytest ]
    runs-on: ubuntu-latest
    env:
      python-version: 3.7
    steps:
      - uses: actions/checkout@v2
      - name: Set up Python ${{ env.python-version }}
        uses: actions/setup-python@v2
        with:
          python-version: ${{ env.python-version }}
      - name: Cache pip
        uses: actions/cache@v2
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
          restore-keys: |
            ${{ runner.os }}-pip-
            ${{ runner.os }}-
      - name: Install dependencies
        run: |
          python -m pip install --progress-bar off --upgrade pip
          pip install --progress-bar off -r docs/requirements.txt
          # Install Wrappers
          for dep in allennlp elastic faiss gpt2 huggingface nltk spacy stanza tweepy vader
            do
              pip install "src/"$dep
            done
      - name: Build Docs
        run: |
          cd docs
          sphinx-build -W -b html -d _build/doctrees . _build/html
          sphinx-build -W -b spelling -d _build/doctrees . _build/spelling
          cd ..

  dispatch:
    needs: [ build, pytest, docs ]
    runs-on: ubuntu-latest
    if: github.repository == 'asyml/forte-wrappers' && github.ref == 'refs/heads/main'
    steps:
      - name: Repository Dispatch
        uses: peter-evans/repository-dispatch@v1
        with:
          token: ${{ secrets.REPO_DISPATCH_PAT_HECTOR }}
          repository: petuum/composing_information_system
          event-type: trigger-composable-workflow