@@ -95,7 +95,7 @@ Are you on a Windows machine with GPU?
9595
9696 ``` cmd
9797 curl https://raw.githubusercontent.com/microsoft/onnxruntime-genai/main/examples/python/phi3-qa.py -o phi3-qa.py
98- python phi3-qa.py -m directml\directml-int4-awq-block-128
98+ python phi3-qa.py -m directml\directml-int4-awq-block-128 -e dml
9999 ```
100100
101101 Once the script has loaded the model, it will ask you for input in a loop, streaming the output as it is produced the model. For example:
@@ -135,7 +135,7 @@ Are you on a Windows machine with GPU?
135135
136136 ``` bash
137137 curl https://raw.githubusercontent.com/microsoft/onnxruntime-genai/main/examples/python/phi3-qa.py -o phi3-qa.py
138- python phi3-qa.py -m cuda/cuda-int4-rtn-block-32
138+ python phi3-qa.py -m cuda/cuda-int4-rtn-block-32 -e cuda
139139 ```
140140
141141 Once the script has loaded the model, it will ask you for input in a loop, streaming the output as it is produced the model. For example:
@@ -169,7 +169,7 @@ Are you on a Windows machine with GPU?
169169
170170 ```bash
171171 curl https://raw.githubusercontent.com/microsoft/onnxruntime-genai/main/examples/python/phi3-qa.py -o phi3-qa.py
172- python phi3-qa.py -m cpu_and_mobile/cpu-int4-rtn-block-32-acc-level-4
172+ python phi3-qa.py -m cpu_and_mobile/cpu-int4-rtn-block-32-acc-level-4 -e cpu
173173 ```
174174
175175 Once the script has loaded the model, it will ask you for input in a loop, streaming the output as it is produced the model. For example:
@@ -188,95 +188,95 @@ Are you on a Windows machine with GPU?
188188
189189```bash
190190huggingface-cli download microsoft/Phi-3-mini-4k-instruct-onnx --include cpu_and_mobile/cpu-int4-rtn-block-32-acc-level-4/* --local-dir .
191- python phi3-qa.py -m cpu_and_mobile/cpu-int4-rtn-block-32-acc-level-4
191+ python phi3-qa.py -m cpu_and_mobile/cpu-int4-rtn-block-32-acc-level-4 -e cpu
192192```
193193
194194### Phi-3 mini 4k context CUDA
195195
196196```bash
197197huggingface-cli download microsoft/Phi-3-mini-4k-instruct-onnx --include cuda/cuda-int4-rtn-block-32/* --local-dir .
198- python phi3-qa.py -m cuda/cuda-int4-rtn-block-32
198+ python phi3-qa.py -m cuda/cuda-int4-rtn-block-32 -e cuda
199199```
200200
201201### Phi-3 mini 4k context DirectML
202202
203203```bash
204204huggingface-cli download microsoft/Phi-3-mini-4k-instruct-onnx --include directml/* --local-dir .
205- python phi3-qa.py -m directml\directml-int4-awq-block-128
205+ python phi3-qa.py -m directml\directml-int4-awq-block-128 -e dml
206206```
207207
208208### Phi-3 mini 128k context CPU
209209
210210```bash
211211huggingface-cli download microsoft/Phi-3-mini-128k-instruct-onnx --include cpu_and_mobile/cpu-int4-rtn-block-32-acc-level-4/* --local-dir .
212- python phi3-qa.py -m cpu_and_mobile/cpu-int4-rtn-block-32-acc-level-4
212+ python phi3-qa.py -m cpu_and_mobile/cpu-int4-rtn-block-32-acc-level-4 -e cpu
213213```
214214
215215### Phi-3 mini 128k context CUDA
216216
217217```bash
218218huggingface-cli download microsoft/Phi-3-mini-128k-instruct-onnx --include cuda/cuda-int4-rtn-block-32/* --local-dir .
219- python phi3-qa.py -m cuda/cuda-int4-rtn-block-32
219+ python phi3-qa.py -m cuda/cuda-int4-rtn-block-32 -e cuda
220220```
221221
222222### Phi-3 mini 128k context DirectML
223223
224224```bash
225225huggingface-cli download microsoft/Phi-3-mini-128k-instruct-onnx --include directml/* --local-dir .
226- python phi3-qa.py -m directml\directml-int4-awq-block-128
226+ python phi3-qa.py -m directml\directml-int4-awq-block-128 -e dml
227227```
228228
229229### Phi-3 medium 4k context CPU
230230
231231```bash
232232git clone https://huggingface.co/microsoft/Phi-3-medium-4k-instruct-onnx-cpu
233- python phi3-qa.py -m Phi-3-medium-4k-instruct-onnx-cpu/cpu-int4-rtn-block-32-acc-level-4
233+ python phi3-qa.py -m Phi-3-medium-4k-instruct-onnx-cpu/cpu-int4-rtn-block-32-acc-level-4 -e cpu
234234```
235235
236236### Phi-3 medium 4k context CUDA
237237
238238```bash
239239git clone https://huggingface.co/microsoft/Phi-3-medium-4k-instruct-onnx-cuda
240- python phi3-qa.py -m Phi-3-medium-4k-instruct-onnx-cuda/cuda-int4-rtn-block-32
240+ python phi3-qa.py -m Phi-3-medium-4k-instruct-onnx-cuda/cuda-int4-rtn-block-32 -e cuda
241241```
242242
243243### Phi-3 medium 4k context DirectML
244244
245245```bash
246246git clone https://huggingface.co/microsoft/Phi-3-medium-4k-instruct-onnx-directml
247- python phi3-qa.py -m Phi-3-medium-4k-instruct-onnx-directml/directml-int4-awq-block-128
247+ python phi3-qa.py -m Phi-3-medium-4k-instruct-onnx-directml/directml-int4-awq-block-128 -e dml
248248```
249249
250250### Phi-3 medium 128k context CPU
251251
252252```bash
253253git clone https://huggingface.co/microsoft/Phi-3-medium-128k-instruct-onnx-cpu
254- python phi3-qa.py -m Phi-3-medium-128k-instruct-onnx-cpu/cpu-int4-rtn-block-32-acc-level-4
254+ python phi3-qa.py -m Phi-3-medium-128k-instruct-onnx-cpu/cpu-int4-rtn-block-32-acc-level-4 -e cpu
255255```
256256
257257### Phi-3 medium 128k context CUDA
258258
259259```bash
260260git clone https://huggingface.co/microsoft/Phi-3-medium-128k-instruct-onnx-cuda
261- python phi3-qa.py -m Phi-3-medium-128k-instruct-onnx-cuda/cuda-int4-rtn-block-32
261+ python phi3-qa.py -m Phi-3-medium-128k-instruct-onnx-cuda/cuda-int4-rtn-block-32 -e cuda
262262```
263263
264264### Phi-3 medium 128k context DirectML
265265
266266```bash
267267git clone https://huggingface.co/microsoft/Phi-3-medium-128k-instruct-onnx-directml
268- python phi3-qa.py -m Phi-3-medium-128k-instruct-onnx-directml/directml-int4-awq-block-128
268+ python phi3-qa.py -m Phi-3-medium-128k-instruct-onnx-directml/directml-int4-awq-block-128 -e dml
269269```
270270
271271### Phi-3.5 mini 128k context CUDA
272272```bash
273273huggingface-cli download microsoft/Phi-3.5-mini-instruct-onnx --include cuda/cuda-int4-awq-block-128/* --local-dir .
274- python phi3-qa.py -m cuda/cuda-int4-awq-block-128
274+ python phi3-qa.py -m cuda/cuda-int4-awq-block-128 -e cuda
275275```
276276
277277### Phi-3.5 mini 128k context CPU
278278
279279```bash
280280huggingface-cli download microsoft/Phi-3.5-mini-instruct-onnx --include cpu_and_mobile/cpu-int4-awq-block-128-acc-level-4/* --local-dir .
281- python phi3-qa.py -m cpu_and_mobile/cpu-int4-awq-block-128-acc-level-4
281+ python phi3-qa.py -m cpu_and_mobile/cpu-int4-awq-block-128-acc-level-4 -e cpu
282282```
0 commit comments