diff --git a/benchmark/README.md b/benchmark/README.md index d10880b..3577883 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -12,4 +12,22 @@ cd SentEval/data/downstream 2. Run test script ```bash python transformers_test.py MODEL_NAME -``` \ No newline at end of file +``` + +## QPS Test + +Please note that `qps_test.py` uses: +- `localhost:8000`: to connect triton client +- `'Hello, world.''`: as test sentence + +```bash +python qps_test --model paraphrase-albert-small-v2 --pipe --onnx --triton --num 100 +``` + +**Args:** +- `--model`: mandatory, string, model name +- `--pipe`: optional, on/off flag to enable qps test for pipe +- `--onnx`: optional, on/off flag to enable qps test for onnx +- `--triton`: optional, on/off flag to enable qps for triton (please make sure that triton client is ready) +- `--num`: optional, integer, defaults to 100, batch size in each loop (10 loops in total) +- `--device`: optional, int, defaults to -1, cuda index or use cpu when -1 \ No newline at end of file diff --git a/benchmark/qps_test.py b/benchmark/qps_test.py index 2221b27..1c326ee 100644 --- a/benchmark/qps_test.py +++ b/benchmark/qps_test.py @@ -65,7 +65,7 @@ if args.num and args.pipe: print('Pipe qps:', mean(qps)) if args.triton: - client = triton_client.Client(url='172.16.70.4:8101') + client = triton_client.Client(url='localhost:8000') out2 = client(text)[0][0] print('Triton: OK') diff --git a/benchmark/test_client.py b/benchmark/test_client.py index bb6aa3b..7ca35e2 100644 --- a/benchmark/test_client.py +++ b/benchmark/test_client.py @@ -3,7 +3,7 @@ import sys import time num = int(sys.argv[-1]) -client = triton_client.Client(url='172.16.70.4:8101') +client = triton_client.Client(url='localhost:8000') data = 'hello' # data = 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.'