pingu_98/mpt-7B-inference
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

remove hard coded thread count

Browse source
This commit is contained in:
Anton Bacaj 2023-06-26 05:51:40 +00:00
parent b6705b3762
commit 123531059e
1 changed files with 2 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
inference.py
View file

@ -1,3 +1,4 @@
import os
from ctransformers import AutoModelForCausalLM, AutoConfig from ctransformers import AutoModelForCausalLM, AutoConfig
@ -31,7 +32,7 @@ def generate(llm, system_prompt, user_prompt):
seed=42, seed=42,
reset=True, # reset history (cache) reset=True, # reset history (cache)
stream=True, # streaming per word/token stream=True, # streaming per word/token
threads=24, # adjust for your CPU threads=os.cpu_count() / 2, # adjust for your CPU
stop=["<|im_end|>", "|<"], stop=["<|im_end|>", "|<"],
) )