pingu_98/mpt-7B-inference
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update inference.py

Browse source 
Changed model to 7B
This commit is contained in:
James Devine 2023-08-29 13:11:42 +02:00 committed by GitHub
parent 9bb139be8b
commit e825edd38d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 3 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
inference.py
View file

@ -18,7 +18,7 @@ class GenerationConfig:
def format_prompt(system_prompt: str, user_prompt: str):
"""format prompt based on: https://huggingface.co/spaces/mosaicml/mpt-30b-chat/blob/main/app.py"""
"""format prompt based on: https://huggingface.co/mosaicml/mpt-7b-chat/tree/main"""
system_prompt = f"<|im_start|>system\n{system_prompt}<|im_end|>\n"
user_prompt = f"<|im_start|>user\n{user_prompt}<|im_end|>\n"
@ -45,9 +45,9 @@ def generate(
if __name__ == "__main__":
config = AutoConfig.from_pretrained("mosaicml/mpt-30b-chat", context_length=8192)
config = AutoConfig.from_pretrained("mosaicml/mpt-7b-chat", context_length=8192)
llm = AutoModelForCausalLM.from_pretrained(
os.path.abspath("models/mpt-30b-chat.ggmlv0.q4_1.bin"),
os.path.abspath("models/mpt-7b-chat.ggmlv0.q4_1.bin"),
model_type="mpt",
config=config,
)