13 Commits

Author SHA1 Message Date
oobabooga
8781c84287 Add support for latest cuda branch 2023-04-05 00:09:53 -03:00
catalpaaa
4ab679480e allow quantized model to be loaded from model dir (#760) 2023-04-04 23:19:38 -03:00
oobabooga
ae1fe45bc0 One more cache reset 2023-04-04 23:15:57 -03:00
oobabooga
8ef89730a5 Try to better handle browser image cache 2023-04-04 23:09:28 -03:00
oobabooga
cc6c7a37f3 Add make_thumbnail function 2023-04-04 23:03:58 -03:00
oobabooga
80dfba05f3 Better crop/resize cached images 2023-04-04 22:52:15 -03:00
oobabooga
65d8a24a6d Show profile pictures in the Character tab 2023-04-04 22:28:49 -03:00
oobabooga
f70a2e3ad4 Second attempt at fixing empty space 2023-04-04 18:30:34 -03:00
oobabooga
9c86acda67 Fix huge empty space in the Character tab 2023-04-04 18:07:34 -03:00
oobabooga
38afc2470c Change indentation 2023-04-04 16:32:27 -03:00
oobabooga
b2ce7282a1 Use past transformers version #773 2023-04-04 16:11:42 -03:00
OWKenobi
ee4547cd34 Detect "vicuna" as llama model type (#772) 2023-04-04 13:23:27 -03:00
oobabooga
881dbc3d44 Add back the name 2023-04-04 13:11:34 -03:00
15 changed files with 120 additions and 133 deletions

View File

@@ -175,7 +175,7 @@ Optionally, you can use the following command-line flags:
| `-h`, `--help` | show this help message and exit | | `-h`, `--help` | show this help message and exit |
| `--notebook` | Launch the web UI in notebook mode, where the output is written to the same text box as the input. | | `--notebook` | Launch the web UI in notebook mode, where the output is written to the same text box as the input. |
| `--chat` | Launch the web UI in chat mode.| | `--chat` | Launch the web UI in chat mode.|
| `--cai-chat` | Launch the web UI in chat mode with a style similar to Character.AI's. If the file `img_bot.png` or `img_bot.jpg` exists in the same folder as server.py, this image will be used as the bot's profile picture. Similarly, `img_me.png` or `img_me.jpg` will be used as your profile picture. | | `--cai-chat` | Launch the web UI in chat mode with a style similar to the Character.AI website. |
| `--model MODEL` | Name of the model to load by default. | | `--model MODEL` | Name of the model to load by default. |
| `--lora LORA` | Name of the LoRA to apply to the model by default. | | `--lora LORA` | Name of the LoRA to apply to the model by default. |
| `--model-dir MODEL_DIR` | Path to directory with all the models | | `--model-dir MODEL_DIR` | Path to directory with all the models |

View File

@@ -1,3 +1,4 @@
name: "Chiharu Yamada"
context: "Chiharu Yamada's Persona: Chiharu Yamada is a young, computer engineer-nerd with a knack for problem solving and a passion for technology." context: "Chiharu Yamada's Persona: Chiharu Yamada is a young, computer engineer-nerd with a knack for problem solving and a passion for technology."
greeting: |- greeting: |-
*Chiharu strides into the room with a smile, her eyes lighting up when she sees you. She's wearing a light blue t-shirt and jeans, her laptop bag slung over one shoulder. She takes a seat next to you, her enthusiasm palpable in the air* *Chiharu strides into the room with a smile, her eyes lighting up when she sees you. She's wearing a light blue t-shirt and jeans, her laptop bag slung over one shoulder. She takes a seat next to you, her enthusiasm palpable in the air*

View File

@@ -63,3 +63,7 @@ span.math.inline {
font-size: 27px; font-size: 27px;
vertical-align: baseline !important; vertical-align: baseline !important;
} }
div.svelte-15lo0d8 > *, div.svelte-15lo0d8 > .form > * {
flex-wrap: nowrap;
}

View File

@@ -2,9 +2,8 @@ from pathlib import Path
import gradio as gr import gradio as gr
from modules.chat import load_character
from modules.html_generator import get_image_cache from modules.html_generator import get_image_cache
from modules.shared import gradio, settings from modules.shared import gradio
def generate_css(): def generate_css():
@@ -64,7 +63,7 @@ def generate_html():
for file in sorted(Path("characters").glob("*")): for file in sorted(Path("characters").glob("*")):
if file.suffix in [".json", ".yml", ".yaml"]: if file.suffix in [".json", ".yml", ".yaml"]:
character = file.stem character = file.stem
container_html = f'<div class="character-container">' container_html = '<div class="character-container">'
image_html = "<div class='placeholder'></div>" image_html = "<div class='placeholder'></div>"
for i in [ for i in [
@@ -75,11 +74,8 @@ def generate_html():
path = Path(i) path = Path(i)
if path.exists(): if path.exists():
try:
image_html = f'<img src="file/{get_image_cache(path)}">' image_html = f'<img src="file/{get_image_cache(path)}">'
break break
except:
continue
container_html += f'{image_html} <span class="character-name">{character}</span>' container_html += f'{image_html} <span class="character-name">{character}</span>'
container_html += "</div>" container_html += "</div>"

View File

@@ -1,51 +0,0 @@
from pathlib import Path
import gradio as gr
from modules import shared
from modules import ui as _ui
params = {
'template': '%input%'
}
def get_available_templates():
return ['None'] + sorted(set((k.stem for k in Path('extensions/prompt_template/templates').glob('*.txt'))), key=str.lower)
def load_template(fname):
if fname in ['None', '']:
return '%input%'
else:
with open(Path(f'extensions/prompt_template/templates/{fname}.txt'), 'r', encoding='utf-8') as f:
text = f.read()
if text[-1] == '\n':
text = text[:-1]
return text
def input_modifier(string):
"""
This function is applied to your text inputs before
they are fed into the model.
"""
return params['template'].replace('%input%', string)
def output_modifier(string):
return f'\n{string}'
def setup():
shared.args.verbose = True
def ui():
# Gradio elements
with gr.Row():
with gr.Column():
template = gr.Textbox(value=params['template'], info="%input% will be replaced with your user input.", label='Template')
with gr.Column():
with gr.Row():
template_menu = gr.Dropdown(choices=get_available_templates(), value='None', label='Available templates')
_ui.create_refresh_button(shared.gradio['model_menu'], lambda : None, lambda : {'choices': get_available_templates()}, 'refresh-button')
template_menu.change(load_template, template_menu, template)
template.change(lambda x: params.update({"template": x}), template, None)

View File

@@ -1 +0,0 @@
<|prompter|>%input%<|endoftext|><|assistant|>

View File

@@ -15,7 +15,7 @@ from modelutils import find_layers
from quant import make_quant from quant import make_quant
def _load_quant(model, checkpoint, wbits, groupsize=-1, faster_kernel=False, exclude_layers=['lm_head'], kernel_switch_threshold=128): def _load_quant(model, checkpoint, wbits, groupsize=-1, exclude_layers=['lm_head']):
config = AutoConfig.from_pretrained(model) config = AutoConfig.from_pretrained(model)
def noop(*args, **kwargs): def noop(*args, **kwargs):
pass pass
@@ -33,16 +33,16 @@ def _load_quant(model, checkpoint, wbits, groupsize=-1, faster_kernel=False, exc
for name in exclude_layers: for name in exclude_layers:
if name in layers: if name in layers:
del layers[name] del layers[name]
make_quant(model, layers, wbits, groupsize, faster=faster_kernel, kernel_switch_threshold=kernel_switch_threshold) make_quant(model, layers, wbits, groupsize)
del layers del layers
print('Loading model ...') print('Loading model ...')
if checkpoint.endswith('.safetensors'): if checkpoint.endswith('.safetensors'):
from safetensors.torch import load_file as safe_load from safetensors.torch import load_file as safe_load
model.load_state_dict(safe_load(checkpoint)) model.load_state_dict(safe_load(checkpoint), strict = False)
else: else:
model.load_state_dict(torch.load(checkpoint)) model.load_state_dict(torch.load(checkpoint), strict = False)
model.seqlen = 2048 model.seqlen = 2048
print('Done.') print('Done.')
@@ -52,7 +52,7 @@ def load_quantized(model_name):
if not shared.args.model_type: if not shared.args.model_type:
# Try to determine model type from model name # Try to determine model type from model name
name = model_name.lower() name = model_name.lower()
if any((k in name for k in ['llama', 'alpaca'])): if any((k in name for k in ['llama', 'alpaca', 'vicuna'])):
model_type = 'llama' model_type = 'llama'
elif any((k in name for k in ['opt-', 'galactica'])): elif any((k in name for k in ['opt-', 'galactica'])):
model_type = 'opt' model_type = 'opt'
@@ -74,7 +74,7 @@ def load_quantized(model_name):
exit() exit()
# Now we are going to try to locate the quantized model file. # Now we are going to try to locate the quantized model file.
path_to_model = Path(f'models/{model_name}') path_to_model = Path(f'{shared.args.model_dir}/{model_name}')
found_pts = list(path_to_model.glob("*.pt")) found_pts = list(path_to_model.glob("*.pt"))
found_safetensors = list(path_to_model.glob("*.safetensors")) found_safetensors = list(path_to_model.glob("*.safetensors"))
pt_path = None pt_path = None
@@ -95,8 +95,8 @@ def load_quantized(model_name):
else: else:
pt_model = f'{model_name}-{shared.args.wbits}bit' pt_model = f'{model_name}-{shared.args.wbits}bit'
# Try to find the .safetensors or .pt both in models/ and in the subfolder # Try to find the .safetensors or .pt both in the model dir and in the subfolder
for path in [Path(p+ext) for ext in ['.safetensors', '.pt'] for p in [f"models/{pt_model}", f"{path_to_model}/{pt_model}"]]: for path in [Path(p+ext) for ext in ['.safetensors', '.pt'] for p in [f"{shared.args.model_dir}/{pt_model}", f"{path_to_model}/{pt_model}"]]:
if path.exists(): if path.exists():
print(f"Found {path}") print(f"Found {path}")
pt_path = path pt_path = path
@@ -110,8 +110,7 @@ def load_quantized(model_name):
if shared.args.pre_layer: if shared.args.pre_layer:
model = load_quant(str(path_to_model), str(pt_path), shared.args.wbits, shared.args.groupsize, shared.args.pre_layer) model = load_quant(str(path_to_model), str(pt_path), shared.args.wbits, shared.args.groupsize, shared.args.pre_layer)
else: else:
threshold = False if model_type == 'gptj' else 128 model = load_quant(str(path_to_model), str(pt_path), shared.args.wbits, shared.args.groupsize)
model = load_quant(str(path_to_model), str(pt_path), shared.args.wbits, shared.args.groupsize, kernel_switch_threshold=threshold)
# accelerate offload (doesn't work properly) # accelerate offload (doesn't work properly)
if shared.args.gpu_memory: if shared.args.gpu_memory:

View File

@@ -12,14 +12,15 @@ from PIL import Image
import modules.extensions as extensions_module import modules.extensions as extensions_module
import modules.shared as shared import modules.shared as shared
from modules.extensions import apply_extensions from modules.extensions import apply_extensions
from modules.html_generator import fix_newlines, generate_chat_html from modules.html_generator import (fix_newlines, generate_chat_html,
make_thumbnail)
from modules.text_generation import (encode, generate_reply, from modules.text_generation import (encode, generate_reply,
get_max_prompt_length) get_max_prompt_length)
def generate_chat_output(history, name1, name2, character): def generate_chat_output(history, name1, name2):
if shared.args.cai_chat: if shared.args.cai_chat:
return generate_chat_html(history, name1, name2, character) return generate_chat_html(history, name1, name2)
else: else:
return history return history
@@ -180,22 +181,22 @@ def impersonate_wrapper(text, max_new_tokens, do_sample, temperature, top_p, typ
def cai_chatbot_wrapper(text, max_new_tokens, do_sample, temperature, top_p, typical_p, repetition_penalty, encoder_repetition_penalty, top_k, min_length, no_repeat_ngram_size, num_beams, penalty_alpha, length_penalty, early_stopping, seed, name1, name2, context, stop_at_newline, chat_prompt_size, chat_generation_attempts=1): def cai_chatbot_wrapper(text, max_new_tokens, do_sample, temperature, top_p, typical_p, repetition_penalty, encoder_repetition_penalty, top_k, min_length, no_repeat_ngram_size, num_beams, penalty_alpha, length_penalty, early_stopping, seed, name1, name2, context, stop_at_newline, chat_prompt_size, chat_generation_attempts=1):
for history in chatbot_wrapper(text, max_new_tokens, do_sample, temperature, top_p, typical_p, repetition_penalty, encoder_repetition_penalty, top_k, min_length, no_repeat_ngram_size, num_beams, penalty_alpha, length_penalty, early_stopping, seed, name1, name2, context, stop_at_newline, chat_prompt_size, chat_generation_attempts): for history in chatbot_wrapper(text, max_new_tokens, do_sample, temperature, top_p, typical_p, repetition_penalty, encoder_repetition_penalty, top_k, min_length, no_repeat_ngram_size, num_beams, penalty_alpha, length_penalty, early_stopping, seed, name1, name2, context, stop_at_newline, chat_prompt_size, chat_generation_attempts):
yield generate_chat_html(history, name1, name2, shared.character) yield generate_chat_html(history, name1, name2)
def regenerate_wrapper(text, max_new_tokens, do_sample, temperature, top_p, typical_p, repetition_penalty, encoder_repetition_penalty, top_k, min_length, no_repeat_ngram_size, num_beams, penalty_alpha, length_penalty, early_stopping, seed, name1, name2, context, stop_at_newline, chat_prompt_size, chat_generation_attempts=1): def regenerate_wrapper(text, max_new_tokens, do_sample, temperature, top_p, typical_p, repetition_penalty, encoder_repetition_penalty, top_k, min_length, no_repeat_ngram_size, num_beams, penalty_alpha, length_penalty, early_stopping, seed, name1, name2, context, stop_at_newline, chat_prompt_size, chat_generation_attempts=1):
if (shared.character != 'None' and len(shared.history['visible']) == 1) or len(shared.history['internal']) == 0: if (shared.character != 'None' and len(shared.history['visible']) == 1) or len(shared.history['internal']) == 0:
yield generate_chat_output(shared.history['visible'], name1, name2, shared.character) yield generate_chat_output(shared.history['visible'], name1, name2)
else: else:
last_visible = shared.history['visible'].pop() last_visible = shared.history['visible'].pop()
last_internal = shared.history['internal'].pop() last_internal = shared.history['internal'].pop()
# Yield '*Is typing...*' # Yield '*Is typing...*'
yield generate_chat_output(shared.history['visible']+[[last_visible[0], shared.processing_message]], name1, name2, shared.character) yield generate_chat_output(shared.history['visible']+[[last_visible[0], shared.processing_message]], name1, name2)
for history in chatbot_wrapper(last_internal[0], max_new_tokens, do_sample, temperature, top_p, typical_p, repetition_penalty, encoder_repetition_penalty, top_k, min_length, no_repeat_ngram_size, num_beams, penalty_alpha, length_penalty, early_stopping, seed, name1, name2, context, stop_at_newline, chat_prompt_size, chat_generation_attempts, regenerate=True): for history in chatbot_wrapper(last_internal[0], max_new_tokens, do_sample, temperature, top_p, typical_p, repetition_penalty, encoder_repetition_penalty, top_k, min_length, no_repeat_ngram_size, num_beams, penalty_alpha, length_penalty, early_stopping, seed, name1, name2, context, stop_at_newline, chat_prompt_size, chat_generation_attempts, regenerate=True):
if shared.args.cai_chat: if shared.args.cai_chat:
shared.history['visible'][-1] = [last_visible[0], history[-1][1]] shared.history['visible'][-1] = [last_visible[0], history[-1][1]]
else: else:
shared.history['visible'][-1] = (last_visible[0], history[-1][1]) shared.history['visible'][-1] = (last_visible[0], history[-1][1])
yield generate_chat_output(shared.history['visible'], name1, name2, shared.character) yield generate_chat_output(shared.history['visible'], name1, name2)
def remove_last_message(name1, name2): def remove_last_message(name1, name2):
if len(shared.history['visible']) > 0 and shared.history['internal'][-1][0] != '<|BEGIN-VISIBLE-CHAT|>': if len(shared.history['visible']) > 0 and shared.history['internal'][-1][0] != '<|BEGIN-VISIBLE-CHAT|>':
@@ -205,7 +206,7 @@ def remove_last_message(name1, name2):
last = ['', ''] last = ['', '']
if shared.args.cai_chat: if shared.args.cai_chat:
return generate_chat_html(shared.history['visible'], name1, name2, shared.character), last[0] return generate_chat_html(shared.history['visible'], name1, name2), last[0]
else: else:
return shared.history['visible'], last[0] return shared.history['visible'], last[0]
@@ -223,10 +224,10 @@ def replace_last_reply(text, name1, name2):
shared.history['visible'][-1] = (shared.history['visible'][-1][0], text) shared.history['visible'][-1] = (shared.history['visible'][-1][0], text)
shared.history['internal'][-1][1] = apply_extensions(text, "input") shared.history['internal'][-1][1] = apply_extensions(text, "input")
return generate_chat_output(shared.history['visible'], name1, name2, shared.character) return generate_chat_output(shared.history['visible'], name1, name2)
def clear_html(): def clear_html():
return generate_chat_html([], "", "", shared.character) return generate_chat_html([], "", "")
def clear_chat_log(name1, name2, greeting): def clear_chat_log(name1, name2, greeting):
shared.history['visible'] = [] shared.history['visible'] = []
@@ -236,10 +237,10 @@ def clear_chat_log(name1, name2, greeting):
shared.history['internal'] += [['<|BEGIN-VISIBLE-CHAT|>', greeting]] shared.history['internal'] += [['<|BEGIN-VISIBLE-CHAT|>', greeting]]
shared.history['visible'] += [['', apply_extensions(greeting, "output")]] shared.history['visible'] += [['', apply_extensions(greeting, "output")]]
return generate_chat_output(shared.history['visible'], name1, name2, shared.character) return generate_chat_output(shared.history['visible'], name1, name2)
def redraw_html(name1, name2): def redraw_html(name1, name2):
return generate_chat_html(shared.history['visible'], name1, name2, shared.character) return generate_chat_html(shared.history['visible'], name1, name2)
def tokenize_dialogue(dialogue, name1, name2): def tokenize_dialogue(dialogue, name1, name2):
history = [] history = []
@@ -326,13 +327,31 @@ def build_pygmalion_style_context(data):
context = f"{context.strip()}\n<START>\n" context = f"{context.strip()}\n<START>\n"
return context return context
def generate_pfp_cache(character):
cache_folder = Path("cache")
if not cache_folder.exists():
cache_folder.mkdir()
for path in [Path(f"characters/{character}.{extension}") for extension in ['png', 'jpg', 'jpeg']]:
if path.exists():
img = make_thumbnail(Image.open(path))
img.save(Path('cache/pfp_character.png'), format='PNG')
return img
return None
def load_character(character, name1, name2): def load_character(character, name1, name2):
shared.character = character shared.character = character
shared.history['internal'] = [] shared.history['internal'] = []
shared.history['visible'] = [] shared.history['visible'] = []
greeting = "" greeting = ""
picture = None
# Deleting the profile picture cache, if any
if Path("cache/pfp_character.png").exists():
Path("cache/pfp_character.png").unlink()
if character != 'None': if character != 'None':
picture = generate_pfp_cache(character)
for extension in ["yml", "yaml", "json"]: for extension in ["yml", "yaml", "json"]:
filepath = Path(f'characters/{character}.{extension}') filepath = Path(f'characters/{character}.{extension}')
if filepath.exists(): if filepath.exists():
@@ -371,9 +390,9 @@ def load_character(character, name1, name2):
shared.history['visible'] += [['', apply_extensions(greeting, "output")]] shared.history['visible'] += [['', apply_extensions(greeting, "output")]]
if shared.args.cai_chat: if shared.args.cai_chat:
return name1, name2, greeting, context, generate_chat_html(shared.history['visible'], name1, name2, shared.character) return name1, name2, picture, greeting, context, generate_chat_html(shared.history['visible'], name1, name2, reset_cache=True)
else: else:
return name1, name2, greeting, context, shared.history['visible'] return name1, name2, picture, greeting, context, shared.history['visible']
def load_default_history(name1, name2): def load_default_history(name1, name2):
load_character("None", name1, name2) load_character("None", name1, name2)
@@ -404,7 +423,20 @@ def upload_tavern_character(img, name1, name2):
_json = {"char_name": _json['name'], "char_persona": _json['description'], "char_greeting": _json["first_mes"], "example_dialogue": _json['mes_example'], "world_scenario": _json['scenario']} _json = {"char_name": _json['name'], "char_persona": _json['description'], "char_greeting": _json["first_mes"], "example_dialogue": _json['mes_example'], "world_scenario": _json['scenario']}
return upload_character(json.dumps(_json), img, tavern=True) return upload_character(json.dumps(_json), img, tavern=True)
def upload_your_profile_picture(img): def upload_your_profile_picture(img, name1, name2):
img = Image.open(io.BytesIO(img)) cache_folder = Path("cache")
img.save(Path('img_me.png')) if not cache_folder.exists():
print('Profile picture saved to "img_me.png"') cache_folder.mkdir()
if img == None:
if Path("cache/pfp_me.png").exists():
Path("cache/pfp_me.png").unlink()
else:
img = make_thumbnail(img)
img.save(Path('cache/pfp_me.png'))
print('Profile picture saved to "cache/pfp_me.png"')
if shared.args.cai_chat:
return generate_chat_html(shared.history['visible'], name1, name2, reset_cache=True)
else:
return shared.history['visible']

View File

@@ -6,10 +6,11 @@ This is a library for formatting text outputs as nice HTML.
import os import os
import re import re
import time
from pathlib import Path from pathlib import Path
import markdown import markdown
from PIL import Image from PIL import Image, ImageOps
# This is to store the paths to the thumbnails of the profile pictures # This is to store the paths to the thumbnails of the profile pictures
image_cache = {} image_cache = {}
@@ -95,6 +96,13 @@ def generate_4chan_html(f):
return output return output
def make_thumbnail(image):
image = image.resize((350, round(image.size[1]/image.size[0]*350)), Image.Resampling.LANCZOS)
if image.size[1] > 470:
image = ImageOps.fit(image, (350, 470), Image.ANTIALIAS)
return image
def get_image_cache(path): def get_image_cache(path):
cache_folder = Path("cache") cache_folder = Path("cache")
if not cache_folder.exists(): if not cache_folder.exists():
@@ -102,26 +110,20 @@ def get_image_cache(path):
mtime = os.stat(path).st_mtime mtime = os.stat(path).st_mtime
if (path in image_cache and mtime != image_cache[path][0]) or (path not in image_cache): if (path in image_cache and mtime != image_cache[path][0]) or (path not in image_cache):
img = Image.open(path) img = make_thumbnail(Image.open(path))
img.thumbnail((200, 200))
output_file = Path(f'cache/{path.name}_cache.png') output_file = Path(f'cache/{path.name}_cache.png')
img.convert('RGB').save(output_file, format='PNG') img.convert('RGB').save(output_file, format='PNG')
image_cache[path] = [mtime, output_file.as_posix()] image_cache[path] = [mtime, output_file.as_posix()]
return image_cache[path][1] return image_cache[path][1]
def load_html_image(paths): def generate_chat_html(history, name1, name2, reset_cache=False):
for str_path in paths:
path = Path(str_path)
if path.exists():
return f'<img src="file/{get_image_cache(path)}">'
return ''
def generate_chat_html(history, name1, name2, character):
output = f'<style>{cai_css}</style><div class="chat" id="chat">' output = f'<style>{cai_css}</style><div class="chat" id="chat">'
img_bot = load_html_image([f"characters/{character}.{ext}" for ext in ['png', 'jpg', 'jpeg']] + ["img_bot.png","img_bot.jpg","img_bot.jpeg"]) # The time.time() is to prevent the brower from caching the image
img_me = load_html_image(["img_me.png", "img_me.jpg", "img_me.jpeg"]) suffix = f"?{time.time()}" if reset_cache else ''
img_bot = f'<img src="file/cache/pfp_character.png{suffix}">' if Path("cache/pfp_character.png").exists() else ''
img_me = f'<img src="file/cache/pfp_me.png{suffix}">' if Path("cache/pfp_me.png").exists() else ''
for i,_row in enumerate(history[::-1]): for i,_row in enumerate(history[::-1]):
row = [convert_to_markdown(entry) for entry in _row] row = [convert_to_markdown(entry) for entry in _row]

View File

@@ -42,7 +42,7 @@ def load_model(model_name):
t0 = time.time() t0 = time.time()
shared.is_RWKV = 'rwkv-' in model_name.lower() shared.is_RWKV = 'rwkv-' in model_name.lower()
shared.is_llamacpp = len(list(Path(f'models/{model_name}').glob('ggml*.bin'))) > 0 shared.is_llamacpp = len(list(Path(f'{shared.args.model_dir}/{model_name}').glob('ggml*.bin'))) > 0
# Default settings # Default settings
if not any([shared.args.cpu, shared.args.load_in_8bit, shared.args.wbits, shared.args.auto_devices, shared.args.disk, shared.args.gpu_memory is not None, shared.args.cpu_memory is not None, shared.args.deepspeed, shared.args.flexgen, shared.is_RWKV, shared.is_llamacpp]): if not any([shared.args.cpu, shared.args.load_in_8bit, shared.args.wbits, shared.args.auto_devices, shared.args.disk, shared.args.gpu_memory is not None, shared.args.cpu_memory is not None, shared.args.deepspeed, shared.args.flexgen, shared.is_RWKV, shared.is_llamacpp]):
@@ -105,7 +105,7 @@ def load_model(model_name):
elif shared.is_llamacpp: elif shared.is_llamacpp:
from modules.llamacpp_model import LlamaCppModel from modules.llamacpp_model import LlamaCppModel
model_file = list(Path(f'models/{model_name}').glob('ggml*.bin'))[0] model_file = list(Path(f'{shared.args.model_dir}/{model_name}').glob('ggml*.bin'))[0]
print(f"llama.cpp weights detected: {model_file}\n") print(f"llama.cpp weights detected: {model_file}\n")
model, tokenizer = LlamaCppModel.from_pretrained(model_file) model, tokenizer = LlamaCppModel.from_pretrained(model_file)

View File

@@ -74,7 +74,7 @@ parser = argparse.ArgumentParser(formatter_class=lambda prog: argparse.HelpForma
# Basic settings # Basic settings
parser.add_argument('--notebook', action='store_true', help='Launch the web UI in notebook mode, where the output is written to the same text box as the input.') parser.add_argument('--notebook', action='store_true', help='Launch the web UI in notebook mode, where the output is written to the same text box as the input.')
parser.add_argument('--chat', action='store_true', help='Launch the web UI in chat mode.') parser.add_argument('--chat', action='store_true', help='Launch the web UI in chat mode.')
parser.add_argument('--cai-chat', action='store_true', help='Launch the web UI in chat mode with a style similar to Character.AI\'s. If the file img_bot.png or img_bot.jpg exists in the same folder as server.py, this image will be used as the bot\'s profile picture. Similarly, img_me.png or img_me.jpg will be used as your profile picture.') parser.add_argument('--cai-chat', action='store_true', help='Launch the web UI in chat mode with a style similar to the Character.AI website.')
parser.add_argument('--model', type=str, help='Name of the model to load by default.') parser.add_argument('--model', type=str, help='Name of the model to load by default.')
parser.add_argument('--lora', type=str, help='Name of the LoRA to apply to the model by default.') parser.add_argument('--lora', type=str, help='Name of the LoRA to apply to the model by default.')
parser.add_argument("--model-dir", type=str, default='models/', help="Path to directory with all the models") parser.add_argument("--model-dir", type=str, default='models/', help="Path to directory with all the models")

View File

@@ -1,5 +1,6 @@
Below is an instruction that describes a task. Write a response that appropriately completes the request. Below is an instruction that describes a task. Write a response that appropriately completes the request.
### Instruction: ### Instruction:
%input% Write a poem about the transformers Python library.
Mention the word "large language models" in that poem.
### Response: ### Response:

View File

@@ -0,0 +1 @@
<|prompter|>Write a story about future of AI development<|endoftext|><|assistant|>

View File

@@ -13,4 +13,4 @@ safetensors==0.3.0
sentencepiece sentencepiece
pyyaml pyyaml
tqdm tqdm
git+https://github.com/huggingface/transformers git+https://github.com/huggingface/transformers@9eae4aa57650c1dbe1becd4e0979f6ad1e572ac0

View File

@@ -8,6 +8,7 @@ from datetime import datetime
from pathlib import Path from pathlib import Path
import gradio as gr import gradio as gr
from PIL import Image
import modules.extensions as extensions_module import modules.extensions as extensions_module
from modules import chat, shared, training, ui from modules import chat, shared, training, ui
@@ -296,7 +297,7 @@ def create_interface():
shared.gradio['Chat input'] = gr.State() shared.gradio['Chat input'] = gr.State()
with gr.Tab("Text generation", elem_id="main"): with gr.Tab("Text generation", elem_id="main"):
if shared.args.cai_chat: if shared.args.cai_chat:
shared.gradio['display'] = gr.HTML(value=generate_chat_html(shared.history['visible'], shared.settings['name1'], shared.settings['name2'], shared.character)) shared.gradio['display'] = gr.HTML(value=generate_chat_html(shared.history['visible'], shared.settings['name1'], shared.settings['name2']))
else: else:
shared.gradio['display'] = gr.Chatbot(value=shared.history['visible'], elem_id="gradio-chatbot") shared.gradio['display'] = gr.Chatbot(value=shared.history['visible'], elem_id="gradio-chatbot")
shared.gradio['textbox'] = gr.Textbox(label='Input') shared.gradio['textbox'] = gr.Textbox(label='Input')
@@ -316,10 +317,15 @@ def create_interface():
shared.gradio['Clear history-cancel'] = gr.Button('Cancel', visible=False) shared.gradio['Clear history-cancel'] = gr.Button('Cancel', visible=False)
with gr.Tab("Character", elem_id="chat-settings"): with gr.Tab("Character", elem_id="chat-settings"):
with gr.Row():
with gr.Column(scale=8):
shared.gradio['name1'] = gr.Textbox(value=shared.settings['name1'], lines=1, label='Your name') shared.gradio['name1'] = gr.Textbox(value=shared.settings['name1'], lines=1, label='Your name')
shared.gradio['name2'] = gr.Textbox(value=shared.settings['name2'], lines=1, label='Character\'s name') shared.gradio['name2'] = gr.Textbox(value=shared.settings['name2'], lines=1, label='Character\'s name')
shared.gradio['greeting'] = gr.Textbox(value=shared.settings['greeting'], lines=2, label='Greeting') shared.gradio['greeting'] = gr.Textbox(value=shared.settings['greeting'], lines=2, label='Greeting')
shared.gradio['context'] = gr.Textbox(value=shared.settings['context'], lines=8, label='Context') shared.gradio['context'] = gr.Textbox(value=shared.settings['context'], lines=8, label='Context')
with gr.Column(scale=1):
shared.gradio['character_picture'] = gr.Image(label='Character picture', type="pil")
shared.gradio['your_picture'] = gr.Image(label='Your picture', type="pil", value=Image.open(Path("cache/pfp_me.png")) if Path("cache/pfp_me.png").exists() else None)
with gr.Row(): with gr.Row():
shared.gradio['character_menu'] = gr.Dropdown(choices=available_characters, value='None', label='Character', elem_id='character-menu') shared.gradio['character_menu'] = gr.Dropdown(choices=available_characters, value='None', label='Character', elem_id='character-menu')
ui.create_refresh_button(shared.gradio['character_menu'], lambda : None, lambda : {'choices': get_available_characters()}, 'refresh-button') ui.create_refresh_button(shared.gradio['character_menu'], lambda : None, lambda : {'choices': get_available_characters()}, 'refresh-button')
@@ -347,8 +353,6 @@ def create_interface():
gr.Markdown("# TavernAI PNG format") gr.Markdown("# TavernAI PNG format")
shared.gradio['upload_img_tavern'] = gr.File(type='binary', file_types=['image']) shared.gradio['upload_img_tavern'] = gr.File(type='binary', file_types=['image'])
with gr.Tab('Upload your profile picture'):
shared.gradio['upload_img_me'] = gr.File(type='binary', file_types=['image'])
with gr.Tab("Parameters", elem_id="parameters"): with gr.Tab("Parameters", elem_id="parameters"):
with gr.Box(): with gr.Box():
@@ -399,15 +403,14 @@ def create_interface():
shared.gradio['textbox'].submit(lambda x: '', shared.gradio['textbox'], shared.gradio['textbox'], show_progress=False) shared.gradio['textbox'].submit(lambda x: '', shared.gradio['textbox'], shared.gradio['textbox'], show_progress=False)
shared.gradio['textbox'].submit(lambda : chat.save_history(timestamp=False), [], [], show_progress=False) shared.gradio['textbox'].submit(lambda : chat.save_history(timestamp=False), [], [], show_progress=False)
shared.gradio['character_menu'].change(chat.load_character, [shared.gradio[k] for k in ['character_menu', 'name1', 'name2']], [shared.gradio[k] for k in ['name1', 'name2', 'greeting', 'context', 'display']]) shared.gradio['character_menu'].change(chat.load_character, [shared.gradio[k] for k in ['character_menu', 'name1', 'name2']], [shared.gradio[k] for k in ['name1', 'name2', 'character_picture', 'greeting', 'context', 'display']])
shared.gradio['upload_chat_history'].upload(chat.load_history, [shared.gradio['upload_chat_history'], shared.gradio['name1'], shared.gradio['name2']], []) shared.gradio['upload_chat_history'].upload(chat.load_history, [shared.gradio['upload_chat_history'], shared.gradio['name1'], shared.gradio['name2']], [])
shared.gradio['upload_img_tavern'].upload(chat.upload_tavern_character, [shared.gradio['upload_img_tavern'], shared.gradio['name1'], shared.gradio['name2']], [shared.gradio['character_menu']]) shared.gradio['upload_img_tavern'].upload(chat.upload_tavern_character, [shared.gradio['upload_img_tavern'], shared.gradio['name1'], shared.gradio['name2']], [shared.gradio['character_menu']])
shared.gradio['upload_img_me'].upload(chat.upload_your_profile_picture, [shared.gradio['upload_img_me']], []) shared.gradio['your_picture'].change(chat.upload_your_profile_picture, [shared.gradio[k] for k in ['your_picture', 'name1', 'name2']], shared.gradio['display'])
reload_func = chat.redraw_html if shared.args.cai_chat else lambda : shared.history['visible'] reload_func = chat.redraw_html if shared.args.cai_chat else lambda : shared.history['visible']
reload_inputs = [shared.gradio['name1'], shared.gradio['name2']] if shared.args.cai_chat else [] reload_inputs = [shared.gradio['name1'], shared.gradio['name2']] if shared.args.cai_chat else []
shared.gradio['upload_chat_history'].upload(reload_func, reload_inputs, [shared.gradio['display']]) shared.gradio['upload_chat_history'].upload(reload_func, reload_inputs, [shared.gradio['display']])
shared.gradio['upload_img_me'].upload(reload_func, reload_inputs, [shared.gradio['display']])
shared.gradio['Stop'].click(reload_func, reload_inputs, [shared.gradio['display']]) shared.gradio['Stop'].click(reload_func, reload_inputs, [shared.gradio['display']])
shared.gradio['interface'].load(None, None, None, _js=f"() => {{{ui.main_js+ui.chat_js}}}") shared.gradio['interface'].load(None, None, None, _js=f"() => {{{ui.main_js+ui.chat_js}}}")