zhengjl/text-generation-webui
Public Access
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Rename variables

Browse Source
This commit is contained in:
oobabooga
2023-04-05 23:38:01 -03:00
parent 572f1d8bdb
commit 9e31fe65ce
3 changed files with 52 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
modules/chat.py
View File

@@ -96,9 +96,9 @@ def extract_message_from_reply(reply, name1, name2, stop_at_newline):
reply = fix_newlines(reply) reply = fix_newlines(reply)
return reply, next_character_found return reply, next_character_found
def chatbot_wrapper(text, generate_params, name1, name2, context, mode, end_of_turn, regenerate=False): def chatbot_wrapper(text, generate_state, name1, name2, context, mode, end_of_turn, regenerate=False):
just_started = True just_started = True
eos_token = '\n' if generate_params['stop_at_newline'] else None eos_token = '\n' if generate_state['stop_at_newline'] else None
name1_original = name1 name1_original = name1
if 'pygmalion' in shared.model_name.lower(): if 'pygmalion' in shared.model_name.lower():
name1 = "You" name1 = "You"
@@ -119,9 +119,9 @@ def chatbot_wrapper(text, generate_params, name1, name2, context, mode, end_of_t
kwargs = {'end_of_turn': end_of_turn, 'is_instruct': mode == 'instruct'} kwargs = {'end_of_turn': end_of_turn, 'is_instruct': mode == 'instruct'}
if custom_generate_chat_prompt is None: if custom_generate_chat_prompt is None:
prompt = generate_chat_prompt(text, generate_params['max_new_tokens'], name1, name2, context, generate_params['chat_prompt_size'], **kwargs) prompt = generate_chat_prompt(text, generate_state['max_new_tokens'], name1, name2, context, generate_state['chat_prompt_size'], **kwargs)
else: else:
prompt = custom_generate_chat_prompt(text, generate_params['max_new_tokens'], name1, name2, context, generate_params['chat_prompt_size'], **kwargs) prompt = custom_generate_chat_prompt(text, generate_state['max_new_tokens'], name1, name2, context, generate_state['chat_prompt_size'], **kwargs)
# Yield *Is typing...* # Yield *Is typing...*
if not regenerate: if not regenerate:
@@ -129,13 +129,13 @@ def chatbot_wrapper(text, generate_params, name1, name2, context, mode, end_of_t
# Generate # Generate
cumulative_reply = '' cumulative_reply = ''
for i in range(generate_params['chat_generation_attempts']): for i in range(generate_state['chat_generation_attempts']):
reply = None reply = None
for reply in generate_reply(f"{prompt}{' ' if len(cumulative_reply) > 0 else ''}{cumulative_reply}", generate_params, eos_token=eos_token, stopping_strings=[f"\n{name1}:", f"\n{name2}:"]): for reply in generate_reply(f"{prompt}{' ' if len(cumulative_reply) > 0 else ''}{cumulative_reply}", generate_state, eos_token=eos_token, stopping_strings=[f"\n{name1}:", f"\n{name2}:"]):
reply = cumulative_reply + reply reply = cumulative_reply + reply
# Extracting the reply # Extracting the reply
reply, next_character_found = extract_message_from_reply(reply, name1, name2, generate_params['stop_at_newline']) reply, next_character_found = extract_message_from_reply(reply, name1, name2, generate_state['stop_at_newline'])
visible_reply = re.sub("(<USER>|<user>|{{user}})", name1_original, reply) visible_reply = re.sub("(<USER>|<user>|{{user}})", name1_original, reply)
visible_reply = apply_extensions(visible_reply, "output") visible_reply = apply_extensions(visible_reply, "output")
@@ -160,23 +160,23 @@ def chatbot_wrapper(text, generate_params, name1, name2, context, mode, end_of_t
yield shared.history['visible'] yield shared.history['visible']
def impersonate_wrapper(text, generate_params, name1, name2, context, mode, end_of_turn): def impersonate_wrapper(text, generate_state, name1, name2, context, mode, end_of_turn):
eos_token = '\n' if generate_params['stop_at_newline'] else None eos_token = '\n' if generate_state['stop_at_newline'] else None
if 'pygmalion' in shared.model_name.lower(): if 'pygmalion' in shared.model_name.lower():
name1 = "You" name1 = "You"
prompt = generate_chat_prompt(text, generate_params['max_new_tokens'], name1, name2, context, generate_params['chat_prompt_size'], impersonate=True, end_of_turn=end_of_turn) prompt = generate_chat_prompt(text, generate_state['max_new_tokens'], name1, name2, context, generate_state['chat_prompt_size'], impersonate=True, end_of_turn=end_of_turn)
# Yield *Is typing...* # Yield *Is typing...*
yield shared.processing_message yield shared.processing_message
cumulative_reply = '' cumulative_reply = ''
for i in range(generate_params['chat_generation_attempts']): for i in range(generate_state['chat_generation_attempts']):
reply = None reply = None
for reply in generate_reply(f"{prompt}{' ' if len(cumulative_reply) > 0 else ''}{cumulative_reply}", generate_params, eos_token=eos_token, stopping_strings=[f"\n{name1}:", f"\n{name2}:"]): for reply in generate_reply(f"{prompt}{' ' if len(cumulative_reply) > 0 else ''}{cumulative_reply}", generate_state, eos_token=eos_token, stopping_strings=[f"\n{name1}:", f"\n{name2}:"]):
reply = cumulative_reply + reply reply = cumulative_reply + reply
reply, next_character_found = extract_message_from_reply(reply, name1, name2, generate_params['stop_at_newline']) reply, next_character_found = extract_message_from_reply(reply, name1, name2, generate_state['stop_at_newline'])
yield reply yield reply
if next_character_found: if next_character_found:
break break
@@ -186,11 +186,11 @@ def impersonate_wrapper(text, generate_params, name1, name2, context, mode, end_
yield reply yield reply
def cai_chatbot_wrapper(text, generate_params, name1, name2, context, mode, end_of_turn): def cai_chatbot_wrapper(text, generate_state, name1, name2, context, mode, end_of_turn):
for history in chatbot_wrapper(text, generate_params, name1, name2, context, mode, end_of_turn, regenerate=False): for history in chatbot_wrapper(text, generate_state, name1, name2, context, mode, end_of_turn, regenerate=False):
yield chat_html_wrapper(history, name1, name2, mode) yield chat_html_wrapper(history, name1, name2, mode)
def regenerate_wrapper(text, generate_params, name1, name2, context, mode, end_of_turn): def regenerate_wrapper(text, generate_state, name1, name2, context, mode, end_of_turn):
if (shared.character != 'None' and len(shared.history['visible']) == 1) or len(shared.history['internal']) == 0: if (shared.character != 'None' and len(shared.history['visible']) == 1) or len(shared.history['internal']) == 0:
yield chat_html_wrapper(shared.history['visible'], name1, name2, mode) yield chat_html_wrapper(shared.history['visible'], name1, name2, mode)
else: else:
@@ -198,7 +198,7 @@ def regenerate_wrapper(text, generate_params, name1, name2, context, mode, end_o
last_internal = shared.history['internal'].pop() last_internal = shared.history['internal'].pop()
# Yield '*Is typing...*' # Yield '*Is typing...*'
yield chat_html_wrapper(shared.history['visible']+[[last_visible[0], shared.processing_message]], name1, name2, mode) yield chat_html_wrapper(shared.history['visible']+[[last_visible[0], shared.processing_message]], name1, name2, mode)
for history in chatbot_wrapper(last_internal[0], generate_params, name1, name2, context, mode, end_of_turn, regenerate=True): for history in chatbot_wrapper(last_internal[0], generate_state, name1, name2, context, mode, end_of_turn, regenerate=True):
shared.history['visible'][-1] = [last_visible[0], history[-1][1]] shared.history['visible'][-1] = [last_visible[0], history[-1][1]]
yield chat_html_wrapper(shared.history['visible'], name1, name2, mode) yield chat_html_wrapper(shared.history['visible'], name1, name2, mode)

56
modules/text_generation.py
View File

@@ -102,11 +102,11 @@ def set_manual_seed(seed):
def stop_everything_event(): def stop_everything_event():
shared.stop_everything = True shared.stop_everything = True
def generate_reply(question, generate_params, eos_token=None, stopping_strings=[]): def generate_reply(question, generate_state, eos_token=None, stopping_strings=[]):
clear_torch_cache() clear_torch_cache()
set_manual_seed(generate_params['seed']) set_manual_seed(generate_state['seed'])
shared.stop_everything = False shared.stop_everything = False
updated_params = {} generate_params = {}
t0 = time.time() t0 = time.time()
original_question = question original_question = question
@@ -119,11 +119,11 @@ def generate_reply(question, generate_params, eos_token=None, stopping_strings=[
# separately and terminate the function call earlier # separately and terminate the function call earlier
if any((shared.is_RWKV, shared.is_llamacpp)): if any((shared.is_RWKV, shared.is_llamacpp)):
for k in ['temperature', 'top_p', 'top_k', 'repetition_penalty']: for k in ['temperature', 'top_p', 'top_k', 'repetition_penalty']:
updated_params[k] = generate_params[k] generate_params[k] = generate_state[k]
updated_params["token_count"] = generate_params["max_new_tokens"] generate_params["token_count"] = generate_state["max_new_tokens"]
try: try:
if shared.args.no_stream: if shared.args.no_stream:
reply = shared.model.generate(context=question, **updated_params) reply = shared.model.generate(context=question, **generate_params)
output = original_question+reply output = original_question+reply
if not shared.is_chat(): if not shared.is_chat():
reply = original_question + apply_extensions(reply, "output") reply = original_question + apply_extensions(reply, "output")
@@ -134,7 +134,7 @@ def generate_reply(question, generate_params, eos_token=None, stopping_strings=[
# RWKV has proper streaming, which is very nice. # RWKV has proper streaming, which is very nice.
# No need to generate 8 tokens at a time. # No need to generate 8 tokens at a time.
for reply in shared.model.generate_with_streaming(context=question, **updated_params): for reply in shared.model.generate_with_streaming(context=question, **generate_params):
output = original_question+reply output = original_question+reply
if not shared.is_chat(): if not shared.is_chat():
reply = original_question + apply_extensions(reply, "output") reply = original_question + apply_extensions(reply, "output")
@@ -149,7 +149,7 @@ def generate_reply(question, generate_params, eos_token=None, stopping_strings=[
print(f"Output generated in {(t1-t0):.2f} seconds ({new_tokens/(t1-t0):.2f} tokens/s, {new_tokens} tokens, context {original_tokens})") print(f"Output generated in {(t1-t0):.2f} seconds ({new_tokens/(t1-t0):.2f} tokens/s, {new_tokens} tokens, context {original_tokens})")
return return
input_ids = encode(question, generate_params['max_new_tokens']) input_ids = encode(question, generate_state['max_new_tokens'])
original_input_ids = input_ids original_input_ids = input_ids
output = input_ids[0] output = input_ids[0]
@@ -162,37 +162,37 @@ def generate_reply(question, generate_params, eos_token=None, stopping_strings=[
t = [encode(string, 0, add_special_tokens=False) for string in stopping_strings] t = [encode(string, 0, add_special_tokens=False) for string in stopping_strings]
stopping_criteria_list.append(_SentinelTokenStoppingCriteria(sentinel_token_ids=t, starting_idx=len(input_ids[0]))) stopping_criteria_list.append(_SentinelTokenStoppingCriteria(sentinel_token_ids=t, starting_idx=len(input_ids[0])))
updated_params["max_new_tokens"] = generate_params['max_new_tokens'] generate_params["max_new_tokens"] = generate_state['max_new_tokens']
if not shared.args.flexgen: if not shared.args.flexgen:
for k in ["do_sample", "temperature", "top_p", "typical_p", "repetition_penalty", "encoder_repetition_penalty", "top_k", "min_length", "no_repeat_ngram_size", "num_beams", "penalty_alpha", "length_penalty", "early_stopping"]: for k in ["do_sample", "temperature", "top_p", "typical_p", "repetition_penalty", "encoder_repetition_penalty", "top_k", "min_length", "no_repeat_ngram_size", "num_beams", "penalty_alpha", "length_penalty", "early_stopping"]:
updated_params[k] = generate_params[k] generate_params[k] = generate_state[k]
updated_params["eos_token_id"] = eos_token_ids generate_params["eos_token_id"] = eos_token_ids
updated_params["stopping_criteria"] = stopping_criteria_list generate_params["stopping_criteria"] = stopping_criteria_list
if shared.args.no_stream: if shared.args.no_stream:
updated_params["min_length"] = 0 generate_params["min_length"] = 0
else: else:
for k in ["do_sample", "temperature"]: for k in ["do_sample", "temperature"]:
updated_params[k] = generate_params[k] generate_params[k] = generate_state[k]
updated_params["stop"] = generate_params["eos_token_ids"][-1] generate_params["stop"] = generate_state["eos_token_ids"][-1]
if not shared.args.no_stream: if not shared.args.no_stream:
updated_params["max_new_tokens"] = 8 generate_params["max_new_tokens"] = 8
if shared.args.no_cache: if shared.args.no_cache:
updated_params.update({"use_cache": False}) generate_params.update({"use_cache": False})
if shared.args.deepspeed: if shared.args.deepspeed:
updated_params.update({"synced_gpus": True}) generate_params.update({"synced_gpus": True})
if shared.soft_prompt: if shared.soft_prompt:
inputs_embeds, filler_input_ids = generate_softprompt_input_tensors(input_ids) inputs_embeds, filler_input_ids = generate_softprompt_input_tensors(input_ids)
updated_params.update({"inputs_embeds": inputs_embeds}) generate_params.update({"inputs_embeds": inputs_embeds})
updated_params.update({"inputs": filler_input_ids}) generate_params.update({"inputs": filler_input_ids})
else: else:
updated_params.update({"inputs": input_ids}) generate_params.update({"inputs": input_ids})
try: try:
# Generate the entire reply at once. # Generate the entire reply at once.
if shared.args.no_stream: if shared.args.no_stream:
with torch.no_grad(): with torch.no_grad():
output = shared.model.generate(**updated_params)[0] output = shared.model.generate(**generate_params)[0]
if cuda: if cuda:
output = output.cuda() output = output.cuda()
if shared.soft_prompt: if shared.soft_prompt:
@@ -220,7 +220,7 @@ def generate_reply(question, generate_params, eos_token=None, stopping_strings=[
if not shared.is_chat(): if not shared.is_chat():
yield formatted_outputs(original_question, shared.model_name) yield formatted_outputs(original_question, shared.model_name)
with generate_with_streaming(**updated_params) as generator: with generate_with_streaming(**generate_params) as generator:
for output in generator: for output in generator:
if shared.soft_prompt: if shared.soft_prompt:
output = torch.cat((input_ids[0], output[filler_input_ids.shape[1]:])) output = torch.cat((input_ids[0], output[filler_input_ids.shape[1]:]))
@@ -236,10 +236,10 @@ def generate_reply(question, generate_params, eos_token=None, stopping_strings=[
# Stream the output naively for FlexGen since it doesn't support 'stopping_criteria' # Stream the output naively for FlexGen since it doesn't support 'stopping_criteria'
else: else:
for i in range(generate_params['max_new_tokens']//8+1): for i in range(generate_state['max_new_tokens']//8+1):
clear_torch_cache() clear_torch_cache()
with torch.no_grad(): with torch.no_grad():
output = shared.model.generate(**updated_params)[0] output = shared.model.generate(**generate_params)[0]
if shared.soft_prompt: if shared.soft_prompt:
output = torch.cat((input_ids[0], output[filler_input_ids.shape[1]:])) output = torch.cat((input_ids[0], output[filler_input_ids.shape[1]:]))
@@ -255,10 +255,10 @@ def generate_reply(question, generate_params, eos_token=None, stopping_strings=[
input_ids = np.reshape(output, (1, output.shape[0])) input_ids = np.reshape(output, (1, output.shape[0]))
if shared.soft_prompt: if shared.soft_prompt:
inputs_embeds, filler_input_ids = generate_softprompt_input_tensors(input_ids) inputs_embeds, filler_input_ids = generate_softprompt_input_tensors(input_ids)
updated_params.update({"inputs_embeds": inputs_embeds}) generate_params.update({"inputs_embeds": inputs_embeds})
updated_params.update({"inputs": filler_input_ids}) generate_params.update({"inputs": filler_input_ids})
else: else:
updated_params.update({"inputs": input_ids}) generate_params.update({"inputs": input_ids})
yield formatted_outputs(reply, shared.model_name) yield formatted_outputs(reply, shared.model_name)

14
server.py
View File

@@ -173,7 +173,7 @@ def create_settings_menus(default_preset):
generate_params = load_preset_values(default_preset if not shared.args.flexgen else 'Naive', {}, return_dict=True) generate_params = load_preset_values(default_preset if not shared.args.flexgen else 'Naive', {}, return_dict=True)
for k in ['max_new_tokens', 'seed', 'stop_at_newline', 'chat_prompt_size', 'chat_generation_attempts']: for k in ['max_new_tokens', 'seed', 'stop_at_newline', 'chat_prompt_size', 'chat_generation_attempts']:
generate_params[k] = shared.settings[k] generate_params[k] = shared.settings[k]
shared.gradio['generation_state'] = gr.State(generate_params) shared.gradio['generate_state'] = gr.State(generate_params)
with gr.Row(): with gr.Row():
with gr.Column(): with gr.Column():
@@ -225,7 +225,7 @@ def create_settings_menus(default_preset):
shared.gradio['upload_softprompt'] = gr.File(type='binary', file_types=['.zip']) shared.gradio['upload_softprompt'] = gr.File(type='binary', file_types=['.zip'])
shared.gradio['model_menu'].change(load_model_wrapper, shared.gradio['model_menu'], shared.gradio['model_menu'], show_progress=True) shared.gradio['model_menu'].change(load_model_wrapper, shared.gradio['model_menu'], shared.gradio['model_menu'], show_progress=True)
shared.gradio['preset_menu'].change(load_preset_values, [shared.gradio[k] for k in ['preset_menu', 'generation_state']], [shared.gradio[k] for k in ['generation_state', 'do_sample', 'temperature', 'top_p', 'typical_p', 'repetition_penalty', 'encoder_repetition_penalty', 'top_k', 'min_length', 'no_repeat_ngram_size', 'num_beams', 'penalty_alpha', 'length_penalty', 'early_stopping']]) shared.gradio['preset_menu'].change(load_preset_values, [shared.gradio[k] for k in ['preset_menu', 'generate_state']], [shared.gradio[k] for k in ['generate_state', 'do_sample', 'temperature', 'top_p', 'typical_p', 'repetition_penalty', 'encoder_repetition_penalty', 'top_k', 'min_length', 'no_repeat_ngram_size', 'num_beams', 'penalty_alpha', 'length_penalty', 'early_stopping']])
shared.gradio['lora_menu'].change(load_lora_wrapper, shared.gradio['lora_menu'], shared.gradio['lora_menu'], show_progress=True) shared.gradio['lora_menu'].change(load_lora_wrapper, shared.gradio['lora_menu'], shared.gradio['lora_menu'], show_progress=True)
shared.gradio['softprompts_menu'].change(load_soft_prompt, shared.gradio['softprompts_menu'], shared.gradio['softprompts_menu'], show_progress=True) shared.gradio['softprompts_menu'].change(load_soft_prompt, shared.gradio['softprompts_menu'], shared.gradio['softprompts_menu'], show_progress=True)
shared.gradio['upload_softprompt'].upload(upload_soft_prompt, shared.gradio['upload_softprompt'], shared.gradio['softprompts_menu']) shared.gradio['upload_softprompt'].upload(upload_soft_prompt, shared.gradio['upload_softprompt'], shared.gradio['softprompts_menu'])
@@ -378,7 +378,7 @@ def create_interface():
create_settings_menus(default_preset) create_settings_menus(default_preset)
shared.input_params = [shared.gradio[k] for k in ['Chat input', 'generation_state', 'name1', 'name2', 'context', 'Chat mode', 'end_of_turn']] shared.input_params = [shared.gradio[k] for k in ['Chat input', 'generate_state', 'name1', 'name2', 'context', 'Chat mode', 'end_of_turn']]
def set_chat_input(textbox): def set_chat_input(textbox):
return textbox, "" return textbox, ""
@@ -458,7 +458,7 @@ def create_interface():
with gr.Tab("Parameters", elem_id="parameters"): with gr.Tab("Parameters", elem_id="parameters"):
create_settings_menus(default_preset) create_settings_menus(default_preset)
shared.input_params = [shared.gradio[k] for k in ['textbox', 'generation_state']] shared.input_params = [shared.gradio[k] for k in ['textbox', 'generate_state']]
output_params = [shared.gradio[k] for k in ['textbox', 'markdown', 'html']] output_params = [shared.gradio[k] for k in ['textbox', 'markdown', 'html']]
gen_events.append(shared.gradio['Generate'].click(generate_reply, shared.input_params, output_params, show_progress=shared.args.no_stream)) gen_events.append(shared.gradio['Generate'].click(generate_reply, shared.input_params, output_params, show_progress=shared.args.no_stream))
gen_events.append(shared.gradio['textbox'].submit(generate_reply, shared.input_params, output_params, show_progress=shared.args.no_stream)) gen_events.append(shared.gradio['textbox'].submit(generate_reply, shared.input_params, output_params, show_progress=shared.args.no_stream))
@@ -491,7 +491,7 @@ def create_interface():
with gr.Tab("Parameters", elem_id="parameters"): with gr.Tab("Parameters", elem_id="parameters"):
create_settings_menus(default_preset) create_settings_menus(default_preset)
shared.input_params = [shared.gradio[k] for k in ['textbox', 'generation_state']] shared.input_params = [shared.gradio[k] for k in ['textbox', 'generate_state']]
output_params = [shared.gradio[k] for k in ['output_textbox', 'markdown', 'html']] output_params = [shared.gradio[k] for k in ['output_textbox', 'markdown', 'html']]
gen_events.append(shared.gradio['Generate'].click(generate_reply, shared.input_params, output_params, show_progress=shared.args.no_stream)) gen_events.append(shared.gradio['Generate'].click(generate_reply, shared.input_params, output_params, show_progress=shared.args.no_stream))
gen_events.append(shared.gradio['textbox'].submit(generate_reply, shared.input_params, output_params, show_progress=shared.args.no_stream)) gen_events.append(shared.gradio['textbox'].submit(generate_reply, shared.input_params, output_params, show_progress=shared.args.no_stream))
@@ -530,9 +530,9 @@ def create_interface():
if k not in shared.gradio: if k not in shared.gradio:
continue continue
if type(shared.gradio[k]) in [gr.Checkbox, gr.Number]: if type(shared.gradio[k]) in [gr.Checkbox, gr.Number]:
shared.gradio[k].change(lambda state, value, copy=k: state.update({copy: value}), inputs=[shared.gradio['generation_state'], shared.gradio[k]], outputs=shared.gradio['generation_state']) shared.gradio[k].change(lambda state, value, copy=k: state.update({copy: value}), inputs=[shared.gradio['generate_state'], shared.gradio[k]], outputs=shared.gradio['generate_state'])
else: else:
shared.gradio[k].release(lambda state, value, copy=k: state.update({copy: value}), inputs=[shared.gradio['generation_state'], shared.gradio[k]], outputs=shared.gradio['generation_state']) shared.gradio[k].release(lambda state, value, copy=k: state.update({copy: value}), inputs=[shared.gradio['generate_state'], shared.gradio[k]], outputs=shared.gradio['generate_state'])
# Authentication # Authentication
auth = None auth = None