New yaml character format (#337 from TheTerrasque/feature/yaml-characters)

This doesn't break backward compatibility with JSON characters.

characters/Example.json

@@ -1,7 +0,0 @@
-{
-    "char_name": "Chiharu Yamada",
-    "char_persona": "Chiharu Yamada is a young, computer engineer-nerd with a knack for problem solving and a passion for technology.",
-    "char_greeting": "*Chiharu strides into the room with a smile, her eyes lighting up when she sees you. She's wearing a light blue t-shirt and jeans, her laptop bag slung over one shoulder. She takes a seat next to you, her enthusiasm palpable in the air*\nHey! I'm so excited to finally meet you. I've heard so many great things about you and I'm eager to pick your brain about computers. I'm sure you have a wealth of knowledge that I can learn from. *She grins, eyes twinkling with excitement* Let's get started!",
-    "world_scenario": "",
-    "example_dialogue": "{{user}}: So how did you get into computer engineering?\n{{char}}: I've always loved tinkering with technology since I was a kid.\n{{user}}: That's really impressive!\n{{char}}: *She chuckles bashfully* Thanks!\n{{user}}: So what do you do when you're not working on computers?\n{{char}}: I love exploring, going out with friends, watching movies, and playing video games.\n{{user}}: What's your favorite type of computer hardware to work with?\n{{char}}: Motherboards, they're like puzzles and the backbone of any system.\n{{user}}: That sounds great!\n{{char}}: Yeah, it's really fun. I'm lucky to be able to do this as a job."
-}

characters/Example.yaml

@@ -0,0 +1,32 @@
+name: Chiharu Yamada
+context: 'Chiharu Yamada''s Persona: Chiharu Yamada is a young, computer engineer-nerd
+  with a knack for problem solving and a passion for technology.'
+greeting: '*Chiharu strides into the room with a smile, her eyes lighting up
+  when she sees you. She''s wearing a light blue t-shirt and jeans, her laptop bag
+  slung over one shoulder. She takes a seat next to you, her enthusiasm palpable in
+  the air*
+
+  Hey! I''m so excited to finally meet you. I''ve heard so many great things about
+  you and I''m eager to pick your brain about computers. I''m sure you have a wealth
+  of knowledge that I can learn from. *She grins, eyes twinkling with excitement*
+  Let''s get started!'
+example_dialogue: '{{user}}: So how did you get into computer engineering?
+
+  {{char}}: I''ve always loved tinkering with technology since I was a kid.
+
+  {{user}}: That''s really impressive!
+
+  {{char}}: *She chuckles bashfully* Thanks!
+
+  {{user}}: So what do you do when you''re not working on computers?
+
+  {{char}}: I love exploring, going out with friends, watching movies, and playing
+  video games.
+
+  {{user}}: What''s your favorite type of computer hardware to work with?
+
+  {{char}}: Motherboards, they''re like puzzles and the backbone of any system.
+
+  {{user}}: That sounds great!
+
+  {{char}}: Yeah, it''s really fun. I''m lucky to be able to do this as a job.'

extensions/gallery/script.py

@@ -62,8 +62,8 @@ def generate_html():
     cards = []
     # Iterate through files in image folder
     for file in sorted(Path("characters").glob("*")):
-        if file.name.endswith(".json"):
-            character = file.name.replace(".json", "")
+        if file.suffix in [".json", ".yml", ".yaml"]:
+            character = file.stem
             container_html = f'<div class="character-container">'
             image_html = "<div class='placeholder'></div>"
 

modules/chat.py

@@ -6,6 +6,7 @@ import re
 from datetime import datetime
 from pathlib import Path
 
+import yaml
 from PIL import Image
 
 import modules.extensions as extensions_module
@@ -322,39 +323,54 @@ def load_history(file, name1, name2):
         shared.history['visible'] = copy.deepcopy(shared.history['internal'])
 
 def load_default_history(name1, name2):
+    shared.character = 'None'
     if Path('logs/persistent.json').exists():
         load_history(open(Path('logs/persistent.json'), 'rb').read(), name1, name2)
     else:
         shared.history['internal'] = []
         shared.history['visible'] = []
 
-def load_character(_character, name1, name2):
+def build_pygmalion_style_context(data):
     context = ""
+    if 'char_persona' in data and data['char_persona'] != '':
+        context += f"{data['char_name']}'s Persona: {data['char_persona']}\n"
+    if 'world_scenario' in data and data['world_scenario'] != '':
+        context += f"Scenario: {data['world_scenario']}\n"
+    context = f"{context.strip()}\n<START>\n"
+    return context
+
+def load_character(_character, name1, name2):
     shared.history['internal'] = []
     shared.history['visible'] = []
     if _character != 'None':
         shared.character = _character
-        data = json.loads(open(Path(f'characters/{_character}.json'), 'r', encoding='utf-8').read())
-        name2 = data['char_name']
-        if 'char_persona' in data and data['char_persona'] != '':
-            context += f"{data['char_name']}'s Persona: {data['char_persona']}\n"
-        if 'world_scenario' in data and data['world_scenario'] != '':
-            context += f"Scenario: {data['world_scenario']}\n"
-        context = f"{context.strip()}\n<START>\n"
+        
+        for extension in  ["yml", "yaml", "json"]:
+            filepath = Path(f'characters/{_character}.{extension}')
+            if filepath.exists():
+                break
+        data = yaml.safe_load(open(filepath, 'r', encoding='utf-8').read())
+
+        if 'context' in data:
+            context = f"{data['context'].strip()}\n\n"
+            name2 = data['name']
+            greeting_field = 'greeting'
+        else:
+            context = build_pygmalion_style_context(data)
+            name2 = data['char_name']
+            greeting_field = 'char_greeting'
+
         if 'example_dialogue' in data and data['example_dialogue'] != '':
             data['example_dialogue'] = data['example_dialogue'].replace('{{user}}', name1).replace('{{char}}', name2)
             data['example_dialogue'] = data['example_dialogue'].replace('<USER>', name1).replace('<BOT>', name2)
             context += f"{data['example_dialogue'].strip()}\n"
-        if 'char_greeting' in data and len(data['char_greeting'].strip()) > 0:
-            shared.history['internal'] += [['<|BEGIN-VISIBLE-CHAT|>', data['char_greeting']]]
-            shared.history['visible'] += [['', apply_extensions(data['char_greeting'], "output")]]
-        else:
-            shared.history['internal'] += [['<|BEGIN-VISIBLE-CHAT|>', "Hello there!"]]
-            shared.history['visible'] += [['', "Hello there!"]]
+        if greeting_field in data and len(data[greeting_field].strip()) > 0:
+            shared.history['internal'] += [['<|BEGIN-VISIBLE-CHAT|>', data[greeting_field]]]
+            shared.history['visible'] += [['', apply_extensions(data[greeting_field], "output")]]
     else:
-        shared.character = None
-        context = shared.settings['context_pygmalion']
-        name2 = shared.settings['name2_pygmalion']
+        shared.character = 'None'
+        context = shared.settings['context']
+        name2 = shared.settings['name2']
 
     if Path(f'logs/{shared.character}_persistent.json').exists():
         load_history(open(Path(f'logs/{shared.character}_persistent.json'), 'rb').read(), name1, name2)

requirements.txt

@@ -1,5 +1,6 @@
 accelerate==0.18.0
 bitsandbytes==0.37.2
+datasets
 flexgen==0.1.7
 gradio==3.24.0
 llamacpp==0.1.11
@@ -10,6 +11,6 @@ requests
 rwkv==0.7.1
 safetensors==0.3.0
 sentencepiece
+pyyaml
 tqdm
-datasets
 git+https://github.com/huggingface/transformers

server.py

@@ -46,7 +46,8 @@ def get_available_prompts():
     return prompts
 
 def get_available_characters():
-    return ['None'] + sorted(set(map(lambda x : '.'.join(str(x.name).split('.')[:-1]), Path('characters').glob('*.json'))), key=str.lower)
+    paths = (x for x in Path('characters').iterdir() if x.suffix in ('.json', '.yaml', '.yml'))
+    return ['None'] + sorted(set((k.stem for k in paths)), key=str.lower)
 
 def get_available_extensions():
     return sorted(set(map(lambda x : x.parts[1], Path('extensions').glob('*/script.py'))), key=str.lower)