Sfoglia il codice sorgente

Keep GALACTICA special tokens (#300)

oobabooga 2 anni fa
parent
commit
a577fb1077
1 ha cambiato i file con 1 aggiunte e 1 eliminazioni
  1. 1 1
      modules/text_generation.py

+ 1 - 1
modules/text_generation.py

@@ -38,7 +38,7 @@ def encode(prompt, tokens_to_generate=0, add_special_tokens=True):
 
 def decode(output_ids):
     # Open Assistant relies on special tokens like <|endoftext|>
-    if re.match('oasst-*', shared.model_name.lower()):
+    if re.match('(oasst|galactica)-*', shared.model_name.lower()):
         return shared.tokenizer.decode(output_ids, skip_special_tokens=False)
     else:
         reply = shared.tokenizer.decode(output_ids, skip_special_tokens=True)