[Template] add jinja plugin, apply_chat_template support dict messages

paddlenlp/transformers/tokenizer_utils.py

@@ -526,8 +526,24 @@ class ChatTemplate:
         def raise_exception(message):
             raise TemplateError(message)
 
+        def regex_findall(s, pattern, multiline=False, dotall=False):
+            flags = 0
+            if multiline:
+                flags |= re.MULTILINE
+            if dotall:
+                flags |= re.DOTALL
+            return re.findall(pattern, s, flags)
+
+        def tojson(x, ensure_ascii=False, indent=None, separators=None, sort_keys=False):
+            # We override the built-in tojson filter because Jinja's default filter escapes HTML characters
+            # We also expose some options like custom indents and separators
+            return json.dumps(x, ensure_ascii=ensure_ascii, indent=indent, separators=separators, sort_keys=sort_keys)
+
         jinja_env = ImmutableSandboxedEnvironment(trim_blocks=True, lstrip_blocks=True, keep_trailing_newline=True)
         jinja_env.globals["raise_exception"] = raise_exception
+        jinja_env.filters["regex_findall"] = regex_findall
+        jinja_env.filters["tojson"] = tojson
+        jinja_env.globals.update(regex_findall=regex_findall)
         return jinja_env.from_string(chat_template)
 
     def render_conversation(
@@ -694,6 +710,13 @@ class ChatTemplateMixin:
                     "apply_chat_template do not support applying batch conversations, "
                     "so you should apply the conversation one by one."
                 )
+        elif isinstance(conversation, dict):
+            conversations = conversation
+            query = self.chat_template.render(
+                conversations, **self.special_tokens_map, add_generation_prompt=add_generation_prompt
+            )
+            return query
+
         query = self.chat_template.render(
             messages=conversations, **self.special_tokens_map, add_generation_prompt=add_generation_prompt
         )