{
    "unconditional": [
        {
            "id": 2,
            "desc": "Share relatable breakup experiences.",
            "other_info": {
                "from": "system_para_0",
                "type_explanation": "This constraint focuses on the meaningful content of the output, specifically requiring relatable breakup experiences. It does not dictate formatting, resource dependencies, or computational limits. The emphasis on 'relatable' suggests a semantic requirement for tone, relatability, and emotional resonance.",
                "meta_expalnation": "The given constraint explicitly asks for sharing 'relatable breakup experiences,' which directly constrains the content or output that the model is expected to provide. It does not govern strategies for managing, selecting, prioritizing, ignoring, deduplicating, or combining other constraints, and hence is not a meta constraint.",
                "evaluation_type": {
                    "constraint_type": "llm",
                    "explanation": "Determining whether the breakup experiences shared are 'relatable' requires semantic and subjective understanding, which falls outside direct code logic or structured content extraction. An LLM is needed to assess this concept semantically."
                },
                "evaluation_generation_success": true
            },
            "dimension": "unconditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Does the response include relatable breakup experiences as part of the content? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "state": "success",
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 1,
            "desc": "Use gentle humor to lighten the mood.",
            "other_info": {
                "from": "system_para_0",
                "type_explanation": "This constraint specifies the tone and style of the content by requiring the use of 'gentle humor.' Tone and style fall under the semantic category because they govern the meaningful presentation and emotional impact of the output.",
                "meta_expalnation": "The given constraint directly specifies the style or tone of the output (i.e., to use gentle humor to lighten the mood). It does not govern how constraints should be selected, prioritized, ignored, deduplicated, or combined. Therefore, it is not classified as a Meta Constraint.",
                "evaluation_type": {
                    "constraint_type": "llm",
                    "explanation": "Using gentle humor to lighten the mood requires a semantic and subjective assessment of whether the humor is perceived as 'gentle' and appropriate to the context. This involves interpreting tone, empathy, and relevance, which can only be validated semantically by an LLM."
                },
                "evaluation_generation_success": true
            },
            "dimension": "unconditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Does the model response use gentle humor? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "state": "modified",
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 2,
            "desc": "Share relatable breakup experiences.",
            "other_info": {
                "from": "system_para_0",
                "type_explanation": "This constraint focuses on the meaningful content of the output, specifically requiring relatable breakup experiences. It does not dictate formatting, resource dependencies, or computational limits. The emphasis on 'relatable' suggests a semantic requirement for tone, relatability, and emotional resonance.",
                "meta_expalnation": "The given constraint explicitly asks for sharing 'relatable breakup experiences,' which directly constrains the content or output that the model is expected to provide. It does not govern strategies for managing, selecting, prioritizing, ignoring, deduplicating, or combining other constraints, and hence is not a meta constraint.",
                "evaluation_type": {
                    "constraint_type": "llm",
                    "explanation": "Determining whether the breakup experiences shared are 'relatable' requires semantic and subjective understanding, which falls outside direct code logic or structured content extraction. An LLM is needed to assess this concept semantically."
                },
                "evaluation_generation_success": true
            },
            "dimension": "unconditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Does the response include relatable breakup experiences as part of the content? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "state": "success",
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 1,
            "desc": "Use gentle humor to lighten the mood.",
            "other_info": {
                "from": "system_para_0",
                "type_explanation": "This constraint specifies the tone and style of the content by requiring the use of 'gentle humor.' Tone and style fall under the semantic category because they govern the meaningful presentation and emotional impact of the output.",
                "meta_expalnation": "The given constraint directly specifies the style or tone of the output (i.e., to use gentle humor to lighten the mood). It does not govern how constraints should be selected, prioritized, ignored, deduplicated, or combined. Therefore, it is not classified as a Meta Constraint.",
                "evaluation_type": {
                    "constraint_type": "llm",
                    "explanation": "Using gentle humor to lighten the mood requires a semantic and subjective assessment of whether the humor is perceived as 'gentle' and appropriate to the context. This involves interpreting tone, empathy, and relevance, which can only be validated semantically by an LLM."
                },
                "evaluation_generation_success": true
            },
            "dimension": "unconditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Does the model response use gentle humor? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "state": "modified",
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 1,
            "desc": "Use gentle humor to lighten the mood.",
            "other_info": {
                "from": "system_para_0",
                "type_explanation": "This constraint specifies the tone and style of the content by requiring the use of 'gentle humor.' Tone and style fall under the semantic category because they govern the meaningful presentation and emotional impact of the output.",
                "meta_expalnation": "The given constraint directly specifies the style or tone of the output (i.e., to use gentle humor to lighten the mood). It does not govern how constraints should be selected, prioritized, ignored, deduplicated, or combined. Therefore, it is not classified as a Meta Constraint.",
                "evaluation_type": {
                    "constraint_type": "llm",
                    "explanation": "Using gentle humor to lighten the mood requires a semantic and subjective assessment of whether the humor is perceived as 'gentle' and appropriate to the context. This involves interpreting tone, empathy, and relevance, which can only be validated semantically by an LLM."
                },
                "evaluation_generation_success": true
            },
            "dimension": "unconditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Does the model response use gentle humor? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "state": "modified",
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 1,
            "desc": "Ensure tone is heartfelt and authentic.",
            "other_info": {
                "from": "system_para_0",
                "type_explanation": "This constraint pertains to the style and tone of the output. Ensuring the tone is heartfelt and authentic focuses on the semantic quality of the communication, which aligns with conveying meaningful and appropriate content rather than structure, formatting, or resource-related concerns.",
                "meta_expalnation": "The given constraint directly describes a requirement for the tone (heartfelt and authentic) that the model's output must follow. This directly constrains the content and style of the output and does not manage or govern other constraints, nor does it specify strategies for selecting, prioritizing, ignoring, deduplicating, or combining constraints. Therefore, it is not a Meta Constraint.",
                "evaluation_type": {
                    "constraint_type": "llm",
                    "explanation": "Determining whether the tone is heartfelt and authentic requires a semantic and subjective understanding of language, which can only be assessed by an LLM."
                },
                "evaluation_generation_success": true
            },
            "dimension": "unconditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Does the model's response use a tone that is both heartfelt and authentic? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "state": "modified",
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 2,
            "desc": "Connect them with appropriate mental health services while acknowledging barriers (cost, access, stigma).",
            "other_info": {
                "from": "system_para_0",
                "type_explanation": "The constraint focuses on content relevance and meaningfulness, specifying the inclusion of barriers (cost, access, stigma) while connecting individuals with mental health services. It emphasizes information completeness and logical clarity rather than formatting or resource limitations.",
                "meta_expalnation": "This constraint directly influences the content or output by specifying that appropriate mental health services should be connected while acknowledging certain barriers (cost, access, stigma). It does not define strategies for managing multiple constraints, like selection, prioritization, ignoring, deduplication, or composition, which are the defining traits of a Meta Constraint.",
                "evaluation_type": {
                    "constraint_type": "llm",
                    "explanation": "The constraint requires semantic understanding to identify and acknowledge barriers (cost, access, stigma), which goes beyond direct logic and involves subjective assessment of context and user needs."
                },
                "evaluation_generation_success": true
            },
            "dimension": "unconditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Does the model response connect the user with appropriate mental health services while explicitly acknowledging barriers? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "state": "modified",
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 8,
            "desc": "Please **first** provide  a 2-3 sentence **summary** of your ideas on the action based on the context provided.",
            "other_info": {
                "from": "system_para_2",
                "type_explanation": "The constraint specifies the structure and presentation of the output by requiring it to be a '2-3 sentence summary,' which governs the format and length of the response rather than its content or resource limitations.",
                "meta_expalnation": "The given constraint directly governs the model's output by specifying content requirements (a 2-3 sentence summary of assessment ideas). It does not include any rules about managing or prioritizing multiple constraints, nor does it concern high-level strategies for selecting, ignoring, or combining constraints. Therefore, it is not a Meta Constraint.",
                "evaluation_type": {
                    "constraint_type": "llm",
                    "explanation": "The constraint is subjective and requires semantic interpretation of whether the assessment summary correctly addresses the context provided. This involves open-ended understanding and cannot be validated directly or through extracting structured elements via code."
                },
                "evaluation_generation_success": false
            },
            "dimension": "unconditional",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Does the model response provide a summary firstly? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "state": "modified",
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 3,
            "desc": "</thought> 结束",
            "other_info": {
                "from": "system_para_1"
            },
            "dimension": "unconditional",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    return response.rstrip().endswith(\"</thought>\")\n"
                }
            ],
            "score": false
        },
        {
            "id": 5,
            "desc": "您可以在thought中使用python packages,但只能从以下packages列表中选择：",
            "other_info": {
                "available_packages": [
                    "math",
                    "time",
                    "stat",
                    "queue",
                    "itertools",
                    "statistics",
                    "random",
                    "collections",
                    "unicodedata"
                ]
            },
            "dimension": "unconditional",
            "type": "resource",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "提取下面文本中的所有的调用package，直接返回list格式结果，不要任何解释、前缀或赋值语句。\n输出格式要求：[\"package1\", \"package2\", ...]:\n\nHere is model response: {response}"
                },
                {
                    "type": "code",
                    "exec": "\nimport ast\n\ndef check_following(response: str) -> bool:\n    available_packages = ['math', 'time', 'stat', 'queue', 'itertools', 'statistics', 'random', 'collections', 'unicodedata']\n    try:\n        items = ast.literal_eval(response)\n        return isinstance(items, list) and all(item in available_packages for item in items)\n    except:\n        return False\n"
                }
            ],
            "score": false
        },
        {
            "id": 0,
            "desc": "Provide immediate evidence-based coping strategies tailored to their specific situation.",
            "other_info": {
                "from": "system_para_0",
                "type_explanation": "This constraint relates to ensuring that the output is meaningful, accurate, and complete, as it requires evidence-based coping strategies tailored to a specific situation. The focus is on the content being appropriate and factual, aligning with the semantic category.",
                "meta_expalnation": "This constraint directly governs output content by requiring evidence-based coping strategies tailored to the specific situation. It does not deal with the selection, prioritization, ignoring, deduplication, or combination of other constraints, which is the hallmark of a Meta Constraint.",
                "evaluation_type": {
                    "constraint_type": "llm",
                    "explanation": "Evaluating whether coping strategies are 'evidence-based' and 'tailored to their specific situation' requires semantic understanding and subjective assessment of relevance and appropriateness. This cannot be fully validated using code alone."
                },
                "evaluation_generation_success": true
            },
            "dimension": "unconditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Does the response provide immediate evidence-based coping strategies that are specifically tailored to the user's situation? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "state": "success",
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 2,
            "desc": "Connect them with appropriate mental health services while acknowledging barriers (cost, access, stigma).",
            "other_info": {
                "from": "system_para_0",
                "type_explanation": "The constraint focuses on content relevance and meaningfulness, specifying the inclusion of barriers (cost, access, stigma) while connecting individuals with mental health services. It emphasizes information completeness and logical clarity rather than formatting or resource limitations.",
                "meta_expalnation": "This constraint directly influences the content or output by specifying that appropriate mental health services should be connected while acknowledging certain barriers (cost, access, stigma). It does not define strategies for managing multiple constraints, like selection, prioritization, ignoring, deduplication, or composition, which are the defining traits of a Meta Constraint.",
                "evaluation_type": {
                    "constraint_type": "llm",
                    "explanation": "The constraint requires semantic understanding to identify and acknowledge barriers (cost, access, stigma), which goes beyond direct logic and involves subjective assessment of context and user needs."
                },
                "evaluation_generation_success": true
            },
            "dimension": "unconditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Does the model response connect the user with appropriate mental health services while explicitly acknowledging barriers? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "state": "modified",
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 3,
            "desc": "Create a concrete daily wellness plan with specific times and activities.",
            "other_info": {
                "from": "system_para_0",
                "type_explanation": "The constraint focuses on ensuring the content is meaningful and complete by requiring a daily wellness plan with specific times and activities. This falls under semantic classification as it pertains to the accuracy, logical consistency, and inclusion of specified elements in the output.",
                "meta_expalnation": "The provided constraint directly governs the task output by specifying the type of content to produce ('a concrete daily wellness plan') and its characteristics (e.g., 'with specific times and activities'). This focuses on the content of the output rather than managing the relationships or strategies for multiple constraints, so it is not a Meta Constraint.",
                "evaluation_type": {
                    "constraint_type": "llm",
                    "explanation": "human_modified"
                },
                "evaluation_generation_success": true
            },
            "dimension": "unconditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Does the response include a concrete daily wellness plan that specifies both the times and the corresponding activities? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "state": "modified",
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 4,
            "desc": "Suggest specific support communities with details on how to join.",
            "other_info": {
                "from": "system_para_0",
                "type_explanation": "The constraint focuses on ensuring the content is meaningful and complete by requiring specific details about communities and how to join them. This aligns with semantic requirements related to content accuracy, relevance, and inclusivity of key information.",
                "meta_expalnation": "The given constraint directly instructs the model to suggest support communities with details on how to join. This impacts the content of the output rather than managing or governing other constraints, which is the defining characteristic of a Meta Constraint.",
                "evaluation_type": {
                    "constraint_type": "llm",
                    "explanation": "Determining and suggesting appropriate support communities requires semantic understanding of the user's situation and preferences, which involves open-ended reasoning and context awareness. Joining details may vary greatly across communities and cannot be consistently verified purely via code logic."
                },
                "evaluation_generation_success": true
            },
            "dimension": "unconditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Does the model's response suggest specific support communities and provide clear details on how to join them? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "state": "success",
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 6,
            "desc": "Teach simple self-regulation techniques they can use immediately.",
            "other_info": {
                "from": "system_para_0",
                "type_explanation": "The constraint emphasizes meaningful content delivery by requiring simple, actionable self-regulation techniques that are immediately usable. This falls under the semantic category since it relates to the style, meaningfulness, and practical completeness of the output.",
                "meta_expalnation": "The constraint directly specifies what the model's output should include — simple self-regulation techniques that can be used immediately. This is a direct constraint on the content and not a higher-level rule for managing other constraints. It does not involve selection, prioritization, disabling, deduplication, or composition of multiple constraints.",
                "evaluation_type": {
                    "constraint_type": "llm",
                    "explanation": "Teaching self-regulation techniques requires semantic understanding, open-ended reasoning, and tailoring based on the user's situation, which cannot be directly validated using code or extracted logic."
                },
                "evaluation_generation_success": true
            },
            "dimension": "unconditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Does the response include simple self-regulation techniques that the user can use immediately? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "state": "success",
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 8,
            "desc": "Please **first** provide  a 2-3 sentence **summary** of your ideas on the action based on the context provided.",
            "other_info": {
                "from": "system_para_2",
                "type_explanation": "The constraint specifies the structure and presentation of the output by requiring it to be a '2-3 sentence summary,' which governs the format and length of the response rather than its content or resource limitations.",
                "meta_expalnation": "The given constraint directly governs the model's output by specifying content requirements (a 2-3 sentence summary of assessment ideas). It does not include any rules about managing or prioritizing multiple constraints, nor does it concern high-level strategies for selecting, ignoring, or combining constraints. Therefore, it is not a Meta Constraint.",
                "evaluation_type": {
                    "constraint_type": "llm",
                    "explanation": "The constraint is subjective and requires semantic interpretation of whether the assessment summary correctly addresses the context provided. This involves open-ended understanding and cannot be validated directly or through extracting structured elements via code."
                },
                "evaluation_generation_success": false
            },
            "dimension": "unconditional",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Does the model response provide a summary firstly? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "state": "modified",
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 16,
            "desc": "Please first provide a **2-3 sentence summary of your ideas on the action based on the context provided**.",
            "other_info": {
                "from": "system_para_2",
                "type_explanation": "The constraint specifies the structure and presentation of the output by requiring it to be a '2-3 sentence summary,' which governs the format and length of the response rather than its content or resource limitations.",
                "meta_expalnation": "The given constraint directly governs the model's output by specifying content requirements (a 2-3 sentence summary of assessment ideas). It does not include any rules about managing or prioritizing multiple constraints, nor does it concern high-level strategies for selecting, ignoring, or combining constraints. Therefore, it is not a Meta Constraint.",
                "evaluation_type": {
                    "constraint_type": "llm_assisted_code",
                    "explanation": "human_modified"
                },
                "evaluation_generation_success": false
            },
            "dimension": "unconditional",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Extract the summary where the response author presents their ideas on the action based on the given context. Return the extracted content verbatim from the response. If multiple segments are found, return them as a Python-style list of strings. If nothing is found, return an empty string (\"\").\n\nHere is the model response: {response}"
                },
                {
                    "type": "code",
                    "exec": "import re\n\ndef check_following(response: str) -> bool:\n    first_part = response.split('\\n\\n')[0] if '\\n\\n' in response else response\n    sentences = re.split('[.!?]', first_part)\n    sentences = [s.strip() for s in sentences if s.strip()]\n    return 2 <= len(sentences) <= 3"
                }
            ],
            "state": "modified",
            "score": false
        },
        {
            "id": 3,
            "desc": "始终为functions使用正确的参数",
            "other_info": {
                "available_functions": [
                    {
                        "name": "get_company_register_name",
                        "takes_inputs": {
                            "identifier": {
                                "description": "统一社会信用代码",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_company_register",
                        "takes_inputs": {
                            "identifier": {
                                "description": "公司名称",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_sub_company_info",
                        "takes_inputs": {
                            "identifier": {
                                "description": "被投资的公司名称",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_company_info",
                        "takes_inputs": {
                            "identifier": {
                                "description": "公司名称、公司简称或公司代码",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_court_code",
                        "takes_inputs": {
                            "identifier": {
                                "description": "法院名称或者法院代字",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    }
                ],
                "from": "system_para_3"
            },
            "dimension": "unconditional",
            "type": "resource",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "提取下面文本中的所有函数调用表达式，直接返回list格式结果，不要任何解释、前缀或赋值语句。\n输出格式要求：[\"function(arg1=value1)\", \"function2(arg2=value2)\", ...]\n\n示例1：\n输入文本:\nresult = search(query=\"test\")\n\n输出函数列表:\n[\"search(query=\"test\")\"]\n\n示例2:\n输入文本:\ndata = get_data(id=123)\nprocessed = process(data)\nprint(\"done\")\n\n输出函数列表:\n[\"get_data(id=123)\", \"process(data)\"]\n\n现在请处理以下文本：\n{response}\n\n输出函数列表:"
                },
                {
                    "type": "code",
                    "exec": "\nimport re\nimport ast\n\ndef check_following(response):\n\n\n    def extract_tool(text):\n        pattern = r'\\[([^]]+)\\]'\n        matches = re.findall(pattern, text)\n        if not matches:\n            return []\n        try:\n            tools = eval(f\"[{matches[-1]}]\")\n            return tools\n        except Exception as e:\n            # print(\"error in extract_tool:\", e)\n            return []\n\n    def validate_function_call(call_str, func_def):\n        \"\"\"\n        验证函数调用是否符合函数定义\n        \n        :param call_str: 函数调用字符串，如 'search(query=\"test\", recency_days=7)'\n        :param func_def: 函数定义字典，包含name和takes_inputs\n        :return: (is_valid, error_msg) 元组，表示是否有效及错误信息\n        \"\"\"\n        # 1. 提取函数名和参数部分\n        try:\n            # 使用AST安全解析函数调用\n            module = ast.parse(call_str.strip())\n            if not isinstance(module, ast.Module) or len(module.body) != 1:\n                return False, \"Invalid statement\"\n            \n            call_node = module.body[0]\n            if not isinstance(call_node, ast.Expr) or not isinstance(call_node.value, ast.Call):\n                return False, \"Not a function call\"\n            \n            # 获取函数名\n            called_func_name = None\n            if isinstance(call_node.value.func, ast.Name):\n                called_func_name = call_node.value.func.id\n            \n            # 2. 检查函数名是否匹配\n            if called_func_name != func_def['name']:\n                return False, f\"Function name mismatch. Expected '{func_def['name']}', got '{called_func_name}'\"\n            \n            # 3. 提取调用参数\n            provided_args = {}\n            for kw in call_node.value.keywords:\n                arg_name = kw.arg\n                # 获取参数值（处理基本类型的字面量）\n                if isinstance(kw.value, ast.Constant):\n                    arg_value = kw.value.value\n                elif isinstance(kw.value, ast.Num):  # Python < 3.8\n                    arg_value = kw.value.n\n                elif isinstance(kw.value, ast.Str):  # Python < 3.8\n                    arg_value = kw.value.s\n                elif isinstance(kw.value, ast.NameConstant):  # Python < 3.8\n                    arg_value = kw.value.value\n                else:\n                    arg_value = None  # 复杂表达式暂不处理\n                \n                provided_args[arg_name] = arg_value\n            \n            # 4. 验证参数\n            expected_args = func_def['takes_inputs']\n            errors = []\n            \n            # 检查必填参数\n            for arg_name, arg_def in expected_args.items():\n                if arg_def['required'] == 'True' and arg_name not in provided_args:\n                    errors.append(f\"Missing required argument: '{arg_name}'\")\n            \n            # 检查未知参数\n            for arg_name in provided_args:\n                if arg_name not in expected_args:\n                    errors.append(f\"Unexpected argument: '{arg_name}'\")\n            \n            # 检查参数类型\n            for arg_name, arg_value in provided_args.items():\n                if arg_name in expected_args:\n                    expected_type = expected_args[arg_name]['type']\n                    actual_type = type(arg_value).__name__\n                    \n                    # 简单类型检查\n                    if expected_type == 'int' and not isinstance(arg_value, int):\n                        errors.append(f\"Argument '{arg_name}' should be {expected_type}, got {actual_type}\")\n                    elif expected_type == 'str' and not isinstance(arg_value, str):\n                        errors.append(f\"Argument '{arg_name}' should be {expected_type}, got {actual_type}\")\n            \n            if errors:\n                return False, \" | \".join(errors)\n            \n            return True, \"Valid call\"\n        \n        except Exception as e:\n            return False, f\"Parsing error: {str(e)}\"\n\n    available_functions = [{'name': 'get_company_register_name', 'takes_inputs': {'identifier': {'description': '统一社会信用代码', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_company_register', 'takes_inputs': {'identifier': {'description': '公司名称', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_sub_company_info', 'takes_inputs': {'identifier': {'description': '被投资的公司名称', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_company_info', 'takes_inputs': {'identifier': {'description': '公司名称、公司简称或公司代码', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_court_code', 'takes_inputs': {'identifier': {'description': '法院名称或者法院代字', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}]\n    test_calls =  extract_tool(response)\n    # print(test_calls)\n    for call in test_calls:\n        Flag = False\n        for function_def in available_functions:\n            is_valid, msg = validate_function_call(call, function_def)\n            # print(function_def[\"name\"], msg)\n            if is_valid == True:\n                Flag = True\n                break\n        if Flag == False:\n            return False\n    return True\n"
                }
            ],
            "score": false
        },
        {
            "id": 3,
            "desc": "始终为functions使用正确的参数",
            "other_info": {
                "available_functions": [
                    {
                        "name": "get_company_register_name",
                        "takes_inputs": {
                            "identifier": {
                                "description": "统一社会信用代码",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_address_info",
                        "takes_inputs": {
                            "identifier": {
                                "description": "公司地址",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_company_register",
                        "takes_inputs": {
                            "identifier": {
                                "description": "公司名称",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_sub_company_info",
                        "takes_inputs": {
                            "identifier": {
                                "description": "被投资的公司名称",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_sub_company_info_list",
                        "takes_inputs": {
                            "identifier": {
                                "description": "母公司的名称",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    }
                ],
                "from": "system_para_3"
            },
            "dimension": "unconditional",
            "type": "resource",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "提取下面文本中的所有函数调用表达式，直接返回list格式结果，不要任何解释、前缀或赋值语句。\n输出格式要求：[\"function(arg1=value1)\", \"function2(arg2=value2)\", ...]\n\n示例1：\n输入文本:\nresult = search(query=\"test\")\n\n输出函数列表:\n[\"search(query=\"test\")\"]\n\n示例2:\n输入文本:\ndata = get_data(id=123)\nprocessed = process(data)\nprint(\"done\")\n\n输出函数列表:\n[\"get_data(id=123)\", \"process(data)\"]\n\n现在请处理以下文本：\n{response}\n\n输出函数列表:"
                },
                {
                    "type": "code",
                    "exec": "\nimport re\nimport ast\n\ndef check_following(response):\n\n\n    def extract_tool(text):\n        pattern = r'\\[([^]]+)\\]'\n        matches = re.findall(pattern, text)\n        if not matches:\n            return []\n        try:\n            tools = eval(f\"[{matches[-1]}]\")\n            return tools\n        except Exception as e:\n            # print(\"error in extract_tool:\", e)\n            return []\n\n    def validate_function_call(call_str, func_def):\n        \"\"\"\n        验证函数调用是否符合函数定义\n        \n        :param call_str: 函数调用字符串，如 'search(query=\"test\", recency_days=7)'\n        :param func_def: 函数定义字典，包含name和takes_inputs\n        :return: (is_valid, error_msg) 元组，表示是否有效及错误信息\n        \"\"\"\n        # 1. 提取函数名和参数部分\n        try:\n            # 使用AST安全解析函数调用\n            module = ast.parse(call_str.strip())\n            if not isinstance(module, ast.Module) or len(module.body) != 1:\n                return False, \"Invalid statement\"\n            \n            call_node = module.body[0]\n            if not isinstance(call_node, ast.Expr) or not isinstance(call_node.value, ast.Call):\n                return False, \"Not a function call\"\n            \n            # 获取函数名\n            called_func_name = None\n            if isinstance(call_node.value.func, ast.Name):\n                called_func_name = call_node.value.func.id\n            \n            # 2. 检查函数名是否匹配\n            if called_func_name != func_def['name']:\n                return False, f\"Function name mismatch. Expected '{func_def['name']}', got '{called_func_name}'\"\n            \n            # 3. 提取调用参数\n            provided_args = {}\n            for kw in call_node.value.keywords:\n                arg_name = kw.arg\n                # 获取参数值（处理基本类型的字面量）\n                if isinstance(kw.value, ast.Constant):\n                    arg_value = kw.value.value\n                elif isinstance(kw.value, ast.Num):  # Python < 3.8\n                    arg_value = kw.value.n\n                elif isinstance(kw.value, ast.Str):  # Python < 3.8\n                    arg_value = kw.value.s\n                elif isinstance(kw.value, ast.NameConstant):  # Python < 3.8\n                    arg_value = kw.value.value\n                else:\n                    arg_value = None  # 复杂表达式暂不处理\n                \n                provided_args[arg_name] = arg_value\n            \n            # 4. 验证参数\n            expected_args = func_def['takes_inputs']\n            errors = []\n            \n            # 检查必填参数\n            for arg_name, arg_def in expected_args.items():\n                if arg_def['required'] == 'True' and arg_name not in provided_args:\n                    errors.append(f\"Missing required argument: '{arg_name}'\")\n            \n            # 检查未知参数\n            for arg_name in provided_args:\n                if arg_name not in expected_args:\n                    errors.append(f\"Unexpected argument: '{arg_name}'\")\n            \n            # 检查参数类型\n            for arg_name, arg_value in provided_args.items():\n                if arg_name in expected_args:\n                    expected_type = expected_args[arg_name]['type']\n                    actual_type = type(arg_value).__name__\n                    \n                    # 简单类型检查\n                    if expected_type == 'int' and not isinstance(arg_value, int):\n                        errors.append(f\"Argument '{arg_name}' should be {expected_type}, got {actual_type}\")\n                    elif expected_type == 'str' and not isinstance(arg_value, str):\n                        errors.append(f\"Argument '{arg_name}' should be {expected_type}, got {actual_type}\")\n            \n            if errors:\n                return False, \" | \".join(errors)\n            \n            return True, \"Valid call\"\n        \n        except Exception as e:\n            return False, f\"Parsing error: {str(e)}\"\n\n    available_functions = [{'name': 'get_company_register_name', 'takes_inputs': {'identifier': {'description': '统一社会信用代码', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_address_info', 'takes_inputs': {'identifier': {'description': '公司地址', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_company_register', 'takes_inputs': {'identifier': {'description': '公司名称', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_sub_company_info', 'takes_inputs': {'identifier': {'description': '被投资的公司名称', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_sub_company_info_list', 'takes_inputs': {'identifier': {'description': '母公司的名称', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}]\n    test_calls =  extract_tool(response)\n    # print(test_calls)\n    for call in test_calls:\n        Flag = False\n        for function_def in available_functions:\n            is_valid, msg = validate_function_call(call, function_def)\n            # print(function_def[\"name\"], msg)\n            if is_valid == True:\n                Flag = True\n                break\n        if Flag == False:\n            return False\n    return True\n"
                }
            ],
            "score": false
        },
        {
            "id": 6,
            "desc": "立即使用</code>结束本次编码",
            "other_info": {},
            "dimension": "unconditional",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "required_keys": [
                        "response"
                    ],
                    "exec": "检查文本中每个 print 语句后是否存在 </code>（允许中间有换行或空格）。直接回答 YES 或 NO。\n文本：{response}\n回答："
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 3,
            "desc": "</thought> 结束",
            "other_info": {
                "from": "system_para_1"
            },
            "dimension": "unconditional",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    return response.rstrip().endswith(\"</thought>\")\n"
                }
            ],
            "score": false
        },
        {
            "id": 3,
            "desc": "</thought> 结束",
            "other_info": {
                "from": "system_para_1"
            },
            "dimension": "unconditional",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    return response.rstrip().endswith(\"</thought>\")\n"
                }
            ],
            "score": false
        },
        {
            "id": 3,
            "desc": "始终为functions使用正确的参数",
            "other_info": {
                "available_functions": [
                    {
                        "name": "get_company_register",
                        "takes_inputs": {
                            "identifier": {
                                "description": "公司名称",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_company_register_name",
                        "takes_inputs": {
                            "identifier": {
                                "description": "统一社会信用代码",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_company_info",
                        "takes_inputs": {
                            "identifier": {
                                "description": "公司名称、公司简称或公司代码",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_sub_company_info",
                        "takes_inputs": {
                            "identifier": {
                                "description": "被投资的公司名称",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_legal_document_company_list",
                        "takes_inputs": {
                            "identifier": {
                                "description": "关联公司名称，形如xxx有限公司",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    }
                ],
                "from": "system_para_3"
            },
            "dimension": "unconditional",
            "type": "resource",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "提取下面文本中的所有函数调用表达式，直接返回list格式结果，不要任何解释、前缀或赋值语句。\n输出格式要求：[\"function(arg1=value1)\", \"function2(arg2=value2)\", ...]\n\n示例1：\n输入文本:\nresult = search(query=\"test\")\n\n输出函数列表:\n[\"search(query=\"test\")\"]\n\n示例2:\n输入文本:\ndata = get_data(id=123)\nprocessed = process(data)\nprint(\"done\")\n\n输出函数列表:\n[\"get_data(id=123)\", \"process(data)\"]\n\n现在请处理以下文本：\n{response}\n\n输出函数列表:"
                },
                {
                    "type": "code",
                    "exec": "\nimport re\nimport ast\n\ndef check_following(response):\n\n\n    def extract_tool(text):\n        pattern = r'\\[([^]]+)\\]'\n        matches = re.findall(pattern, text)\n        if not matches:\n            return []\n        try:\n            tools = eval(f\"[{matches[-1]}]\")\n            return tools\n        except Exception as e:\n            # print(\"error in extract_tool:\", e)\n            return []\n\n    def validate_function_call(call_str, func_def):\n        \"\"\"\n        验证函数调用是否符合函数定义\n        \n        :param call_str: 函数调用字符串，如 'search(query=\"test\", recency_days=7)'\n        :param func_def: 函数定义字典，包含name和takes_inputs\n        :return: (is_valid, error_msg) 元组，表示是否有效及错误信息\n        \"\"\"\n        # 1. 提取函数名和参数部分\n        try:\n            # 使用AST安全解析函数调用\n            module = ast.parse(call_str.strip())\n            if not isinstance(module, ast.Module) or len(module.body) != 1:\n                return False, \"Invalid statement\"\n            \n            call_node = module.body[0]\n            if not isinstance(call_node, ast.Expr) or not isinstance(call_node.value, ast.Call):\n                return False, \"Not a function call\"\n            \n            # 获取函数名\n            called_func_name = None\n            if isinstance(call_node.value.func, ast.Name):\n                called_func_name = call_node.value.func.id\n            \n            # 2. 检查函数名是否匹配\n            if called_func_name != func_def['name']:\n                return False, f\"Function name mismatch. Expected '{func_def['name']}', got '{called_func_name}'\"\n            \n            # 3. 提取调用参数\n            provided_args = {}\n            for kw in call_node.value.keywords:\n                arg_name = kw.arg\n                # 获取参数值（处理基本类型的字面量）\n                if isinstance(kw.value, ast.Constant):\n                    arg_value = kw.value.value\n                elif isinstance(kw.value, ast.Num):  # Python < 3.8\n                    arg_value = kw.value.n\n                elif isinstance(kw.value, ast.Str):  # Python < 3.8\n                    arg_value = kw.value.s\n                elif isinstance(kw.value, ast.NameConstant):  # Python < 3.8\n                    arg_value = kw.value.value\n                else:\n                    arg_value = None  # 复杂表达式暂不处理\n                \n                provided_args[arg_name] = arg_value\n            \n            # 4. 验证参数\n            expected_args = func_def['takes_inputs']\n            errors = []\n            \n            # 检查必填参数\n            for arg_name, arg_def in expected_args.items():\n                if arg_def['required'] == 'True' and arg_name not in provided_args:\n                    errors.append(f\"Missing required argument: '{arg_name}'\")\n            \n            # 检查未知参数\n            for arg_name in provided_args:\n                if arg_name not in expected_args:\n                    errors.append(f\"Unexpected argument: '{arg_name}'\")\n            \n            # 检查参数类型\n            for arg_name, arg_value in provided_args.items():\n                if arg_name in expected_args:\n                    expected_type = expected_args[arg_name]['type']\n                    actual_type = type(arg_value).__name__\n                    \n                    # 简单类型检查\n                    if expected_type == 'int' and not isinstance(arg_value, int):\n                        errors.append(f\"Argument '{arg_name}' should be {expected_type}, got {actual_type}\")\n                    elif expected_type == 'str' and not isinstance(arg_value, str):\n                        errors.append(f\"Argument '{arg_name}' should be {expected_type}, got {actual_type}\")\n            \n            if errors:\n                return False, \" | \".join(errors)\n            \n            return True, \"Valid call\"\n        \n        except Exception as e:\n            return False, f\"Parsing error: {str(e)}\"\n\n    available_functions = [{'name': 'get_company_register', 'takes_inputs': {'identifier': {'description': '公司名称', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_company_register_name', 'takes_inputs': {'identifier': {'description': '统一社会信用代码', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_company_info', 'takes_inputs': {'identifier': {'description': '公司名称、公司简称或公司代码', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_sub_company_info', 'takes_inputs': {'identifier': {'description': '被投资的公司名称', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_legal_document_company_list', 'takes_inputs': {'identifier': {'description': '关联公司名称，形如xxx有限公司', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}]\n    test_calls =  extract_tool(response)\n    # print(test_calls)\n    for call in test_calls:\n        Flag = False\n        for function_def in available_functions:\n            is_valid, msg = validate_function_call(call, function_def)\n            # print(function_def[\"name\"], msg)\n            if is_valid == True:\n                Flag = True\n                break\n        if Flag == False:\n            return False\n    return True\n"
                }
            ],
            "score": false
        },
        {
            "id": 6,
            "desc": "立即使用</code>结束本次编码",
            "other_info": {},
            "dimension": "unconditional",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "required_keys": [
                        "response"
                    ],
                    "exec": "检查文本中每个 print 语句后是否存在 </code>（允许中间有换行或空格）。直接回答 YES 或 NO。\n文本：{response}\n回答："
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 8,
            "desc": "您可以在代码中使用import，但只能从以下模块列表中导入",
            "other_info": {
                "available_packages": [
                    "math",
                    "time",
                    "stat",
                    "queue",
                    "itertools",
                    "statistics",
                    "random",
                    "collections",
                    "unicodedata"
                ],
                "from": "system_para_1"
            },
            "dimension": "unconditional",
            "type": "resource",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "提取下面文本中的所有的调用package，直接返回list格式结果，不要任何解释、前缀或赋值语句。\n输出格式要求：[\"package1\", \"package2\", ...]:\n\nHere is model response: {response}"
                },
                {
                    "type": "code",
                    "exec": "\nimport ast\n\ndef check_following(response: str) -> bool:\n    available_packages = ['math', 'time', 'stat', 'queue', 'itertools', 'statistics', 'random', 'collections', 'unicodedata']\n    try:\n        items = ast.literal_eval(response)\n        return isinstance(items, list) and all(item in available_packages for item in items)\n    except:\n        return False\n"
                }
            ],
            "score": false
        },
        {
            "id": 3,
            "desc": "</thought> 结束",
            "other_info": {
                "from": "system_para_1"
            },
            "dimension": "unconditional",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    return response.rstrip().endswith(\"</thought>\")\n"
                }
            ],
            "score": false
        },
        {
            "id": 5,
            "desc": "您可以在thought中使用python packages,但只能从以下packages列表中选择：",
            "other_info": {
                "available_packages": [
                    "math",
                    "time",
                    "stat",
                    "queue",
                    "itertools",
                    "statistics",
                    "random",
                    "collections",
                    "unicodedata"
                ]
            },
            "dimension": "unconditional",
            "type": "resource",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "提取下面文本中的所有的调用package，直接返回list格式结果，不要任何解释、前缀或赋值语句。\n输出格式要求：[\"package1\", \"package2\", ...]:\n\nHere is model response: {response}"
                },
                {
                    "type": "code",
                    "exec": "\nimport ast\n\ndef check_following(response: str) -> bool:\n    available_packages = ['math', 'time', 'stat', 'queue', 'itertools', 'statistics', 'random', 'collections', 'unicodedata']\n    try:\n        items = ast.literal_eval(response)\n        return isinstance(items, list) and all(item in available_packages for item in items)\n    except:\n        return False\n"
                }
            ],
            "score": false
        },
        {
            "id": 3,
            "desc": "始终为functions使用正确的参数",
            "other_info": {
                "available_functions": [
                    {
                        "name": "get_company_register_name",
                        "takes_inputs": {
                            "identifier": {
                                "description": "统一社会信用代码",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_company_register",
                        "takes_inputs": {
                            "identifier": {
                                "description": "公司名称",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_company_info",
                        "takes_inputs": {
                            "identifier": {
                                "description": "公司名称、公司简称或公司代码",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_sub_company_info",
                        "takes_inputs": {
                            "identifier": {
                                "description": "被投资的公司名称",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_sub_company_info_list",
                        "takes_inputs": {
                            "identifier": {
                                "description": "母公司的名称",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    }
                ],
                "from": "system_para_3"
            },
            "dimension": "unconditional",
            "type": "resource",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "提取下面文本中的所有函数调用表达式，直接返回list格式结果，不要任何解释、前缀或赋值语句。\n输出格式要求：[\"function(arg1=value1)\", \"function2(arg2=value2)\", ...]\n\n示例1：\n输入文本:\nresult = search(query=\"test\")\n\n输出函数列表:\n[\"search(query=\"test\")\"]\n\n示例2:\n输入文本:\ndata = get_data(id=123)\nprocessed = process(data)\nprint(\"done\")\n\n输出函数列表:\n[\"get_data(id=123)\", \"process(data)\"]\n\n现在请处理以下文本：\n{response}\n\n输出函数列表:"
                },
                {
                    "type": "code",
                    "exec": "\nimport re\nimport ast\n\ndef check_following(response):\n\n\n    def extract_tool(text):\n        pattern = r'\\[([^]]+)\\]'\n        matches = re.findall(pattern, text)\n        if not matches:\n            return []\n        try:\n            tools = eval(f\"[{matches[-1]}]\")\n            return tools\n        except Exception as e:\n            # print(\"error in extract_tool:\", e)\n            return []\n\n    def validate_function_call(call_str, func_def):\n        \"\"\"\n        验证函数调用是否符合函数定义\n        \n        :param call_str: 函数调用字符串，如 'search(query=\"test\", recency_days=7)'\n        :param func_def: 函数定义字典，包含name和takes_inputs\n        :return: (is_valid, error_msg) 元组，表示是否有效及错误信息\n        \"\"\"\n        # 1. 提取函数名和参数部分\n        try:\n            # 使用AST安全解析函数调用\n            module = ast.parse(call_str.strip())\n            if not isinstance(module, ast.Module) or len(module.body) != 1:\n                return False, \"Invalid statement\"\n            \n            call_node = module.body[0]\n            if not isinstance(call_node, ast.Expr) or not isinstance(call_node.value, ast.Call):\n                return False, \"Not a function call\"\n            \n            # 获取函数名\n            called_func_name = None\n            if isinstance(call_node.value.func, ast.Name):\n                called_func_name = call_node.value.func.id\n            \n            # 2. 检查函数名是否匹配\n            if called_func_name != func_def['name']:\n                return False, f\"Function name mismatch. Expected '{func_def['name']}', got '{called_func_name}'\"\n            \n            # 3. 提取调用参数\n            provided_args = {}\n            for kw in call_node.value.keywords:\n                arg_name = kw.arg\n                # 获取参数值（处理基本类型的字面量）\n                if isinstance(kw.value, ast.Constant):\n                    arg_value = kw.value.value\n                elif isinstance(kw.value, ast.Num):  # Python < 3.8\n                    arg_value = kw.value.n\n                elif isinstance(kw.value, ast.Str):  # Python < 3.8\n                    arg_value = kw.value.s\n                elif isinstance(kw.value, ast.NameConstant):  # Python < 3.8\n                    arg_value = kw.value.value\n                else:\n                    arg_value = None  # 复杂表达式暂不处理\n                \n                provided_args[arg_name] = arg_value\n            \n            # 4. 验证参数\n            expected_args = func_def['takes_inputs']\n            errors = []\n            \n            # 检查必填参数\n            for arg_name, arg_def in expected_args.items():\n                if arg_def['required'] == 'True' and arg_name not in provided_args:\n                    errors.append(f\"Missing required argument: '{arg_name}'\")\n            \n            # 检查未知参数\n            for arg_name in provided_args:\n                if arg_name not in expected_args:\n                    errors.append(f\"Unexpected argument: '{arg_name}'\")\n            \n            # 检查参数类型\n            for arg_name, arg_value in provided_args.items():\n                if arg_name in expected_args:\n                    expected_type = expected_args[arg_name]['type']\n                    actual_type = type(arg_value).__name__\n                    \n                    # 简单类型检查\n                    if expected_type == 'int' and not isinstance(arg_value, int):\n                        errors.append(f\"Argument '{arg_name}' should be {expected_type}, got {actual_type}\")\n                    elif expected_type == 'str' and not isinstance(arg_value, str):\n                        errors.append(f\"Argument '{arg_name}' should be {expected_type}, got {actual_type}\")\n            \n            if errors:\n                return False, \" | \".join(errors)\n            \n            return True, \"Valid call\"\n        \n        except Exception as e:\n            return False, f\"Parsing error: {str(e)}\"\n\n    available_functions = [{'name': 'get_company_register_name', 'takes_inputs': {'identifier': {'description': '统一社会信用代码', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_company_register', 'takes_inputs': {'identifier': {'description': '公司名称', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_company_info', 'takes_inputs': {'identifier': {'description': '公司名称、公司简称或公司代码', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_sub_company_info', 'takes_inputs': {'identifier': {'description': '被投资的公司名称', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_sub_company_info_list', 'takes_inputs': {'identifier': {'description': '母公司的名称', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}]\n    test_calls =  extract_tool(response)\n    # print(test_calls)\n    for call in test_calls:\n        Flag = False\n        for function_def in available_functions:\n            is_valid, msg = validate_function_call(call, function_def)\n            # print(function_def[\"name\"], msg)\n            if is_valid == True:\n                Flag = True\n                break\n        if Flag == False:\n            return False\n    return True\n"
                }
            ],
            "score": false
        },
        {
            "id": 6,
            "desc": "立即使用</code>结束本次编码",
            "other_info": {},
            "dimension": "unconditional",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "required_keys": [
                        "response"
                    ],
                    "exec": "检查文本中每个 print 语句后是否存在 </code>（允许中间有换行或空格）。直接回答 YES 或 NO。\n文本：{response}\n回答："
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 8,
            "desc": "您可以在代码中使用import，但只能从以下模块列表中导入",
            "other_info": {
                "available_packages": [
                    "math",
                    "time",
                    "stat",
                    "queue",
                    "itertools",
                    "statistics",
                    "random",
                    "collections",
                    "unicodedata"
                ],
                "from": "system_para_1"
            },
            "dimension": "unconditional",
            "type": "resource",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "提取下面文本中的所有的调用package，直接返回list格式结果，不要任何解释、前缀或赋值语句。\n输出格式要求：[\"package1\", \"package2\", ...]:\n\nHere is model response: {response}"
                },
                {
                    "type": "code",
                    "exec": "\nimport ast\n\ndef check_following(response: str) -> bool:\n    available_packages = ['math', 'time', 'stat', 'queue', 'itertools', 'statistics', 'random', 'collections', 'unicodedata']\n    try:\n        items = ast.literal_eval(response)\n        return isinstance(items, list) and all(item in available_packages for item in items)\n    except:\n        return False\n"
                }
            ],
            "score": false
        },
        {
            "id": 3,
            "desc": "始终为functions使用正确的参数",
            "other_info": {
                "available_functions": [
                    {
                        "name": "convert_amount",
                        "takes_inputs": {
                            "num_str": {
                                "description": "需要转化的带有'千','万','亿','元'单位的数字字符串",
                                "type": "str",
                                "required": false
                            },
                            "money_unit": {
                                "description": "需要转化的单位，如'千','万','亿','元'等",
                                "type": "str",
                                "required": false
                            }
                        }
                    },
                    {
                        "name": "get_sum",
                        "takes_inputs": {
                            "nums": {
                                "description": "需要求和的list",
                                "type": "List[int] | List[float] | List[str]",
                                "required": false
                            }
                        }
                    },
                    {
                        "name": "get_company_register",
                        "takes_inputs": {
                            "identifier": {
                                "description": "公司名称",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_company_register_name",
                        "takes_inputs": {
                            "identifier": {
                                "description": "统一社会信用代码",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_sub_company_info",
                        "takes_inputs": {
                            "identifier": {
                                "description": "被投资的公司名称",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    }
                ],
                "from": "system_para_3"
            },
            "dimension": "unconditional",
            "type": "resource",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "提取下面文本中的所有函数调用表达式，直接返回list格式结果，不要任何解释、前缀或赋值语句。\n输出格式要求：[\"function(arg1=value1)\", \"function2(arg2=value2)\", ...]\n\n示例1：\n输入文本:\nresult = search(query=\"test\")\n\n输出函数列表:\n[\"search(query=\"test\")\"]\n\n示例2:\n输入文本:\ndata = get_data(id=123)\nprocessed = process(data)\nprint(\"done\")\n\n输出函数列表:\n[\"get_data(id=123)\", \"process(data)\"]\n\n现在请处理以下文本：\n{response}\n\n输出函数列表:"
                },
                {
                    "type": "code",
                    "exec": "\nimport re\nimport ast\n\ndef check_following(response):\n\n\n    def extract_tool(text):\n        pattern = r'\\[([^]]+)\\]'\n        matches = re.findall(pattern, text)\n        if not matches:\n            return []\n        try:\n            tools = eval(f\"[{matches[-1]}]\")\n            return tools\n        except Exception as e:\n            # print(\"error in extract_tool:\", e)\n            return []\n\n    def validate_function_call(call_str, func_def):\n        \"\"\"\n        验证函数调用是否符合函数定义\n        \n        :param call_str: 函数调用字符串，如 'search(query=\"test\", recency_days=7)'\n        :param func_def: 函数定义字典，包含name和takes_inputs\n        :return: (is_valid, error_msg) 元组，表示是否有效及错误信息\n        \"\"\"\n        # 1. 提取函数名和参数部分\n        try:\n            # 使用AST安全解析函数调用\n            module = ast.parse(call_str.strip())\n            if not isinstance(module, ast.Module) or len(module.body) != 1:\n                return False, \"Invalid statement\"\n            \n            call_node = module.body[0]\n            if not isinstance(call_node, ast.Expr) or not isinstance(call_node.value, ast.Call):\n                return False, \"Not a function call\"\n            \n            # 获取函数名\n            called_func_name = None\n            if isinstance(call_node.value.func, ast.Name):\n                called_func_name = call_node.value.func.id\n            \n            # 2. 检查函数名是否匹配\n            if called_func_name != func_def['name']:\n                return False, f\"Function name mismatch. Expected '{func_def['name']}', got '{called_func_name}'\"\n            \n            # 3. 提取调用参数\n            provided_args = {}\n            for kw in call_node.value.keywords:\n                arg_name = kw.arg\n                # 获取参数值（处理基本类型的字面量）\n                if isinstance(kw.value, ast.Constant):\n                    arg_value = kw.value.value\n                elif isinstance(kw.value, ast.Num):  # Python < 3.8\n                    arg_value = kw.value.n\n                elif isinstance(kw.value, ast.Str):  # Python < 3.8\n                    arg_value = kw.value.s\n                elif isinstance(kw.value, ast.NameConstant):  # Python < 3.8\n                    arg_value = kw.value.value\n                else:\n                    arg_value = None  # 复杂表达式暂不处理\n                \n                provided_args[arg_name] = arg_value\n            \n            # 4. 验证参数\n            expected_args = func_def['takes_inputs']\n            errors = []\n            \n            # 检查必填参数\n            for arg_name, arg_def in expected_args.items():\n                if arg_def['required'] == 'True' and arg_name not in provided_args:\n                    errors.append(f\"Missing required argument: '{arg_name}'\")\n            \n            # 检查未知参数\n            for arg_name in provided_args:\n                if arg_name not in expected_args:\n                    errors.append(f\"Unexpected argument: '{arg_name}'\")\n            \n            # 检查参数类型\n            for arg_name, arg_value in provided_args.items():\n                if arg_name in expected_args:\n                    expected_type = expected_args[arg_name]['type']\n                    actual_type = type(arg_value).__name__\n                    \n                    # 简单类型检查\n                    if expected_type == 'int' and not isinstance(arg_value, int):\n                        errors.append(f\"Argument '{arg_name}' should be {expected_type}, got {actual_type}\")\n                    elif expected_type == 'str' and not isinstance(arg_value, str):\n                        errors.append(f\"Argument '{arg_name}' should be {expected_type}, got {actual_type}\")\n            \n            if errors:\n                return False, \" | \".join(errors)\n            \n            return True, \"Valid call\"\n        \n        except Exception as e:\n            return False, f\"Parsing error: {str(e)}\"\n\n    available_functions = [{'name': 'convert_amount', 'takes_inputs': {'num_str': {'description': \"需要转化的带有'千','万','亿','元'单位的数字字符串\", 'type': 'str', 'required': False}, 'money_unit': {'description': \"需要转化的单位，如'千','万','亿','元'等\", 'type': 'str', 'required': False}}}, {'name': 'get_sum', 'takes_inputs': {'nums': {'description': '需要求和的list', 'type': 'List[int] | List[float] | List[str]', 'required': False}}}, {'name': 'get_company_register', 'takes_inputs': {'identifier': {'description': '公司名称', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_company_register_name', 'takes_inputs': {'identifier': {'description': '统一社会信用代码', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_sub_company_info', 'takes_inputs': {'identifier': {'description': '被投资的公司名称', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}]\n    test_calls =  extract_tool(response)\n    # print(test_calls)\n    for call in test_calls:\n        Flag = False\n        for function_def in available_functions:\n            is_valid, msg = validate_function_call(call, function_def)\n            # print(function_def[\"name\"], msg)\n            if is_valid == True:\n                Flag = True\n                break\n        if Flag == False:\n            return False\n    return True\n"
                }
            ],
            "score": false
        },
        {
            "id": 6,
            "desc": "立即使用</code>结束本次编码",
            "other_info": {},
            "dimension": "unconditional",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "required_keys": [
                        "response"
                    ],
                    "exec": "检查文本中每个 print 语句后是否存在 </code>（允许中间有换行或空格）。直接回答 YES 或 NO。\n文本：{response}\n回答："
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 3,
            "desc": "始终为functions使用正确的参数",
            "other_info": {
                "available_functions": [
                    {
                        "name": "get_sub_company_info",
                        "takes_inputs": {
                            "identifier": {
                                "description": "被投资的公司名称",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_sub_company_info_list",
                        "takes_inputs": {
                            "identifier": {
                                "description": "母公司的名称",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_company_register",
                        "takes_inputs": {
                            "identifier": {
                                "description": "公司名称",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_address_info",
                        "takes_inputs": {
                            "identifier": {
                                "description": "公司地址",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_company_register_name",
                        "takes_inputs": {
                            "identifier": {
                                "description": "统一社会信用代码",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    }
                ],
                "from": "system_para_3"
            },
            "dimension": "unconditional",
            "type": "resource",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "提取下面文本中的所有函数调用表达式，直接返回list格式结果，不要任何解释、前缀或赋值语句。\n输出格式要求：[\"function(arg1=value1)\", \"function2(arg2=value2)\", ...]\n\n示例1：\n输入文本:\nresult = search(query=\"test\")\n\n输出函数列表:\n[\"search(query=\"test\")\"]\n\n示例2:\n输入文本:\ndata = get_data(id=123)\nprocessed = process(data)\nprint(\"done\")\n\n输出函数列表:\n[\"get_data(id=123)\", \"process(data)\"]\n\n现在请处理以下文本：\n{response}\n\n输出函数列表:"
                },
                {
                    "type": "code",
                    "exec": "\nimport re\nimport ast\n\ndef check_following(response):\n\n\n    def extract_tool(text):\n        pattern = r'\\[([^]]+)\\]'\n        matches = re.findall(pattern, text)\n        if not matches:\n            return []\n        try:\n            tools = eval(f\"[{matches[-1]}]\")\n            return tools\n        except Exception as e:\n            # print(\"error in extract_tool:\", e)\n            return []\n\n    def validate_function_call(call_str, func_def):\n        \"\"\"\n        验证函数调用是否符合函数定义\n        \n        :param call_str: 函数调用字符串，如 'search(query=\"test\", recency_days=7)'\n        :param func_def: 函数定义字典，包含name和takes_inputs\n        :return: (is_valid, error_msg) 元组，表示是否有效及错误信息\n        \"\"\"\n        # 1. 提取函数名和参数部分\n        try:\n            # 使用AST安全解析函数调用\n            module = ast.parse(call_str.strip())\n            if not isinstance(module, ast.Module) or len(module.body) != 1:\n                return False, \"Invalid statement\"\n            \n            call_node = module.body[0]\n            if not isinstance(call_node, ast.Expr) or not isinstance(call_node.value, ast.Call):\n                return False, \"Not a function call\"\n            \n            # 获取函数名\n            called_func_name = None\n            if isinstance(call_node.value.func, ast.Name):\n                called_func_name = call_node.value.func.id\n            \n            # 2. 检查函数名是否匹配\n            if called_func_name != func_def['name']:\n                return False, f\"Function name mismatch. Expected '{func_def['name']}', got '{called_func_name}'\"\n            \n            # 3. 提取调用参数\n            provided_args = {}\n            for kw in call_node.value.keywords:\n                arg_name = kw.arg\n                # 获取参数值（处理基本类型的字面量）\n                if isinstance(kw.value, ast.Constant):\n                    arg_value = kw.value.value\n                elif isinstance(kw.value, ast.Num):  # Python < 3.8\n                    arg_value = kw.value.n\n                elif isinstance(kw.value, ast.Str):  # Python < 3.8\n                    arg_value = kw.value.s\n                elif isinstance(kw.value, ast.NameConstant):  # Python < 3.8\n                    arg_value = kw.value.value\n                else:\n                    arg_value = None  # 复杂表达式暂不处理\n                \n                provided_args[arg_name] = arg_value\n            \n            # 4. 验证参数\n            expected_args = func_def['takes_inputs']\n            errors = []\n            \n            # 检查必填参数\n            for arg_name, arg_def in expected_args.items():\n                if arg_def['required'] == 'True' and arg_name not in provided_args:\n                    errors.append(f\"Missing required argument: '{arg_name}'\")\n            \n            # 检查未知参数\n            for arg_name in provided_args:\n                if arg_name not in expected_args:\n                    errors.append(f\"Unexpected argument: '{arg_name}'\")\n            \n            # 检查参数类型\n            for arg_name, arg_value in provided_args.items():\n                if arg_name in expected_args:\n                    expected_type = expected_args[arg_name]['type']\n                    actual_type = type(arg_value).__name__\n                    \n                    # 简单类型检查\n                    if expected_type == 'int' and not isinstance(arg_value, int):\n                        errors.append(f\"Argument '{arg_name}' should be {expected_type}, got {actual_type}\")\n                    elif expected_type == 'str' and not isinstance(arg_value, str):\n                        errors.append(f\"Argument '{arg_name}' should be {expected_type}, got {actual_type}\")\n            \n            if errors:\n                return False, \" | \".join(errors)\n            \n            return True, \"Valid call\"\n        \n        except Exception as e:\n            return False, f\"Parsing error: {str(e)}\"\n\n    available_functions = [{'name': 'get_sub_company_info', 'takes_inputs': {'identifier': {'description': '被投资的公司名称', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_sub_company_info_list', 'takes_inputs': {'identifier': {'description': '母公司的名称', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_company_register', 'takes_inputs': {'identifier': {'description': '公司名称', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_address_info', 'takes_inputs': {'identifier': {'description': '公司地址', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_company_register_name', 'takes_inputs': {'identifier': {'description': '统一社会信用代码', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}]\n    test_calls =  extract_tool(response)\n    # print(test_calls)\n    for call in test_calls:\n        Flag = False\n        for function_def in available_functions:\n            is_valid, msg = validate_function_call(call, function_def)\n            # print(function_def[\"name\"], msg)\n            if is_valid == True:\n                Flag = True\n                break\n        if Flag == False:\n            return False\n    return True\n"
                }
            ],
            "score": false
        },
        {
            "id": 6,
            "desc": "立即使用</code>结束本次编码",
            "other_info": {},
            "dimension": "unconditional",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "required_keys": [
                        "response"
                    ],
                    "exec": "检查文本中每个 print 语句后是否存在 </code>（允许中间有换行或空格）。直接回答 YES 或 NO。\n文本：{response}\n回答："
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 1,
            "desc": "注意不要在一次thought中连续使用多个functions。",
            "other_info": {
                "function": [
                    "get_company_info",
                    "get_company_register",
                    "get_company_register_name",
                    "get_sub_company_info",
                    "get_sub_company_info_list",
                    "get_legal_document",
                    "get_legal_document_company_list",
                    "get_legal_document_law_list",
                    "get_court_info",
                    "get_court_info_list",
                    "get_court_code",
                    "get_lawfirm_info",
                    "get_lawfirm_info_list",
                    "get_lawfirm_log",
                    "get_address_info",
                    "get_legal_abstract",
                    "get_restriction_case",
                    "get_restriction_case_company_list",
                    "get_restriction_case_court_list",
                    "get_finalized_case",
                    "get_finalized_case_company_list",
                    "get_finalized_case_court_list",
                    "get_dishonesty_case",
                    "get_dishonesty_case_company_list",
                    "get_dishonesty_case_court_list",
                    "get_administrative_case",
                    "get_administrative_case_company_list",
                    "get_administrative_case_court_list",
                    "get_sum",
                    "rank",
                    "field_search",
                    "extract_case_number",
                    "extract_court_code",
                    "convert_amount",
                    "finish"
                ],
                "from": "system_para_3"
            },
            "dimension": "unconditional",
            "type": "resource",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "required_keys": [
                        "response"
                    ],
                    "exec": "请帮我提取出下面这段话中**当前步**使用的函数名称，不关注未来要做的事。\n请仅回答函数名称，并以list形式直接返回给我，不要回答其他的内容。\n\n{response}\n"
                },
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    import re\n\n    def extract_tool(text):\n        pattern = r'\\[([^]]+)\\]'\n        matches = re.findall(pattern, text)\n        if not matches:\n            return []\n        try:\n            tools = eval(f\"[{matches[-1]}]\")\n            return tools\n        except Exception as e:\n            # print(\"error in extract_tool:\", e)\n            return []\n\n    try:\n        tools = ['get_company_info', 'get_company_register', 'get_company_register_name', 'get_sub_company_info', 'get_sub_company_info_list', 'get_legal_document', 'get_legal_document_company_list', 'get_legal_document_law_list', 'get_court_info', 'get_court_info_list', 'get_court_code', 'get_lawfirm_info', 'get_lawfirm_info_list', 'get_lawfirm_log', 'get_address_info', 'get_legal_abstract', 'get_restriction_case', 'get_restriction_case_company_list', 'get_restriction_case_court_list', 'get_finalized_case', 'get_finalized_case_company_list', 'get_finalized_case_court_list', 'get_dishonesty_case', 'get_dishonesty_case_company_list', 'get_dishonesty_case_court_list', 'get_administrative_case', 'get_administrative_case_company_list', 'get_administrative_case_court_list', 'get_sum', 'rank', 'field_search', 'extract_case_number', 'extract_court_code', 'convert_amount', 'finish']\n        functions = extract_tool(response)\n        if len(functions) <= 1:\n            return True\n        return False\n    except Exception as e:\n        print(\"error in resource checker count:\", e)\n        return False\n"
                }
            ],
            "score": false
        },
        {
            "id": 3,
            "desc": "</thought> 结束",
            "other_info": {
                "from": "system_para_1"
            },
            "dimension": "unconditional",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    return response.rstrip().endswith(\"</thought>\")\n"
                }
            ],
            "score": false
        },
        {
            "id": 5,
            "desc": "您可以在thought中使用python packages,但只能从以下packages列表中选择：",
            "other_info": {
                "available_packages": [
                    "math",
                    "time",
                    "stat",
                    "queue",
                    "itertools",
                    "statistics",
                    "random",
                    "collections",
                    "unicodedata"
                ]
            },
            "dimension": "unconditional",
            "type": "resource",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "提取下面文本中的所有的调用package，直接返回list格式结果，不要任何解释、前缀或赋值语句。\n输出格式要求：[\"package1\", \"package2\", ...]:\n\nHere is model response: {response}"
                },
                {
                    "type": "code",
                    "exec": "\nimport ast\n\ndef check_following(response: str) -> bool:\n    available_packages = ['math', 'time', 'stat', 'queue', 'itertools', 'statistics', 'random', 'collections', 'unicodedata']\n    try:\n        items = ast.literal_eval(response)\n        return isinstance(items, list) and all(item in available_packages for item in items)\n    except:\n        return False\n"
                }
            ],
            "score": false
        },
        {
            "id": 3,
            "desc": "始终为functions使用正确的参数",
            "other_info": {
                "available_functions": [
                    {
                        "name": "get_company_register_name",
                        "takes_inputs": {
                            "identifier": {
                                "description": "统一社会信用代码",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_company_register",
                        "takes_inputs": {
                            "identifier": {
                                "description": "公司名称",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_sub_company_info",
                        "takes_inputs": {
                            "identifier": {
                                "description": "被投资的公司名称",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_address_info",
                        "takes_inputs": {
                            "identifier": {
                                "description": "公司地址",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_sub_company_info_list",
                        "takes_inputs": {
                            "identifier": {
                                "description": "母公司的名称",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    }
                ],
                "from": "system_para_3"
            },
            "dimension": "unconditional",
            "type": "resource",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "提取下面文本中的所有函数调用表达式，直接返回list格式结果，不要任何解释、前缀或赋值语句。\n输出格式要求：[\"function(arg1=value1)\", \"function2(arg2=value2)\", ...]\n\n示例1：\n输入文本:\nresult = search(query=\"test\")\n\n输出函数列表:\n[\"search(query=\"test\")\"]\n\n示例2:\n输入文本:\ndata = get_data(id=123)\nprocessed = process(data)\nprint(\"done\")\n\n输出函数列表:\n[\"get_data(id=123)\", \"process(data)\"]\n\n现在请处理以下文本：\n{response}\n\n输出函数列表:"
                },
                {
                    "type": "code",
                    "exec": "\nimport re\nimport ast\n\ndef check_following(response):\n\n\n    def extract_tool(text):\n        pattern = r'\\[([^]]+)\\]'\n        matches = re.findall(pattern, text)\n        if not matches:\n            return []\n        try:\n            tools = eval(f\"[{matches[-1]}]\")\n            return tools\n        except Exception as e:\n            # print(\"error in extract_tool:\", e)\n            return []\n\n    def validate_function_call(call_str, func_def):\n        \"\"\"\n        验证函数调用是否符合函数定义\n        \n        :param call_str: 函数调用字符串，如 'search(query=\"test\", recency_days=7)'\n        :param func_def: 函数定义字典，包含name和takes_inputs\n        :return: (is_valid, error_msg) 元组，表示是否有效及错误信息\n        \"\"\"\n        # 1. 提取函数名和参数部分\n        try:\n            # 使用AST安全解析函数调用\n            module = ast.parse(call_str.strip())\n            if not isinstance(module, ast.Module) or len(module.body) != 1:\n                return False, \"Invalid statement\"\n            \n            call_node = module.body[0]\n            if not isinstance(call_node, ast.Expr) or not isinstance(call_node.value, ast.Call):\n                return False, \"Not a function call\"\n            \n            # 获取函数名\n            called_func_name = None\n            if isinstance(call_node.value.func, ast.Name):\n                called_func_name = call_node.value.func.id\n            \n            # 2. 检查函数名是否匹配\n            if called_func_name != func_def['name']:\n                return False, f\"Function name mismatch. Expected '{func_def['name']}', got '{called_func_name}'\"\n            \n            # 3. 提取调用参数\n            provided_args = {}\n            for kw in call_node.value.keywords:\n                arg_name = kw.arg\n                # 获取参数值（处理基本类型的字面量）\n                if isinstance(kw.value, ast.Constant):\n                    arg_value = kw.value.value\n                elif isinstance(kw.value, ast.Num):  # Python < 3.8\n                    arg_value = kw.value.n\n                elif isinstance(kw.value, ast.Str):  # Python < 3.8\n                    arg_value = kw.value.s\n                elif isinstance(kw.value, ast.NameConstant):  # Python < 3.8\n                    arg_value = kw.value.value\n                else:\n                    arg_value = None  # 复杂表达式暂不处理\n                \n                provided_args[arg_name] = arg_value\n            \n            # 4. 验证参数\n            expected_args = func_def['takes_inputs']\n            errors = []\n            \n            # 检查必填参数\n            for arg_name, arg_def in expected_args.items():\n                if arg_def['required'] == 'True' and arg_name not in provided_args:\n                    errors.append(f\"Missing required argument: '{arg_name}'\")\n            \n            # 检查未知参数\n            for arg_name in provided_args:\n                if arg_name not in expected_args:\n                    errors.append(f\"Unexpected argument: '{arg_name}'\")\n            \n            # 检查参数类型\n            for arg_name, arg_value in provided_args.items():\n                if arg_name in expected_args:\n                    expected_type = expected_args[arg_name]['type']\n                    actual_type = type(arg_value).__name__\n                    \n                    # 简单类型检查\n                    if expected_type == 'int' and not isinstance(arg_value, int):\n                        errors.append(f\"Argument '{arg_name}' should be {expected_type}, got {actual_type}\")\n                    elif expected_type == 'str' and not isinstance(arg_value, str):\n                        errors.append(f\"Argument '{arg_name}' should be {expected_type}, got {actual_type}\")\n            \n            if errors:\n                return False, \" | \".join(errors)\n            \n            return True, \"Valid call\"\n        \n        except Exception as e:\n            return False, f\"Parsing error: {str(e)}\"\n\n    available_functions = [{'name': 'get_company_register_name', 'takes_inputs': {'identifier': {'description': '统一社会信用代码', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_company_register', 'takes_inputs': {'identifier': {'description': '公司名称', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_sub_company_info', 'takes_inputs': {'identifier': {'description': '被投资的公司名称', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_address_info', 'takes_inputs': {'identifier': {'description': '公司地址', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_sub_company_info_list', 'takes_inputs': {'identifier': {'description': '母公司的名称', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}]\n    test_calls =  extract_tool(response)\n    # print(test_calls)\n    for call in test_calls:\n        Flag = False\n        for function_def in available_functions:\n            is_valid, msg = validate_function_call(call, function_def)\n            # print(function_def[\"name\"], msg)\n            if is_valid == True:\n                Flag = True\n                break\n        if Flag == False:\n            return False\n    return True\n"
                }
            ],
            "score": false
        },
        {
            "id": 6,
            "desc": "立即使用</code>结束本次编码",
            "other_info": {},
            "dimension": "unconditional",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "required_keys": [
                        "response"
                    ],
                    "exec": "检查文本中每个 print 语句后是否存在 </code>（允许中间有换行或空格）。直接回答 YES 或 NO。\n文本：{response}\n回答："
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 3,
            "desc": "始终为functions使用正确的参数",
            "other_info": {
                "available_functions": [
                    {
                        "name": "get_sub_company_info",
                        "takes_inputs": {
                            "identifier": {
                                "description": "被投资的公司名称",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_sub_company_info_list",
                        "takes_inputs": {
                            "identifier": {
                                "description": "母公司的名称",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_company_info",
                        "takes_inputs": {
                            "identifier": {
                                "description": "公司名称、公司简称或公司代码",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_company_register",
                        "takes_inputs": {
                            "identifier": {
                                "description": "公司名称",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_legal_document_company_list",
                        "takes_inputs": {
                            "identifier": {
                                "description": "关联公司名称，形如xxx有限公司",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    }
                ],
                "from": "system_para_3"
            },
            "dimension": "unconditional",
            "type": "resource",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "提取下面文本中的所有函数调用表达式，直接返回list格式结果，不要任何解释、前缀或赋值语句。\n输出格式要求：[\"function(arg1=value1)\", \"function2(arg2=value2)\", ...]\n\n示例1：\n输入文本:\nresult = search(query=\"test\")\n\n输出函数列表:\n[\"search(query=\"test\")\"]\n\n示例2:\n输入文本:\ndata = get_data(id=123)\nprocessed = process(data)\nprint(\"done\")\n\n输出函数列表:\n[\"get_data(id=123)\", \"process(data)\"]\n\n现在请处理以下文本：\n{response}\n\n输出函数列表:"
                },
                {
                    "type": "code",
                    "exec": "\nimport re\nimport ast\n\ndef check_following(response):\n\n\n    def extract_tool(text):\n        pattern = r'\\[([^]]+)\\]'\n        matches = re.findall(pattern, text)\n        if not matches:\n            return []\n        try:\n            tools = eval(f\"[{matches[-1]}]\")\n            return tools\n        except Exception as e:\n            # print(\"error in extract_tool:\", e)\n            return []\n\n    def validate_function_call(call_str, func_def):\n        \"\"\"\n        验证函数调用是否符合函数定义\n        \n        :param call_str: 函数调用字符串，如 'search(query=\"test\", recency_days=7)'\n        :param func_def: 函数定义字典，包含name和takes_inputs\n        :return: (is_valid, error_msg) 元组，表示是否有效及错误信息\n        \"\"\"\n        # 1. 提取函数名和参数部分\n        try:\n            # 使用AST安全解析函数调用\n            module = ast.parse(call_str.strip())\n            if not isinstance(module, ast.Module) or len(module.body) != 1:\n                return False, \"Invalid statement\"\n            \n            call_node = module.body[0]\n            if not isinstance(call_node, ast.Expr) or not isinstance(call_node.value, ast.Call):\n                return False, \"Not a function call\"\n            \n            # 获取函数名\n            called_func_name = None\n            if isinstance(call_node.value.func, ast.Name):\n                called_func_name = call_node.value.func.id\n            \n            # 2. 检查函数名是否匹配\n            if called_func_name != func_def['name']:\n                return False, f\"Function name mismatch. Expected '{func_def['name']}', got '{called_func_name}'\"\n            \n            # 3. 提取调用参数\n            provided_args = {}\n            for kw in call_node.value.keywords:\n                arg_name = kw.arg\n                # 获取参数值（处理基本类型的字面量）\n                if isinstance(kw.value, ast.Constant):\n                    arg_value = kw.value.value\n                elif isinstance(kw.value, ast.Num):  # Python < 3.8\n                    arg_value = kw.value.n\n                elif isinstance(kw.value, ast.Str):  # Python < 3.8\n                    arg_value = kw.value.s\n                elif isinstance(kw.value, ast.NameConstant):  # Python < 3.8\n                    arg_value = kw.value.value\n                else:\n                    arg_value = None  # 复杂表达式暂不处理\n                \n                provided_args[arg_name] = arg_value\n            \n            # 4. 验证参数\n            expected_args = func_def['takes_inputs']\n            errors = []\n            \n            # 检查必填参数\n            for arg_name, arg_def in expected_args.items():\n                if arg_def['required'] == 'True' and arg_name not in provided_args:\n                    errors.append(f\"Missing required argument: '{arg_name}'\")\n            \n            # 检查未知参数\n            for arg_name in provided_args:\n                if arg_name not in expected_args:\n                    errors.append(f\"Unexpected argument: '{arg_name}'\")\n            \n            # 检查参数类型\n            for arg_name, arg_value in provided_args.items():\n                if arg_name in expected_args:\n                    expected_type = expected_args[arg_name]['type']\n                    actual_type = type(arg_value).__name__\n                    \n                    # 简单类型检查\n                    if expected_type == 'int' and not isinstance(arg_value, int):\n                        errors.append(f\"Argument '{arg_name}' should be {expected_type}, got {actual_type}\")\n                    elif expected_type == 'str' and not isinstance(arg_value, str):\n                        errors.append(f\"Argument '{arg_name}' should be {expected_type}, got {actual_type}\")\n            \n            if errors:\n                return False, \" | \".join(errors)\n            \n            return True, \"Valid call\"\n        \n        except Exception as e:\n            return False, f\"Parsing error: {str(e)}\"\n\n    available_functions = [{'name': 'get_sub_company_info', 'takes_inputs': {'identifier': {'description': '被投资的公司名称', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_sub_company_info_list', 'takes_inputs': {'identifier': {'description': '母公司的名称', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_company_info', 'takes_inputs': {'identifier': {'description': '公司名称、公司简称或公司代码', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_company_register', 'takes_inputs': {'identifier': {'description': '公司名称', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_legal_document_company_list', 'takes_inputs': {'identifier': {'description': '关联公司名称，形如xxx有限公司', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}]\n    test_calls =  extract_tool(response)\n    # print(test_calls)\n    for call in test_calls:\n        Flag = False\n        for function_def in available_functions:\n            is_valid, msg = validate_function_call(call, function_def)\n            # print(function_def[\"name\"], msg)\n            if is_valid == True:\n                Flag = True\n                break\n        if Flag == False:\n            return False\n    return True\n"
                }
            ],
            "score": false
        },
        {
            "id": 5,
            "desc": "每次使用预设函数后都应该将结果打印出来",
            "other_info": {
                "from": "system_para_1",
                "available_functions": [
                    {
                        "name": "get_sub_company_info",
                        "takes_inputs": {
                            "identifier": {
                                "description": "被投资的公司名称",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_sub_company_info_list",
                        "takes_inputs": {
                            "identifier": {
                                "description": "母公司的名称",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_company_info",
                        "takes_inputs": {
                            "identifier": {
                                "description": "公司名称、公司简称或公司代码",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_company_register",
                        "takes_inputs": {
                            "identifier": {
                                "description": "公司名称",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_legal_document_company_list",
                        "takes_inputs": {
                            "identifier": {
                                "description": "关联公司名称，形如xxx有限公司",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    }
                ]
            },
            "dimension": "unconditional",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "required_keys": [
                        "response"
                    ],
                    "exec": "检查下面文本中的每个函数调用表达式，判断其下一个执行语句是否为print。直接回答YES或NO，不要任何其他内容。\n文本：{response}\n回答："
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 6,
            "desc": "立即使用</code>结束本次编码",
            "other_info": {},
            "dimension": "unconditional",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "required_keys": [
                        "response"
                    ],
                    "exec": "检查文本中每个 print 语句后是否存在 </code>（允许中间有换行或空格）。直接回答 YES 或 NO。\n文本：{response}\n回答："
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 3,
            "desc": "始终为functions使用正确的参数",
            "other_info": {
                "available_functions": [
                    {
                        "name": "get_legal_document_company_list",
                        "takes_inputs": {
                            "identifier": {
                                "description": "关联公司名称，形如xxx有限公司",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_legal_document",
                        "takes_inputs": {
                            "identifier": {
                                "description": "案号",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_administrative_case_company_list",
                        "takes_inputs": {
                            "identifier": {
                                "description": "行政处罚公司名称",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_legal_document_law_list",
                        "takes_inputs": {
                            "identifier": {
                                "description": "律师事务所名称",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_finalized_case_company_list",
                        "takes_inputs": {
                            "identifier": {
                                "description": "终本公司名称",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    }
                ],
                "from": "system_para_3"
            },
            "dimension": "unconditional",
            "type": "resource",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "提取下面文本中的所有函数调用表达式，直接返回list格式结果，不要任何解释、前缀或赋值语句。\n输出格式要求：[\"function(arg1=value1)\", \"function2(arg2=value2)\", ...]\n\n示例1：\n输入文本:\nresult = search(query=\"test\")\n\n输出函数列表:\n[\"search(query=\"test\")\"]\n\n示例2:\n输入文本:\ndata = get_data(id=123)\nprocessed = process(data)\nprint(\"done\")\n\n输出函数列表:\n[\"get_data(id=123)\", \"process(data)\"]\n\n现在请处理以下文本：\n{response}\n\n输出函数列表:"
                },
                {
                    "type": "code",
                    "exec": "\nimport re\nimport ast\n\ndef check_following(response):\n\n\n    def extract_tool(text):\n        pattern = r'\\[([^]]+)\\]'\n        matches = re.findall(pattern, text)\n        if not matches:\n            return []\n        try:\n            tools = eval(f\"[{matches[-1]}]\")\n            return tools\n        except Exception as e:\n            # print(\"error in extract_tool:\", e)\n            return []\n\n    def validate_function_call(call_str, func_def):\n        \"\"\"\n        验证函数调用是否符合函数定义\n        \n        :param call_str: 函数调用字符串，如 'search(query=\"test\", recency_days=7)'\n        :param func_def: 函数定义字典，包含name和takes_inputs\n        :return: (is_valid, error_msg) 元组，表示是否有效及错误信息\n        \"\"\"\n        # 1. 提取函数名和参数部分\n        try:\n            # 使用AST安全解析函数调用\n            module = ast.parse(call_str.strip())\n            if not isinstance(module, ast.Module) or len(module.body) != 1:\n                return False, \"Invalid statement\"\n            \n            call_node = module.body[0]\n            if not isinstance(call_node, ast.Expr) or not isinstance(call_node.value, ast.Call):\n                return False, \"Not a function call\"\n            \n            # 获取函数名\n            called_func_name = None\n            if isinstance(call_node.value.func, ast.Name):\n                called_func_name = call_node.value.func.id\n            \n            # 2. 检查函数名是否匹配\n            if called_func_name != func_def['name']:\n                return False, f\"Function name mismatch. Expected '{func_def['name']}', got '{called_func_name}'\"\n            \n            # 3. 提取调用参数\n            provided_args = {}\n            for kw in call_node.value.keywords:\n                arg_name = kw.arg\n                # 获取参数值（处理基本类型的字面量）\n                if isinstance(kw.value, ast.Constant):\n                    arg_value = kw.value.value\n                elif isinstance(kw.value, ast.Num):  # Python < 3.8\n                    arg_value = kw.value.n\n                elif isinstance(kw.value, ast.Str):  # Python < 3.8\n                    arg_value = kw.value.s\n                elif isinstance(kw.value, ast.NameConstant):  # Python < 3.8\n                    arg_value = kw.value.value\n                else:\n                    arg_value = None  # 复杂表达式暂不处理\n                \n                provided_args[arg_name] = arg_value\n            \n            # 4. 验证参数\n            expected_args = func_def['takes_inputs']\n            errors = []\n            \n            # 检查必填参数\n            for arg_name, arg_def in expected_args.items():\n                if arg_def['required'] == 'True' and arg_name not in provided_args:\n                    errors.append(f\"Missing required argument: '{arg_name}'\")\n            \n            # 检查未知参数\n            for arg_name in provided_args:\n                if arg_name not in expected_args:\n                    errors.append(f\"Unexpected argument: '{arg_name}'\")\n            \n            # 检查参数类型\n            for arg_name, arg_value in provided_args.items():\n                if arg_name in expected_args:\n                    expected_type = expected_args[arg_name]['type']\n                    actual_type = type(arg_value).__name__\n                    \n                    # 简单类型检查\n                    if expected_type == 'int' and not isinstance(arg_value, int):\n                        errors.append(f\"Argument '{arg_name}' should be {expected_type}, got {actual_type}\")\n                    elif expected_type == 'str' and not isinstance(arg_value, str):\n                        errors.append(f\"Argument '{arg_name}' should be {expected_type}, got {actual_type}\")\n            \n            if errors:\n                return False, \" | \".join(errors)\n            \n            return True, \"Valid call\"\n        \n        except Exception as e:\n            return False, f\"Parsing error: {str(e)}\"\n\n    available_functions = [{'name': 'get_legal_document_company_list', 'takes_inputs': {'identifier': {'description': '关联公司名称，形如xxx有限公司', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_legal_document', 'takes_inputs': {'identifier': {'description': '案号', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_administrative_case_company_list', 'takes_inputs': {'identifier': {'description': '行政处罚公司名称', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_legal_document_law_list', 'takes_inputs': {'identifier': {'description': '律师事务所名称', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_finalized_case_company_list', 'takes_inputs': {'identifier': {'description': '终本公司名称', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}]\n    test_calls =  extract_tool(response)\n    # print(test_calls)\n    for call in test_calls:\n        Flag = False\n        for function_def in available_functions:\n            is_valid, msg = validate_function_call(call, function_def)\n            # print(function_def[\"name\"], msg)\n            if is_valid == True:\n                Flag = True\n                break\n        if Flag == False:\n            return False\n    return True\n"
                }
            ],
            "score": false
        },
        {
            "id": 6,
            "desc": "立即使用</code>结束本次编码",
            "other_info": {},
            "dimension": "unconditional",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "required_keys": [
                        "response"
                    ],
                    "exec": "检查文本中每个 print 语句后是否存在 </code>（允许中间有换行或空格）。直接回答 YES 或 NO。\n文本：{response}\n回答："
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 8,
            "desc": "您可以在代码中使用import，但只能从以下模块列表中导入",
            "other_info": {
                "available_packages": [
                    "math",
                    "time",
                    "stat",
                    "queue",
                    "itertools",
                    "statistics",
                    "random",
                    "collections",
                    "unicodedata"
                ],
                "from": "system_para_1"
            },
            "dimension": "unconditional",
            "type": "resource",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "提取下面文本中的所有的调用package，直接返回list格式结果，不要任何解释、前缀或赋值语句。\n输出格式要求：[\"package1\", \"package2\", ...]:\n\nHere is model response: {response}"
                },
                {
                    "type": "code",
                    "exec": "\nimport ast\n\ndef check_following(response: str) -> bool:\n    available_packages = ['math', 'time', 'stat', 'queue', 'itertools', 'statistics', 'random', 'collections', 'unicodedata']\n    try:\n        items = ast.literal_eval(response)\n        return isinstance(items, list) and all(item in available_packages for item in items)\n    except:\n        return False\n"
                }
            ],
            "score": false
        },
        {
            "id": 3,
            "desc": "始终为functions使用正确的参数",
            "other_info": {
                "available_functions": [
                    {
                        "name": "convert_amount",
                        "takes_inputs": {
                            "num_str": {
                                "description": "需要转化的带有'千','万','亿','元'单位的数字字符串",
                                "type": "str",
                                "required": false
                            },
                            "money_unit": {
                                "description": "需要转化的单位，如'千','万','亿','元'等",
                                "type": "str",
                                "required": false
                            }
                        }
                    },
                    {
                        "name": "get_company_register",
                        "takes_inputs": {
                            "identifier": {
                                "description": "公司名称",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_sum",
                        "takes_inputs": {
                            "nums": {
                                "description": "需要求和的list",
                                "type": "List[int] | List[float] | List[str]",
                                "required": false
                            }
                        }
                    },
                    {
                        "name": "get_sub_company_info",
                        "takes_inputs": {
                            "identifier": {
                                "description": "被投资的公司名称",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_company_register_name",
                        "takes_inputs": {
                            "identifier": {
                                "description": "统一社会信用代码",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    }
                ],
                "from": "system_para_3"
            },
            "dimension": "unconditional",
            "type": "resource",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "提取下面文本中的所有函数调用表达式，直接返回list格式结果，不要任何解释、前缀或赋值语句。\n输出格式要求：[\"function(arg1=value1)\", \"function2(arg2=value2)\", ...]\n\n示例1：\n输入文本:\nresult = search(query=\"test\")\n\n输出函数列表:\n[\"search(query=\"test\")\"]\n\n示例2:\n输入文本:\ndata = get_data(id=123)\nprocessed = process(data)\nprint(\"done\")\n\n输出函数列表:\n[\"get_data(id=123)\", \"process(data)\"]\n\n现在请处理以下文本：\n{response}\n\n输出函数列表:"
                },
                {
                    "type": "code",
                    "exec": "\nimport re\nimport ast\n\ndef check_following(response):\n\n\n    def extract_tool(text):\n        pattern = r'\\[([^]]+)\\]'\n        matches = re.findall(pattern, text)\n        if not matches:\n            return []\n        try:\n            tools = eval(f\"[{matches[-1]}]\")\n            return tools\n        except Exception as e:\n            # print(\"error in extract_tool:\", e)\n            return []\n\n    def validate_function_call(call_str, func_def):\n        \"\"\"\n        验证函数调用是否符合函数定义\n        \n        :param call_str: 函数调用字符串，如 'search(query=\"test\", recency_days=7)'\n        :param func_def: 函数定义字典，包含name和takes_inputs\n        :return: (is_valid, error_msg) 元组，表示是否有效及错误信息\n        \"\"\"\n        # 1. 提取函数名和参数部分\n        try:\n            # 使用AST安全解析函数调用\n            module = ast.parse(call_str.strip())\n            if not isinstance(module, ast.Module) or len(module.body) != 1:\n                return False, \"Invalid statement\"\n            \n            call_node = module.body[0]\n            if not isinstance(call_node, ast.Expr) or not isinstance(call_node.value, ast.Call):\n                return False, \"Not a function call\"\n            \n            # 获取函数名\n            called_func_name = None\n            if isinstance(call_node.value.func, ast.Name):\n                called_func_name = call_node.value.func.id\n            \n            # 2. 检查函数名是否匹配\n            if called_func_name != func_def['name']:\n                return False, f\"Function name mismatch. Expected '{func_def['name']}', got '{called_func_name}'\"\n            \n            # 3. 提取调用参数\n            provided_args = {}\n            for kw in call_node.value.keywords:\n                arg_name = kw.arg\n                # 获取参数值（处理基本类型的字面量）\n                if isinstance(kw.value, ast.Constant):\n                    arg_value = kw.value.value\n                elif isinstance(kw.value, ast.Num):  # Python < 3.8\n                    arg_value = kw.value.n\n                elif isinstance(kw.value, ast.Str):  # Python < 3.8\n                    arg_value = kw.value.s\n                elif isinstance(kw.value, ast.NameConstant):  # Python < 3.8\n                    arg_value = kw.value.value\n                else:\n                    arg_value = None  # 复杂表达式暂不处理\n                \n                provided_args[arg_name] = arg_value\n            \n            # 4. 验证参数\n            expected_args = func_def['takes_inputs']\n            errors = []\n            \n            # 检查必填参数\n            for arg_name, arg_def in expected_args.items():\n                if arg_def['required'] == 'True' and arg_name not in provided_args:\n                    errors.append(f\"Missing required argument: '{arg_name}'\")\n            \n            # 检查未知参数\n            for arg_name in provided_args:\n                if arg_name not in expected_args:\n                    errors.append(f\"Unexpected argument: '{arg_name}'\")\n            \n            # 检查参数类型\n            for arg_name, arg_value in provided_args.items():\n                if arg_name in expected_args:\n                    expected_type = expected_args[arg_name]['type']\n                    actual_type = type(arg_value).__name__\n                    \n                    # 简单类型检查\n                    if expected_type == 'int' and not isinstance(arg_value, int):\n                        errors.append(f\"Argument '{arg_name}' should be {expected_type}, got {actual_type}\")\n                    elif expected_type == 'str' and not isinstance(arg_value, str):\n                        errors.append(f\"Argument '{arg_name}' should be {expected_type}, got {actual_type}\")\n            \n            if errors:\n                return False, \" | \".join(errors)\n            \n            return True, \"Valid call\"\n        \n        except Exception as e:\n            return False, f\"Parsing error: {str(e)}\"\n\n    available_functions = [{'name': 'convert_amount', 'takes_inputs': {'num_str': {'description': \"需要转化的带有'千','万','亿','元'单位的数字字符串\", 'type': 'str', 'required': False}, 'money_unit': {'description': \"需要转化的单位，如'千','万','亿','元'等\", 'type': 'str', 'required': False}}}, {'name': 'get_company_register', 'takes_inputs': {'identifier': {'description': '公司名称', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_sum', 'takes_inputs': {'nums': {'description': '需要求和的list', 'type': 'List[int] | List[float] | List[str]', 'required': False}}}, {'name': 'get_sub_company_info', 'takes_inputs': {'identifier': {'description': '被投资的公司名称', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_company_register_name', 'takes_inputs': {'identifier': {'description': '统一社会信用代码', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}]\n    test_calls =  extract_tool(response)\n    # print(test_calls)\n    for call in test_calls:\n        Flag = False\n        for function_def in available_functions:\n            is_valid, msg = validate_function_call(call, function_def)\n            # print(function_def[\"name\"], msg)\n            if is_valid == True:\n                Flag = True\n                break\n        if Flag == False:\n            return False\n    return True\n"
                }
            ],
            "score": false
        },
        {
            "id": 6,
            "desc": "立即使用</code>结束本次编码",
            "other_info": {},
            "dimension": "unconditional",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "required_keys": [
                        "response"
                    ],
                    "exec": "检查文本中每个 print 语句后是否存在 </code>（允许中间有换行或空格）。直接回答 YES 或 NO。\n文本：{response}\n回答："
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 0,
            "desc": "您可以使用一系列functions。",
            "other_info": {
                "function": [
                    "get_sub_company_info",
                    "get_company_info",
                    "get_company_register",
                    "get_sub_company_info_list",
                    "get_company_register_name",
                    "finish"
                ],
                "from": "system_para_1"
            },
            "dimension": "unconditional",
            "type": "resource",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "required_keys": [
                        "response"
                    ],
                    "exec": "请帮我提取出下面这段话中**当前步**使用的函数名称，不关注未来要做的事。\n请仅回答函数名称，并以list形式直接返回给我，不要回答其他的内容。\n\n{response}\n"
                },
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    import re\n\n    def extract_tool(text):\n        pattern = r'\\[([^]]+)\\]'\n        matches = re.findall(pattern, text)\n        if not matches:\n            return []\n        try:\n            tools = eval(f\"[{matches[-1]}]\")\n            return tools\n        except Exception as e:\n            # print(\"error in extract_tool:\", e)\n            return []\n\n    try:\n        tools = ['get_sub_company_info', 'get_company_info', 'get_company_register', 'get_sub_company_info_list', 'get_company_register_name', 'finish']\n        functions = extract_tool(response)\n        if isinstance(functions, list):\n            for item in functions:\n                if item not in tools:\n                    return False\n        return True\n    except Exception as e:\n        print(\"error in resource checker:\", e)\n        return False\n"
                }
            ],
            "score": false
        },
        {
            "id": 1,
            "desc": "注意不要在一次thought中连续使用多个functions。",
            "other_info": {
                "function": [
                    "get_sub_company_info",
                    "get_company_info",
                    "get_company_register",
                    "get_sub_company_info_list",
                    "get_company_register_name",
                    "finish"
                ],
                "from": "system_para_3"
            },
            "dimension": "unconditional",
            "type": "resource",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "required_keys": [
                        "response"
                    ],
                    "exec": "请帮我提取出下面这段话中**当前步**使用的函数名称，不关注未来要做的事。\n请仅回答函数名称，并以list形式直接返回给我，不要回答其他的内容。\n\n{response}\n"
                },
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    import re\n\n    def extract_tool(text):\n        pattern = r'\\[([^]]+)\\]'\n        matches = re.findall(pattern, text)\n        if not matches:\n            return []\n        try:\n            tools = eval(f\"[{matches[-1]}]\")\n            return tools\n        except Exception as e:\n            # print(\"error in extract_tool:\", e)\n            return []\n\n    try:\n        tools = ['get_sub_company_info', 'get_company_info', 'get_company_register', 'get_sub_company_info_list', 'get_company_register_name', 'finish']\n        functions = extract_tool(response)\n        if len(functions) <= 1:\n            return True\n        return False\n    except Exception as e:\n        print(\"error in resource checker count:\", e)\n        return False\n"
                }
            ],
            "score": false
        },
        {
            "id": 3,
            "desc": "</thought> 结束",
            "other_info": {
                "from": "system_para_1"
            },
            "dimension": "unconditional",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    return response.rstrip().endswith(\"</thought>\")\n"
                }
            ],
            "score": false
        },
        {
            "id": 5,
            "desc": "您可以在thought中使用python packages,但只能从以下packages列表中选择：",
            "other_info": {
                "available_packages": [
                    "math",
                    "time",
                    "stat",
                    "queue",
                    "itertools",
                    "statistics",
                    "random",
                    "collections",
                    "unicodedata"
                ]
            },
            "dimension": "unconditional",
            "type": "resource",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "提取下面文本中的所有的调用package，直接返回list格式结果，不要任何解释、前缀或赋值语句。\n输出格式要求：[\"package1\", \"package2\", ...]:\n\nHere is model response: {response}"
                },
                {
                    "type": "code",
                    "exec": "\nimport ast\n\ndef check_following(response: str) -> bool:\n    available_packages = ['math', 'time', 'stat', 'queue', 'itertools', 'statistics', 'random', 'collections', 'unicodedata']\n    try:\n        items = ast.literal_eval(response)\n        return isinstance(items, list) and all(item in available_packages for item in items)\n    except:\n        return False\n"
                }
            ],
            "score": false
        },
        {
            "id": 4,
            "desc": "除了第一轮外，对上一轮的<observation></observation>进行总结。",
            "other_info": {
                "from": "system_para_2"
            },
            "dimension": "unconditional",
            "type": "semantic",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "response是否在一开始就总结了上一轮的<observation></observation>？直接回答 YES 或 NO。\n\nresponse: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 3,
            "desc": "始终为functions使用正确的参数",
            "other_info": {
                "available_functions": [
                    {
                        "name": "get_legal_document_company_list",
                        "takes_inputs": {
                            "identifier": {
                                "description": "关联公司名称，形如xxx有限公司",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_legal_document",
                        "takes_inputs": {
                            "identifier": {
                                "description": "案号",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_legal_abstract",
                        "takes_inputs": {
                            "identifier": {
                                "description": "案号",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_restriction_case",
                        "takes_inputs": {
                            "identifier": {
                                "description": "案号",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_company_register",
                        "takes_inputs": {
                            "identifier": {
                                "description": "公司名称",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    }
                ],
                "from": "system_para_3"
            },
            "dimension": "unconditional",
            "type": "resource",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "提取下面文本中的所有函数调用表达式，直接返回list格式结果，不要任何解释、前缀或赋值语句。\n输出格式要求：[\"function(arg1=value1)\", \"function2(arg2=value2)\", ...]\n\n示例1：\n输入文本:\nresult = search(query=\"test\")\n\n输出函数列表:\n[\"search(query=\"test\")\"]\n\n示例2:\n输入文本:\ndata = get_data(id=123)\nprocessed = process(data)\nprint(\"done\")\n\n输出函数列表:\n[\"get_data(id=123)\", \"process(data)\"]\n\n现在请处理以下文本：\n{response}\n\n输出函数列表:"
                },
                {
                    "type": "code",
                    "exec": "\nimport re\nimport ast\n\ndef check_following(response):\n\n\n    def extract_tool(text):\n        pattern = r'\\[([^]]+)\\]'\n        matches = re.findall(pattern, text)\n        if not matches:\n            return []\n        try:\n            tools = eval(f\"[{matches[-1]}]\")\n            return tools\n        except Exception as e:\n            # print(\"error in extract_tool:\", e)\n            return []\n\n    def validate_function_call(call_str, func_def):\n        \"\"\"\n        验证函数调用是否符合函数定义\n        \n        :param call_str: 函数调用字符串，如 'search(query=\"test\", recency_days=7)'\n        :param func_def: 函数定义字典，包含name和takes_inputs\n        :return: (is_valid, error_msg) 元组，表示是否有效及错误信息\n        \"\"\"\n        # 1. 提取函数名和参数部分\n        try:\n            # 使用AST安全解析函数调用\n            module = ast.parse(call_str.strip())\n            if not isinstance(module, ast.Module) or len(module.body) != 1:\n                return False, \"Invalid statement\"\n            \n            call_node = module.body[0]\n            if not isinstance(call_node, ast.Expr) or not isinstance(call_node.value, ast.Call):\n                return False, \"Not a function call\"\n            \n            # 获取函数名\n            called_func_name = None\n            if isinstance(call_node.value.func, ast.Name):\n                called_func_name = call_node.value.func.id\n            \n            # 2. 检查函数名是否匹配\n            if called_func_name != func_def['name']:\n                return False, f\"Function name mismatch. Expected '{func_def['name']}', got '{called_func_name}'\"\n            \n            # 3. 提取调用参数\n            provided_args = {}\n            for kw in call_node.value.keywords:\n                arg_name = kw.arg\n                # 获取参数值（处理基本类型的字面量）\n                if isinstance(kw.value, ast.Constant):\n                    arg_value = kw.value.value\n                elif isinstance(kw.value, ast.Num):  # Python < 3.8\n                    arg_value = kw.value.n\n                elif isinstance(kw.value, ast.Str):  # Python < 3.8\n                    arg_value = kw.value.s\n                elif isinstance(kw.value, ast.NameConstant):  # Python < 3.8\n                    arg_value = kw.value.value\n                else:\n                    arg_value = None  # 复杂表达式暂不处理\n                \n                provided_args[arg_name] = arg_value\n            \n            # 4. 验证参数\n            expected_args = func_def['takes_inputs']\n            errors = []\n            \n            # 检查必填参数\n            for arg_name, arg_def in expected_args.items():\n                if arg_def['required'] == 'True' and arg_name not in provided_args:\n                    errors.append(f\"Missing required argument: '{arg_name}'\")\n            \n            # 检查未知参数\n            for arg_name in provided_args:\n                if arg_name not in expected_args:\n                    errors.append(f\"Unexpected argument: '{arg_name}'\")\n            \n            # 检查参数类型\n            for arg_name, arg_value in provided_args.items():\n                if arg_name in expected_args:\n                    expected_type = expected_args[arg_name]['type']\n                    actual_type = type(arg_value).__name__\n                    \n                    # 简单类型检查\n                    if expected_type == 'int' and not isinstance(arg_value, int):\n                        errors.append(f\"Argument '{arg_name}' should be {expected_type}, got {actual_type}\")\n                    elif expected_type == 'str' and not isinstance(arg_value, str):\n                        errors.append(f\"Argument '{arg_name}' should be {expected_type}, got {actual_type}\")\n            \n            if errors:\n                return False, \" | \".join(errors)\n            \n            return True, \"Valid call\"\n        \n        except Exception as e:\n            return False, f\"Parsing error: {str(e)}\"\n\n    available_functions = [{'name': 'get_legal_document_company_list', 'takes_inputs': {'identifier': {'description': '关联公司名称，形如xxx有限公司', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_legal_document', 'takes_inputs': {'identifier': {'description': '案号', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_legal_abstract', 'takes_inputs': {'identifier': {'description': '案号', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_restriction_case', 'takes_inputs': {'identifier': {'description': '案号', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_company_register', 'takes_inputs': {'identifier': {'description': '公司名称', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}]\n    test_calls =  extract_tool(response)\n    # print(test_calls)\n    for call in test_calls:\n        Flag = False\n        for function_def in available_functions:\n            is_valid, msg = validate_function_call(call, function_def)\n            # print(function_def[\"name\"], msg)\n            if is_valid == True:\n                Flag = True\n                break\n        if Flag == False:\n            return False\n    return True\n"
                }
            ],
            "score": false
        },
        {
            "id": 3,
            "desc": "</thought> 结束",
            "other_info": {
                "from": "system_para_1"
            },
            "dimension": "unconditional",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    return response.rstrip().endswith(\"</thought>\")\n"
                }
            ],
            "score": false
        },
        {
            "id": 3,
            "desc": "始终为functions使用正确的参数",
            "other_info": {
                "available_functions": [
                    {
                        "name": "get_company_register",
                        "takes_inputs": {
                            "identifier": {
                                "description": "公司名称",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_address_info",
                        "takes_inputs": {
                            "identifier": {
                                "description": "公司地址",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_company_register_name",
                        "takes_inputs": {
                            "identifier": {
                                "description": "统一社会信用代码",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_company_info",
                        "takes_inputs": {
                            "identifier": {
                                "description": "公司名称、公司简称或公司代码",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_sub_company_info_list",
                        "takes_inputs": {
                            "identifier": {
                                "description": "母公司的名称",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    }
                ],
                "from": "system_para_3"
            },
            "dimension": "unconditional",
            "type": "resource",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "提取下面文本中的所有函数调用表达式，直接返回list格式结果，不要任何解释、前缀或赋值语句。\n输出格式要求：[\"function(arg1=value1)\", \"function2(arg2=value2)\", ...]\n\n示例1：\n输入文本:\nresult = search(query=\"test\")\n\n输出函数列表:\n[\"search(query=\"test\")\"]\n\n示例2:\n输入文本:\ndata = get_data(id=123)\nprocessed = process(data)\nprint(\"done\")\n\n输出函数列表:\n[\"get_data(id=123)\", \"process(data)\"]\n\n现在请处理以下文本：\n{response}\n\n输出函数列表:"
                },
                {
                    "type": "code",
                    "exec": "\nimport re\nimport ast\n\ndef check_following(response):\n\n\n    def extract_tool(text):\n        pattern = r'\\[([^]]+)\\]'\n        matches = re.findall(pattern, text)\n        if not matches:\n            return []\n        try:\n            tools = eval(f\"[{matches[-1]}]\")\n            return tools\n        except Exception as e:\n            # print(\"error in extract_tool:\", e)\n            return []\n\n    def validate_function_call(call_str, func_def):\n        \"\"\"\n        验证函数调用是否符合函数定义\n        \n        :param call_str: 函数调用字符串，如 'search(query=\"test\", recency_days=7)'\n        :param func_def: 函数定义字典，包含name和takes_inputs\n        :return: (is_valid, error_msg) 元组，表示是否有效及错误信息\n        \"\"\"\n        # 1. 提取函数名和参数部分\n        try:\n            # 使用AST安全解析函数调用\n            module = ast.parse(call_str.strip())\n            if not isinstance(module, ast.Module) or len(module.body) != 1:\n                return False, \"Invalid statement\"\n            \n            call_node = module.body[0]\n            if not isinstance(call_node, ast.Expr) or not isinstance(call_node.value, ast.Call):\n                return False, \"Not a function call\"\n            \n            # 获取函数名\n            called_func_name = None\n            if isinstance(call_node.value.func, ast.Name):\n                called_func_name = call_node.value.func.id\n            \n            # 2. 检查函数名是否匹配\n            if called_func_name != func_def['name']:\n                return False, f\"Function name mismatch. Expected '{func_def['name']}', got '{called_func_name}'\"\n            \n            # 3. 提取调用参数\n            provided_args = {}\n            for kw in call_node.value.keywords:\n                arg_name = kw.arg\n                # 获取参数值（处理基本类型的字面量）\n                if isinstance(kw.value, ast.Constant):\n                    arg_value = kw.value.value\n                elif isinstance(kw.value, ast.Num):  # Python < 3.8\n                    arg_value = kw.value.n\n                elif isinstance(kw.value, ast.Str):  # Python < 3.8\n                    arg_value = kw.value.s\n                elif isinstance(kw.value, ast.NameConstant):  # Python < 3.8\n                    arg_value = kw.value.value\n                else:\n                    arg_value = None  # 复杂表达式暂不处理\n                \n                provided_args[arg_name] = arg_value\n            \n            # 4. 验证参数\n            expected_args = func_def['takes_inputs']\n            errors = []\n            \n            # 检查必填参数\n            for arg_name, arg_def in expected_args.items():\n                if arg_def['required'] == 'True' and arg_name not in provided_args:\n                    errors.append(f\"Missing required argument: '{arg_name}'\")\n            \n            # 检查未知参数\n            for arg_name in provided_args:\n                if arg_name not in expected_args:\n                    errors.append(f\"Unexpected argument: '{arg_name}'\")\n            \n            # 检查参数类型\n            for arg_name, arg_value in provided_args.items():\n                if arg_name in expected_args:\n                    expected_type = expected_args[arg_name]['type']\n                    actual_type = type(arg_value).__name__\n                    \n                    # 简单类型检查\n                    if expected_type == 'int' and not isinstance(arg_value, int):\n                        errors.append(f\"Argument '{arg_name}' should be {expected_type}, got {actual_type}\")\n                    elif expected_type == 'str' and not isinstance(arg_value, str):\n                        errors.append(f\"Argument '{arg_name}' should be {expected_type}, got {actual_type}\")\n            \n            if errors:\n                return False, \" | \".join(errors)\n            \n            return True, \"Valid call\"\n        \n        except Exception as e:\n            return False, f\"Parsing error: {str(e)}\"\n\n    available_functions = [{'name': 'get_company_register', 'takes_inputs': {'identifier': {'description': '公司名称', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_address_info', 'takes_inputs': {'identifier': {'description': '公司地址', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_company_register_name', 'takes_inputs': {'identifier': {'description': '统一社会信用代码', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_company_info', 'takes_inputs': {'identifier': {'description': '公司名称、公司简称或公司代码', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_sub_company_info_list', 'takes_inputs': {'identifier': {'description': '母公司的名称', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}]\n    test_calls =  extract_tool(response)\n    # print(test_calls)\n    for call in test_calls:\n        Flag = False\n        for function_def in available_functions:\n            is_valid, msg = validate_function_call(call, function_def)\n            # print(function_def[\"name\"], msg)\n            if is_valid == True:\n                Flag = True\n                break\n        if Flag == False:\n            return False\n    return True\n"
                }
            ],
            "score": false
        },
        {
            "id": 6,
            "desc": "立即使用</code>结束本次编码",
            "other_info": {},
            "dimension": "unconditional",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "required_keys": [
                        "response"
                    ],
                    "exec": "检查文本中每个 print 语句后是否存在 </code>（允许中间有换行或空格）。直接回答 YES 或 NO。\n文本：{response}\n回答："
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 8,
            "desc": "您可以在代码中使用import，但只能从以下模块列表中导入",
            "other_info": {
                "available_packages": [
                    "math",
                    "time",
                    "stat",
                    "queue",
                    "itertools",
                    "statistics",
                    "random",
                    "collections",
                    "unicodedata"
                ],
                "from": "system_para_1"
            },
            "dimension": "unconditional",
            "type": "resource",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "提取下面文本中的所有的调用package，直接返回list格式结果，不要任何解释、前缀或赋值语句。\n输出格式要求：[\"package1\", \"package2\", ...]:\n\nHere is model response: {response}"
                },
                {
                    "type": "code",
                    "exec": "\nimport ast\n\ndef check_following(response: str) -> bool:\n    available_packages = ['math', 'time', 'stat', 'queue', 'itertools', 'statistics', 'random', 'collections', 'unicodedata']\n    try:\n        items = ast.literal_eval(response)\n        return isinstance(items, list) and all(item in available_packages for item in items)\n    except:\n        return False\n"
                }
            ],
            "score": false
        },
        {
            "id": 3,
            "desc": "始终为functions使用正确的参数",
            "other_info": {
                "available_functions": [
                    {
                        "name": "extract_case_number",
                        "takes_inputs": {
                            "text": {
                                "description": "包含案件号的文本",
                                "type": "str",
                                "required": false
                            }
                        }
                    },
                    {
                        "name": "get_restriction_case",
                        "takes_inputs": {
                            "identifier": {
                                "description": "案号",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_legal_document_company_list",
                        "takes_inputs": {
                            "identifier": {
                                "description": "关联公司名称，形如xxx有限公司",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_legal_abstract",
                        "takes_inputs": {
                            "identifier": {
                                "description": "案号",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_finalized_case",
                        "takes_inputs": {
                            "identifier": {
                                "description": "案号",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    }
                ],
                "from": "system_para_3"
            },
            "dimension": "unconditional",
            "type": "resource",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "提取下面文本中的所有函数调用表达式，直接返回list格式结果，不要任何解释、前缀或赋值语句。\n输出格式要求：[\"function(arg1=value1)\", \"function2(arg2=value2)\", ...]\n\n示例1：\n输入文本:\nresult = search(query=\"test\")\n\n输出函数列表:\n[\"search(query=\"test\")\"]\n\n示例2:\n输入文本:\ndata = get_data(id=123)\nprocessed = process(data)\nprint(\"done\")\n\n输出函数列表:\n[\"get_data(id=123)\", \"process(data)\"]\n\n现在请处理以下文本：\n{response}\n\n输出函数列表:"
                },
                {
                    "type": "code",
                    "exec": "\nimport re\nimport ast\n\ndef check_following(response):\n\n\n    def extract_tool(text):\n        pattern = r'\\[([^]]+)\\]'\n        matches = re.findall(pattern, text)\n        if not matches:\n            return []\n        try:\n            tools = eval(f\"[{matches[-1]}]\")\n            return tools\n        except Exception as e:\n            # print(\"error in extract_tool:\", e)\n            return []\n\n    def validate_function_call(call_str, func_def):\n        \"\"\"\n        验证函数调用是否符合函数定义\n        \n        :param call_str: 函数调用字符串，如 'search(query=\"test\", recency_days=7)'\n        :param func_def: 函数定义字典，包含name和takes_inputs\n        :return: (is_valid, error_msg) 元组，表示是否有效及错误信息\n        \"\"\"\n        # 1. 提取函数名和参数部分\n        try:\n            # 使用AST安全解析函数调用\n            module = ast.parse(call_str.strip())\n            if not isinstance(module, ast.Module) or len(module.body) != 1:\n                return False, \"Invalid statement\"\n            \n            call_node = module.body[0]\n            if not isinstance(call_node, ast.Expr) or not isinstance(call_node.value, ast.Call):\n                return False, \"Not a function call\"\n            \n            # 获取函数名\n            called_func_name = None\n            if isinstance(call_node.value.func, ast.Name):\n                called_func_name = call_node.value.func.id\n            \n            # 2. 检查函数名是否匹配\n            if called_func_name != func_def['name']:\n                return False, f\"Function name mismatch. Expected '{func_def['name']}', got '{called_func_name}'\"\n            \n            # 3. 提取调用参数\n            provided_args = {}\n            for kw in call_node.value.keywords:\n                arg_name = kw.arg\n                # 获取参数值（处理基本类型的字面量）\n                if isinstance(kw.value, ast.Constant):\n                    arg_value = kw.value.value\n                elif isinstance(kw.value, ast.Num):  # Python < 3.8\n                    arg_value = kw.value.n\n                elif isinstance(kw.value, ast.Str):  # Python < 3.8\n                    arg_value = kw.value.s\n                elif isinstance(kw.value, ast.NameConstant):  # Python < 3.8\n                    arg_value = kw.value.value\n                else:\n                    arg_value = None  # 复杂表达式暂不处理\n                \n                provided_args[arg_name] = arg_value\n            \n            # 4. 验证参数\n            expected_args = func_def['takes_inputs']\n            errors = []\n            \n            # 检查必填参数\n            for arg_name, arg_def in expected_args.items():\n                if arg_def['required'] == 'True' and arg_name not in provided_args:\n                    errors.append(f\"Missing required argument: '{arg_name}'\")\n            \n            # 检查未知参数\n            for arg_name in provided_args:\n                if arg_name not in expected_args:\n                    errors.append(f\"Unexpected argument: '{arg_name}'\")\n            \n            # 检查参数类型\n            for arg_name, arg_value in provided_args.items():\n                if arg_name in expected_args:\n                    expected_type = expected_args[arg_name]['type']\n                    actual_type = type(arg_value).__name__\n                    \n                    # 简单类型检查\n                    if expected_type == 'int' and not isinstance(arg_value, int):\n                        errors.append(f\"Argument '{arg_name}' should be {expected_type}, got {actual_type}\")\n                    elif expected_type == 'str' and not isinstance(arg_value, str):\n                        errors.append(f\"Argument '{arg_name}' should be {expected_type}, got {actual_type}\")\n            \n            if errors:\n                return False, \" | \".join(errors)\n            \n            return True, \"Valid call\"\n        \n        except Exception as e:\n            return False, f\"Parsing error: {str(e)}\"\n\n    available_functions = [{'name': 'extract_case_number', 'takes_inputs': {'text': {'description': '包含案件号的文本', 'type': 'str', 'required': False}}}, {'name': 'get_restriction_case', 'takes_inputs': {'identifier': {'description': '案号', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_legal_document_company_list', 'takes_inputs': {'identifier': {'description': '关联公司名称，形如xxx有限公司', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_legal_abstract', 'takes_inputs': {'identifier': {'description': '案号', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_finalized_case', 'takes_inputs': {'identifier': {'description': '案号', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}]\n    test_calls =  extract_tool(response)\n    # print(test_calls)\n    for call in test_calls:\n        Flag = False\n        for function_def in available_functions:\n            is_valid, msg = validate_function_call(call, function_def)\n            # print(function_def[\"name\"], msg)\n            if is_valid == True:\n                Flag = True\n                break\n        if Flag == False:\n            return False\n    return True\n"
                }
            ],
            "score": false
        },
        {
            "id": 8,
            "desc": "您可以在代码中使用import，但只能从以下模块列表中导入",
            "other_info": {
                "available_packages": [
                    "math",
                    "time",
                    "stat",
                    "queue",
                    "itertools",
                    "statistics",
                    "random",
                    "collections",
                    "unicodedata"
                ],
                "from": "system_para_1"
            },
            "dimension": "unconditional",
            "type": "resource",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "提取下面文本中的所有的调用package，直接返回list格式结果，不要任何解释、前缀或赋值语句。\n输出格式要求：[\"package1\", \"package2\", ...]:\n\nHere is model response: {response}"
                },
                {
                    "type": "code",
                    "exec": "\nimport ast\n\ndef check_following(response: str) -> bool:\n    available_packages = ['math', 'time', 'stat', 'queue', 'itertools', 'statistics', 'random', 'collections', 'unicodedata']\n    try:\n        items = ast.literal_eval(response)\n        return isinstance(items, list) and all(item in available_packages for item in items)\n    except:\n        return False\n"
                }
            ],
            "score": false
        },
        {
            "id": 2,
            "desc": "Connect them with appropriate mental health services while acknowledging barriers (cost, access, stigma).",
            "other_info": {
                "from": "system_para_0",
                "type_explanation": "The constraint focuses on content relevance and meaningfulness, specifying the inclusion of barriers (cost, access, stigma) while connecting individuals with mental health services. It emphasizes information completeness and logical clarity rather than formatting or resource limitations.",
                "meta_expalnation": "This constraint directly influences the content or output by specifying that appropriate mental health services should be connected while acknowledging certain barriers (cost, access, stigma). It does not define strategies for managing multiple constraints, like selection, prioritization, ignoring, deduplication, or composition, which are the defining traits of a Meta Constraint.",
                "evaluation_type": {
                    "constraint_type": "llm",
                    "explanation": "The constraint requires semantic understanding to identify and acknowledge barriers (cost, access, stigma), which goes beyond direct logic and involves subjective assessment of context and user needs."
                },
                "evaluation_generation_success": true
            },
            "dimension": "unconditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Does the model response connect the user with appropriate mental health services while explicitly acknowledging barriers? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "state": "modified",
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 3,
            "desc": "Create a concrete daily wellness plan with specific times and activities.",
            "other_info": {
                "from": "system_para_0",
                "type_explanation": "The constraint focuses on ensuring the content is meaningful and complete by requiring a daily wellness plan with specific times and activities. This falls under semantic classification as it pertains to the accuracy, logical consistency, and inclusion of specified elements in the output.",
                "meta_expalnation": "The provided constraint directly governs the task output by specifying the type of content to produce ('a concrete daily wellness plan') and its characteristics (e.g., 'with specific times and activities'). This focuses on the content of the output rather than managing the relationships or strategies for multiple constraints, so it is not a Meta Constraint.",
                "evaluation_type": {
                    "constraint_type": "llm",
                    "explanation": "human_modified"
                },
                "evaluation_generation_success": true
            },
            "dimension": "unconditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Does the response include a concrete daily wellness plan that specifies both the times and the corresponding activities? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "state": "modified",
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 4,
            "desc": "Suggest specific support communities with details on how to join.",
            "other_info": {
                "from": "system_para_0",
                "type_explanation": "The constraint focuses on ensuring the content is meaningful and complete by requiring specific details about communities and how to join them. This aligns with semantic requirements related to content accuracy, relevance, and inclusivity of key information.",
                "meta_expalnation": "The given constraint directly instructs the model to suggest support communities with details on how to join. This impacts the content of the output rather than managing or governing other constraints, which is the defining characteristic of a Meta Constraint.",
                "evaluation_type": {
                    "constraint_type": "llm",
                    "explanation": "Determining and suggesting appropriate support communities requires semantic understanding of the user's situation and preferences, which involves open-ended reasoning and context awareness. Joining details may vary greatly across communities and cannot be consistently verified purely via code logic."
                },
                "evaluation_generation_success": true
            },
            "dimension": "unconditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Does the model's response suggest specific support communities and provide clear details on how to join them? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "state": "success",
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 8,
            "desc": "Please **first** provide  a 2-3 sentence **summary** of your ideas on the action based on the context provided.",
            "other_info": {
                "from": "system_para_2",
                "type_explanation": "The constraint specifies the structure and presentation of the output by requiring it to be a '2-3 sentence summary,' which governs the format and length of the response rather than its content or resource limitations.",
                "meta_expalnation": "The given constraint directly governs the model's output by specifying content requirements (a 2-3 sentence summary of assessment ideas). It does not include any rules about managing or prioritizing multiple constraints, nor does it concern high-level strategies for selecting, ignoring, or combining constraints. Therefore, it is not a Meta Constraint.",
                "evaluation_type": {
                    "constraint_type": "llm",
                    "explanation": "The constraint is subjective and requires semantic interpretation of whether the assessment summary correctly addresses the context provided. This involves open-ended understanding and cannot be validated directly or through extracting structured elements via code."
                },
                "evaluation_generation_success": false
            },
            "dimension": "unconditional",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Does the model response provide a summary firstly? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "state": "modified",
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 16,
            "desc": "Please first provide a **2-3 sentence summary of your ideas on the action based on the context provided**.",
            "other_info": {
                "from": "system_para_2",
                "type_explanation": "The constraint specifies the structure and presentation of the output by requiring it to be a '2-3 sentence summary,' which governs the format and length of the response rather than its content or resource limitations.",
                "meta_expalnation": "The given constraint directly governs the model's output by specifying content requirements (a 2-3 sentence summary of assessment ideas). It does not include any rules about managing or prioritizing multiple constraints, nor does it concern high-level strategies for selecting, ignoring, or combining constraints. Therefore, it is not a Meta Constraint.",
                "evaluation_type": {
                    "constraint_type": "llm_assisted_code",
                    "explanation": "human_modified"
                },
                "evaluation_generation_success": false
            },
            "dimension": "unconditional",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Extract the summary where the response author presents their ideas on the action based on the given context. Return the extracted content verbatim from the response. If multiple segments are found, return them as a Python-style list of strings. If nothing is found, return an empty string (\"\").\n\nHere is the model response: {response}"
                },
                {
                    "type": "code",
                    "exec": "import re\n\ndef check_following(response: str) -> bool:\n    first_part = response.split('\\n\\n')[0] if '\\n\\n' in response else response\n    sentences = re.split('[.!?]', first_part)\n    sentences = [s.strip() for s in sentences if s.strip()]\n    return 2 <= len(sentences) <= 3"
                }
            ],
            "state": "modified",
            "score": false
        },
        {
            "id": 3,
            "desc": "</thought> 结束",
            "other_info": {
                "from": "system_para_1"
            },
            "dimension": "unconditional",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    return response.rstrip().endswith(\"</thought>\")\n"
                }
            ],
            "score": false
        },
        {
            "id": 5,
            "desc": "您可以在thought中使用python packages,但只能从以下packages列表中选择：",
            "other_info": {
                "available_packages": [
                    "math",
                    "time",
                    "stat",
                    "queue",
                    "itertools",
                    "statistics",
                    "random",
                    "collections",
                    "unicodedata"
                ]
            },
            "dimension": "unconditional",
            "type": "resource",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "提取下面文本中的所有的调用package，直接返回list格式结果，不要任何解释、前缀或赋值语句。\n输出格式要求：[\"package1\", \"package2\", ...]:\n\nHere is model response: {response}"
                },
                {
                    "type": "code",
                    "exec": "\nimport ast\n\ndef check_following(response: str) -> bool:\n    available_packages = ['math', 'time', 'stat', 'queue', 'itertools', 'statistics', 'random', 'collections', 'unicodedata']\n    try:\n        items = ast.literal_eval(response)\n        return isinstance(items, list) and all(item in available_packages for item in items)\n    except:\n        return False\n"
                }
            ],
            "score": false
        },
        {
            "id": 3,
            "desc": "</thought> 结束",
            "other_info": {
                "from": "system_para_1"
            },
            "dimension": "unconditional",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    return response.rstrip().endswith(\"</thought>\")\n"
                }
            ],
            "score": false
        },
        {
            "id": 5,
            "desc": "您可以在thought中使用python packages,但只能从以下packages列表中选择：",
            "other_info": {
                "available_packages": [
                    "math",
                    "time",
                    "stat",
                    "queue",
                    "itertools",
                    "statistics",
                    "random",
                    "collections",
                    "unicodedata"
                ]
            },
            "dimension": "unconditional",
            "type": "resource",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "提取下面文本中的所有的调用package，直接返回list格式结果，不要任何解释、前缀或赋值语句。\n输出格式要求：[\"package1\", \"package2\", ...]:\n\nHere is model response: {response}"
                },
                {
                    "type": "code",
                    "exec": "\nimport ast\n\ndef check_following(response: str) -> bool:\n    available_packages = ['math', 'time', 'stat', 'queue', 'itertools', 'statistics', 'random', 'collections', 'unicodedata']\n    try:\n        items = ast.literal_eval(response)\n        return isinstance(items, list) and all(item in available_packages for item in items)\n    except:\n        return False\n"
                }
            ],
            "score": false
        },
        {
            "id": 3,
            "desc": "始终为functions使用正确的参数",
            "other_info": {
                "available_functions": [
                    {
                        "name": "get_legal_document_company_list",
                        "takes_inputs": {
                            "identifier": {
                                "description": "关联公司名称，形如xxx有限公司",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_legal_document",
                        "takes_inputs": {
                            "identifier": {
                                "description": "案号",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_company_register",
                        "takes_inputs": {
                            "identifier": {
                                "description": "公司名称",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_finalized_case",
                        "takes_inputs": {
                            "identifier": {
                                "description": "案号",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_company_register_name",
                        "takes_inputs": {
                            "identifier": {
                                "description": "统一社会信用代码",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    }
                ],
                "from": "system_para_3"
            },
            "dimension": "unconditional",
            "type": "resource",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "提取下面文本中的所有函数调用表达式，直接返回list格式结果，不要任何解释、前缀或赋值语句。\n输出格式要求：[\"function(arg1=value1)\", \"function2(arg2=value2)\", ...]\n\n示例1：\n输入文本:\nresult = search(query=\"test\")\n\n输出函数列表:\n[\"search(query=\"test\")\"]\n\n示例2:\n输入文本:\ndata = get_data(id=123)\nprocessed = process(data)\nprint(\"done\")\n\n输出函数列表:\n[\"get_data(id=123)\", \"process(data)\"]\n\n现在请处理以下文本：\n{response}\n\n输出函数列表:"
                },
                {
                    "type": "code",
                    "exec": "\nimport re\nimport ast\n\ndef check_following(response):\n\n\n    def extract_tool(text):\n        pattern = r'\\[([^]]+)\\]'\n        matches = re.findall(pattern, text)\n        if not matches:\n            return []\n        try:\n            tools = eval(f\"[{matches[-1]}]\")\n            return tools\n        except Exception as e:\n            # print(\"error in extract_tool:\", e)\n            return []\n\n    def validate_function_call(call_str, func_def):\n        \"\"\"\n        验证函数调用是否符合函数定义\n        \n        :param call_str: 函数调用字符串，如 'search(query=\"test\", recency_days=7)'\n        :param func_def: 函数定义字典，包含name和takes_inputs\n        :return: (is_valid, error_msg) 元组，表示是否有效及错误信息\n        \"\"\"\n        # 1. 提取函数名和参数部分\n        try:\n            # 使用AST安全解析函数调用\n            module = ast.parse(call_str.strip())\n            if not isinstance(module, ast.Module) or len(module.body) != 1:\n                return False, \"Invalid statement\"\n            \n            call_node = module.body[0]\n            if not isinstance(call_node, ast.Expr) or not isinstance(call_node.value, ast.Call):\n                return False, \"Not a function call\"\n            \n            # 获取函数名\n            called_func_name = None\n            if isinstance(call_node.value.func, ast.Name):\n                called_func_name = call_node.value.func.id\n            \n            # 2. 检查函数名是否匹配\n            if called_func_name != func_def['name']:\n                return False, f\"Function name mismatch. Expected '{func_def['name']}', got '{called_func_name}'\"\n            \n            # 3. 提取调用参数\n            provided_args = {}\n            for kw in call_node.value.keywords:\n                arg_name = kw.arg\n                # 获取参数值（处理基本类型的字面量）\n                if isinstance(kw.value, ast.Constant):\n                    arg_value = kw.value.value\n                elif isinstance(kw.value, ast.Num):  # Python < 3.8\n                    arg_value = kw.value.n\n                elif isinstance(kw.value, ast.Str):  # Python < 3.8\n                    arg_value = kw.value.s\n                elif isinstance(kw.value, ast.NameConstant):  # Python < 3.8\n                    arg_value = kw.value.value\n                else:\n                    arg_value = None  # 复杂表达式暂不处理\n                \n                provided_args[arg_name] = arg_value\n            \n            # 4. 验证参数\n            expected_args = func_def['takes_inputs']\n            errors = []\n            \n            # 检查必填参数\n            for arg_name, arg_def in expected_args.items():\n                if arg_def['required'] == 'True' and arg_name not in provided_args:\n                    errors.append(f\"Missing required argument: '{arg_name}'\")\n            \n            # 检查未知参数\n            for arg_name in provided_args:\n                if arg_name not in expected_args:\n                    errors.append(f\"Unexpected argument: '{arg_name}'\")\n            \n            # 检查参数类型\n            for arg_name, arg_value in provided_args.items():\n                if arg_name in expected_args:\n                    expected_type = expected_args[arg_name]['type']\n                    actual_type = type(arg_value).__name__\n                    \n                    # 简单类型检查\n                    if expected_type == 'int' and not isinstance(arg_value, int):\n                        errors.append(f\"Argument '{arg_name}' should be {expected_type}, got {actual_type}\")\n                    elif expected_type == 'str' and not isinstance(arg_value, str):\n                        errors.append(f\"Argument '{arg_name}' should be {expected_type}, got {actual_type}\")\n            \n            if errors:\n                return False, \" | \".join(errors)\n            \n            return True, \"Valid call\"\n        \n        except Exception as e:\n            return False, f\"Parsing error: {str(e)}\"\n\n    available_functions = [{'name': 'get_legal_document_company_list', 'takes_inputs': {'identifier': {'description': '关联公司名称，形如xxx有限公司', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_legal_document', 'takes_inputs': {'identifier': {'description': '案号', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_company_register', 'takes_inputs': {'identifier': {'description': '公司名称', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_finalized_case', 'takes_inputs': {'identifier': {'description': '案号', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_company_register_name', 'takes_inputs': {'identifier': {'description': '统一社会信用代码', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}]\n    test_calls =  extract_tool(response)\n    # print(test_calls)\n    for call in test_calls:\n        Flag = False\n        for function_def in available_functions:\n            is_valid, msg = validate_function_call(call, function_def)\n            # print(function_def[\"name\"], msg)\n            if is_valid == True:\n                Flag = True\n                break\n        if Flag == False:\n            return False\n    return True\n"
                }
            ],
            "score": false
        },
        {
            "id": 8,
            "desc": "您可以在代码中使用import，但只能从以下模块列表中导入",
            "other_info": {
                "available_packages": [
                    "math",
                    "time",
                    "stat",
                    "queue",
                    "itertools",
                    "statistics",
                    "random",
                    "collections",
                    "unicodedata"
                ],
                "from": "system_para_1"
            },
            "dimension": "unconditional",
            "type": "resource",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "提取下面文本中的所有的调用package，直接返回list格式结果，不要任何解释、前缀或赋值语句。\n输出格式要求：[\"package1\", \"package2\", ...]:\n\nHere is model response: {response}"
                },
                {
                    "type": "code",
                    "exec": "\nimport ast\n\ndef check_following(response: str) -> bool:\n    available_packages = ['math', 'time', 'stat', 'queue', 'itertools', 'statistics', 'random', 'collections', 'unicodedata']\n    try:\n        items = ast.literal_eval(response)\n        return isinstance(items, list) and all(item in available_packages for item in items)\n    except:\n        return False\n"
                }
            ],
            "score": false
        },
        {
            "id": 3,
            "desc": "始终为functions使用正确的参数",
            "other_info": {
                "available_functions": [
                    {
                        "name": "get_company_info",
                        "takes_inputs": {
                            "identifier": {
                                "description": "公司名称、公司简称或公司代码",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_sub_company_info",
                        "takes_inputs": {
                            "identifier": {
                                "description": "被投资的公司名称",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_company_register",
                        "takes_inputs": {
                            "identifier": {
                                "description": "公司名称",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_sub_company_info_list",
                        "takes_inputs": {
                            "identifier": {
                                "description": "母公司的名称",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_company_register_name",
                        "takes_inputs": {
                            "identifier": {
                                "description": "统一社会信用代码",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    }
                ],
                "from": "system_para_3"
            },
            "dimension": "unconditional",
            "type": "resource",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "提取下面文本中的所有函数调用表达式，直接返回list格式结果，不要任何解释、前缀或赋值语句。\n输出格式要求：[\"function(arg1=value1)\", \"function2(arg2=value2)\", ...]\n\n示例1：\n输入文本:\nresult = search(query=\"test\")\n\n输出函数列表:\n[\"search(query=\"test\")\"]\n\n示例2:\n输入文本:\ndata = get_data(id=123)\nprocessed = process(data)\nprint(\"done\")\n\n输出函数列表:\n[\"get_data(id=123)\", \"process(data)\"]\n\n现在请处理以下文本：\n{response}\n\n输出函数列表:"
                },
                {
                    "type": "code",
                    "exec": "\nimport re\nimport ast\n\ndef check_following(response):\n\n\n    def extract_tool(text):\n        pattern = r'\\[([^]]+)\\]'\n        matches = re.findall(pattern, text)\n        if not matches:\n            return []\n        try:\n            tools = eval(f\"[{matches[-1]}]\")\n            return tools\n        except Exception as e:\n            # print(\"error in extract_tool:\", e)\n            return []\n\n    def validate_function_call(call_str, func_def):\n        \"\"\"\n        验证函数调用是否符合函数定义\n        \n        :param call_str: 函数调用字符串，如 'search(query=\"test\", recency_days=7)'\n        :param func_def: 函数定义字典，包含name和takes_inputs\n        :return: (is_valid, error_msg) 元组，表示是否有效及错误信息\n        \"\"\"\n        # 1. 提取函数名和参数部分\n        try:\n            # 使用AST安全解析函数调用\n            module = ast.parse(call_str.strip())\n            if not isinstance(module, ast.Module) or len(module.body) != 1:\n                return False, \"Invalid statement\"\n            \n            call_node = module.body[0]\n            if not isinstance(call_node, ast.Expr) or not isinstance(call_node.value, ast.Call):\n                return False, \"Not a function call\"\n            \n            # 获取函数名\n            called_func_name = None\n            if isinstance(call_node.value.func, ast.Name):\n                called_func_name = call_node.value.func.id\n            \n            # 2. 检查函数名是否匹配\n            if called_func_name != func_def['name']:\n                return False, f\"Function name mismatch. Expected '{func_def['name']}', got '{called_func_name}'\"\n            \n            # 3. 提取调用参数\n            provided_args = {}\n            for kw in call_node.value.keywords:\n                arg_name = kw.arg\n                # 获取参数值（处理基本类型的字面量）\n                if isinstance(kw.value, ast.Constant):\n                    arg_value = kw.value.value\n                elif isinstance(kw.value, ast.Num):  # Python < 3.8\n                    arg_value = kw.value.n\n                elif isinstance(kw.value, ast.Str):  # Python < 3.8\n                    arg_value = kw.value.s\n                elif isinstance(kw.value, ast.NameConstant):  # Python < 3.8\n                    arg_value = kw.value.value\n                else:\n                    arg_value = None  # 复杂表达式暂不处理\n                \n                provided_args[arg_name] = arg_value\n            \n            # 4. 验证参数\n            expected_args = func_def['takes_inputs']\n            errors = []\n            \n            # 检查必填参数\n            for arg_name, arg_def in expected_args.items():\n                if arg_def['required'] == 'True' and arg_name not in provided_args:\n                    errors.append(f\"Missing required argument: '{arg_name}'\")\n            \n            # 检查未知参数\n            for arg_name in provided_args:\n                if arg_name not in expected_args:\n                    errors.append(f\"Unexpected argument: '{arg_name}'\")\n            \n            # 检查参数类型\n            for arg_name, arg_value in provided_args.items():\n                if arg_name in expected_args:\n                    expected_type = expected_args[arg_name]['type']\n                    actual_type = type(arg_value).__name__\n                    \n                    # 简单类型检查\n                    if expected_type == 'int' and not isinstance(arg_value, int):\n                        errors.append(f\"Argument '{arg_name}' should be {expected_type}, got {actual_type}\")\n                    elif expected_type == 'str' and not isinstance(arg_value, str):\n                        errors.append(f\"Argument '{arg_name}' should be {expected_type}, got {actual_type}\")\n            \n            if errors:\n                return False, \" | \".join(errors)\n            \n            return True, \"Valid call\"\n        \n        except Exception as e:\n            return False, f\"Parsing error: {str(e)}\"\n\n    available_functions = [{'name': 'get_company_info', 'takes_inputs': {'identifier': {'description': '公司名称、公司简称或公司代码', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_sub_company_info', 'takes_inputs': {'identifier': {'description': '被投资的公司名称', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_company_register', 'takes_inputs': {'identifier': {'description': '公司名称', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_sub_company_info_list', 'takes_inputs': {'identifier': {'description': '母公司的名称', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_company_register_name', 'takes_inputs': {'identifier': {'description': '统一社会信用代码', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}]\n    test_calls =  extract_tool(response)\n    # print(test_calls)\n    for call in test_calls:\n        Flag = False\n        for function_def in available_functions:\n            is_valid, msg = validate_function_call(call, function_def)\n            # print(function_def[\"name\"], msg)\n            if is_valid == True:\n                Flag = True\n                break\n        if Flag == False:\n            return False\n    return True\n"
                }
            ],
            "score": false
        },
        {
            "id": 6,
            "desc": "立即使用</code>结束本次编码",
            "other_info": {},
            "dimension": "unconditional",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "required_keys": [
                        "response"
                    ],
                    "exec": "检查文本中每个 print 语句后是否存在 </code>（允许中间有换行或空格）。直接回答 YES 或 NO。\n文本：{response}\n回答："
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 3,
            "desc": "始终为functions使用正确的参数",
            "other_info": {
                "available_functions": [
                    {
                        "name": "get_sub_company_info",
                        "takes_inputs": {
                            "identifier": {
                                "description": "被投资的公司名称",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_sub_company_info_list",
                        "takes_inputs": {
                            "identifier": {
                                "description": "母公司的名称",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_company_register",
                        "takes_inputs": {
                            "identifier": {
                                "description": "公司名称",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_company_register_name",
                        "takes_inputs": {
                            "identifier": {
                                "description": "统一社会信用代码",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_company_info",
                        "takes_inputs": {
                            "identifier": {
                                "description": "公司名称、公司简称或公司代码",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    }
                ],
                "from": "system_para_3"
            },
            "dimension": "unconditional",
            "type": "resource",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "提取下面文本中的所有函数调用表达式，直接返回list格式结果，不要任何解释、前缀或赋值语句。\n输出格式要求：[\"function(arg1=value1)\", \"function2(arg2=value2)\", ...]\n\n示例1：\n输入文本:\nresult = search(query=\"test\")\n\n输出函数列表:\n[\"search(query=\"test\")\"]\n\n示例2:\n输入文本:\ndata = get_data(id=123)\nprocessed = process(data)\nprint(\"done\")\n\n输出函数列表:\n[\"get_data(id=123)\", \"process(data)\"]\n\n现在请处理以下文本：\n{response}\n\n输出函数列表:"
                },
                {
                    "type": "code",
                    "exec": "\nimport re\nimport ast\n\ndef check_following(response):\n\n\n    def extract_tool(text):\n        pattern = r'\\[([^]]+)\\]'\n        matches = re.findall(pattern, text)\n        if not matches:\n            return []\n        try:\n            tools = eval(f\"[{matches[-1]}]\")\n            return tools\n        except Exception as e:\n            # print(\"error in extract_tool:\", e)\n            return []\n\n    def validate_function_call(call_str, func_def):\n        \"\"\"\n        验证函数调用是否符合函数定义\n        \n        :param call_str: 函数调用字符串，如 'search(query=\"test\", recency_days=7)'\n        :param func_def: 函数定义字典，包含name和takes_inputs\n        :return: (is_valid, error_msg) 元组，表示是否有效及错误信息\n        \"\"\"\n        # 1. 提取函数名和参数部分\n        try:\n            # 使用AST安全解析函数调用\n            module = ast.parse(call_str.strip())\n            if not isinstance(module, ast.Module) or len(module.body) != 1:\n                return False, \"Invalid statement\"\n            \n            call_node = module.body[0]\n            if not isinstance(call_node, ast.Expr) or not isinstance(call_node.value, ast.Call):\n                return False, \"Not a function call\"\n            \n            # 获取函数名\n            called_func_name = None\n            if isinstance(call_node.value.func, ast.Name):\n                called_func_name = call_node.value.func.id\n            \n            # 2. 检查函数名是否匹配\n            if called_func_name != func_def['name']:\n                return False, f\"Function name mismatch. Expected '{func_def['name']}', got '{called_func_name}'\"\n            \n            # 3. 提取调用参数\n            provided_args = {}\n            for kw in call_node.value.keywords:\n                arg_name = kw.arg\n                # 获取参数值（处理基本类型的字面量）\n                if isinstance(kw.value, ast.Constant):\n                    arg_value = kw.value.value\n                elif isinstance(kw.value, ast.Num):  # Python < 3.8\n                    arg_value = kw.value.n\n                elif isinstance(kw.value, ast.Str):  # Python < 3.8\n                    arg_value = kw.value.s\n                elif isinstance(kw.value, ast.NameConstant):  # Python < 3.8\n                    arg_value = kw.value.value\n                else:\n                    arg_value = None  # 复杂表达式暂不处理\n                \n                provided_args[arg_name] = arg_value\n            \n            # 4. 验证参数\n            expected_args = func_def['takes_inputs']\n            errors = []\n            \n            # 检查必填参数\n            for arg_name, arg_def in expected_args.items():\n                if arg_def['required'] == 'True' and arg_name not in provided_args:\n                    errors.append(f\"Missing required argument: '{arg_name}'\")\n            \n            # 检查未知参数\n            for arg_name in provided_args:\n                if arg_name not in expected_args:\n                    errors.append(f\"Unexpected argument: '{arg_name}'\")\n            \n            # 检查参数类型\n            for arg_name, arg_value in provided_args.items():\n                if arg_name in expected_args:\n                    expected_type = expected_args[arg_name]['type']\n                    actual_type = type(arg_value).__name__\n                    \n                    # 简单类型检查\n                    if expected_type == 'int' and not isinstance(arg_value, int):\n                        errors.append(f\"Argument '{arg_name}' should be {expected_type}, got {actual_type}\")\n                    elif expected_type == 'str' and not isinstance(arg_value, str):\n                        errors.append(f\"Argument '{arg_name}' should be {expected_type}, got {actual_type}\")\n            \n            if errors:\n                return False, \" | \".join(errors)\n            \n            return True, \"Valid call\"\n        \n        except Exception as e:\n            return False, f\"Parsing error: {str(e)}\"\n\n    available_functions = [{'name': 'get_sub_company_info', 'takes_inputs': {'identifier': {'description': '被投资的公司名称', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_sub_company_info_list', 'takes_inputs': {'identifier': {'description': '母公司的名称', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_company_register', 'takes_inputs': {'identifier': {'description': '公司名称', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_company_register_name', 'takes_inputs': {'identifier': {'description': '统一社会信用代码', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_company_info', 'takes_inputs': {'identifier': {'description': '公司名称、公司简称或公司代码', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}]\n    test_calls =  extract_tool(response)\n    # print(test_calls)\n    for call in test_calls:\n        Flag = False\n        for function_def in available_functions:\n            is_valid, msg = validate_function_call(call, function_def)\n            # print(function_def[\"name\"], msg)\n            if is_valid == True:\n                Flag = True\n                break\n        if Flag == False:\n            return False\n    return True\n"
                }
            ],
            "score": false
        },
        {
            "id": 6,
            "desc": "立即使用</code>结束本次编码",
            "other_info": {},
            "dimension": "unconditional",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "required_keys": [
                        "response"
                    ],
                    "exec": "检查文本中每个 print 语句后是否存在 </code>（允许中间有换行或空格）。直接回答 YES 或 NO。\n文本：{response}\n回答："
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 8,
            "desc": "Please **first** provide  a 2-3 sentence **summary** of your ideas on the action based on the context provided.",
            "other_info": {
                "from": "system_para_2",
                "type_explanation": "The constraint specifies the structure and presentation of the output by requiring it to be a '2-3 sentence summary,' which governs the format and length of the response rather than its content or resource limitations.",
                "meta_expalnation": "The given constraint directly governs the model's output by specifying content requirements (a 2-3 sentence summary of assessment ideas). It does not include any rules about managing or prioritizing multiple constraints, nor does it concern high-level strategies for selecting, ignoring, or combining constraints. Therefore, it is not a Meta Constraint.",
                "evaluation_type": {
                    "constraint_type": "llm",
                    "explanation": "The constraint is subjective and requires semantic interpretation of whether the assessment summary correctly addresses the context provided. This involves open-ended understanding and cannot be validated directly or through extracting structured elements via code."
                },
                "evaluation_generation_success": false
            },
            "dimension": "unconditional",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Does the model response provide a summary firstly? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "state": "modified",
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 16,
            "desc": "Please first provide a **2-3 sentence summary of your ideas on the action based on the context provided**.",
            "other_info": {
                "from": "system_para_2",
                "type_explanation": "The constraint specifies the structure and presentation of the output by requiring it to be a '2-3 sentence summary,' which governs the format and length of the response rather than its content or resource limitations.",
                "meta_expalnation": "The given constraint directly governs the model's output by specifying content requirements (a 2-3 sentence summary of assessment ideas). It does not include any rules about managing or prioritizing multiple constraints, nor does it concern high-level strategies for selecting, ignoring, or combining constraints. Therefore, it is not a Meta Constraint.",
                "evaluation_type": {
                    "constraint_type": "llm_assisted_code",
                    "explanation": "human_modified"
                },
                "evaluation_generation_success": false
            },
            "dimension": "unconditional",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Extract the summary where the response author presents their ideas on the action based on the given context. Return the extracted content verbatim from the response. If multiple segments are found, return them as a Python-style list of strings. If nothing is found, return an empty string (\"\").\n\nHere is the model response: {response}"
                },
                {
                    "type": "code",
                    "exec": "import re\n\ndef check_following(response: str) -> bool:\n    first_part = response.split('\\n\\n')[0] if '\\n\\n' in response else response\n    sentences = re.split('[.!?]', first_part)\n    sentences = [s.strip() for s in sentences if s.strip()]\n    return 2 <= len(sentences) <= 3"
                }
            ],
            "state": "modified",
            "score": false
        },
        {
            "id": 3,
            "desc": "始终为functions使用正确的参数",
            "other_info": {
                "available_functions": [
                    {
                        "name": "get_address_info",
                        "takes_inputs": {
                            "identifier": {
                                "description": "公司地址",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_company_register_name",
                        "takes_inputs": {
                            "identifier": {
                                "description": "统一社会信用代码",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_company_register",
                        "takes_inputs": {
                            "identifier": {
                                "description": "公司名称",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_sub_company_info",
                        "takes_inputs": {
                            "identifier": {
                                "description": "被投资的公司名称",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_sub_company_info_list",
                        "takes_inputs": {
                            "identifier": {
                                "description": "母公司的名称",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    }
                ],
                "from": "system_para_3"
            },
            "dimension": "unconditional",
            "type": "resource",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "提取下面文本中的所有函数调用表达式，直接返回list格式结果，不要任何解释、前缀或赋值语句。\n输出格式要求：[\"function(arg1=value1)\", \"function2(arg2=value2)\", ...]\n\n示例1：\n输入文本:\nresult = search(query=\"test\")\n\n输出函数列表:\n[\"search(query=\"test\")\"]\n\n示例2:\n输入文本:\ndata = get_data(id=123)\nprocessed = process(data)\nprint(\"done\")\n\n输出函数列表:\n[\"get_data(id=123)\", \"process(data)\"]\n\n现在请处理以下文本：\n{response}\n\n输出函数列表:"
                },
                {
                    "type": "code",
                    "exec": "\nimport re\nimport ast\n\ndef check_following(response):\n\n\n    def extract_tool(text):\n        pattern = r'\\[([^]]+)\\]'\n        matches = re.findall(pattern, text)\n        if not matches:\n            return []\n        try:\n            tools = eval(f\"[{matches[-1]}]\")\n            return tools\n        except Exception as e:\n            # print(\"error in extract_tool:\", e)\n            return []\n\n    def validate_function_call(call_str, func_def):\n        \"\"\"\n        验证函数调用是否符合函数定义\n        \n        :param call_str: 函数调用字符串，如 'search(query=\"test\", recency_days=7)'\n        :param func_def: 函数定义字典，包含name和takes_inputs\n        :return: (is_valid, error_msg) 元组，表示是否有效及错误信息\n        \"\"\"\n        # 1. 提取函数名和参数部分\n        try:\n            # 使用AST安全解析函数调用\n            module = ast.parse(call_str.strip())\n            if not isinstance(module, ast.Module) or len(module.body) != 1:\n                return False, \"Invalid statement\"\n            \n            call_node = module.body[0]\n            if not isinstance(call_node, ast.Expr) or not isinstance(call_node.value, ast.Call):\n                return False, \"Not a function call\"\n            \n            # 获取函数名\n            called_func_name = None\n            if isinstance(call_node.value.func, ast.Name):\n                called_func_name = call_node.value.func.id\n            \n            # 2. 检查函数名是否匹配\n            if called_func_name != func_def['name']:\n                return False, f\"Function name mismatch. Expected '{func_def['name']}', got '{called_func_name}'\"\n            \n            # 3. 提取调用参数\n            provided_args = {}\n            for kw in call_node.value.keywords:\n                arg_name = kw.arg\n                # 获取参数值（处理基本类型的字面量）\n                if isinstance(kw.value, ast.Constant):\n                    arg_value = kw.value.value\n                elif isinstance(kw.value, ast.Num):  # Python < 3.8\n                    arg_value = kw.value.n\n                elif isinstance(kw.value, ast.Str):  # Python < 3.8\n                    arg_value = kw.value.s\n                elif isinstance(kw.value, ast.NameConstant):  # Python < 3.8\n                    arg_value = kw.value.value\n                else:\n                    arg_value = None  # 复杂表达式暂不处理\n                \n                provided_args[arg_name] = arg_value\n            \n            # 4. 验证参数\n            expected_args = func_def['takes_inputs']\n            errors = []\n            \n            # 检查必填参数\n            for arg_name, arg_def in expected_args.items():\n                if arg_def['required'] == 'True' and arg_name not in provided_args:\n                    errors.append(f\"Missing required argument: '{arg_name}'\")\n            \n            # 检查未知参数\n            for arg_name in provided_args:\n                if arg_name not in expected_args:\n                    errors.append(f\"Unexpected argument: '{arg_name}'\")\n            \n            # 检查参数类型\n            for arg_name, arg_value in provided_args.items():\n                if arg_name in expected_args:\n                    expected_type = expected_args[arg_name]['type']\n                    actual_type = type(arg_value).__name__\n                    \n                    # 简单类型检查\n                    if expected_type == 'int' and not isinstance(arg_value, int):\n                        errors.append(f\"Argument '{arg_name}' should be {expected_type}, got {actual_type}\")\n                    elif expected_type == 'str' and not isinstance(arg_value, str):\n                        errors.append(f\"Argument '{arg_name}' should be {expected_type}, got {actual_type}\")\n            \n            if errors:\n                return False, \" | \".join(errors)\n            \n            return True, \"Valid call\"\n        \n        except Exception as e:\n            return False, f\"Parsing error: {str(e)}\"\n\n    available_functions = [{'name': 'get_address_info', 'takes_inputs': {'identifier': {'description': '公司地址', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_company_register_name', 'takes_inputs': {'identifier': {'description': '统一社会信用代码', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_company_register', 'takes_inputs': {'identifier': {'description': '公司名称', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_sub_company_info', 'takes_inputs': {'identifier': {'description': '被投资的公司名称', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_sub_company_info_list', 'takes_inputs': {'identifier': {'description': '母公司的名称', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}]\n    test_calls =  extract_tool(response)\n    # print(test_calls)\n    for call in test_calls:\n        Flag = False\n        for function_def in available_functions:\n            is_valid, msg = validate_function_call(call, function_def)\n            # print(function_def[\"name\"], msg)\n            if is_valid == True:\n                Flag = True\n                break\n        if Flag == False:\n            return False\n    return True\n"
                }
            ],
            "score": false
        },
        {
            "id": 6,
            "desc": "立即使用</code>结束本次编码",
            "other_info": {},
            "dimension": "unconditional",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "required_keys": [
                        "response"
                    ],
                    "exec": "检查文本中每个 print 语句后是否存在 </code>（允许中间有换行或空格）。直接回答 YES 或 NO。\n文本：{response}\n回答："
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 3,
            "desc": "</thought> 结束",
            "other_info": {
                "from": "system_para_1"
            },
            "dimension": "unconditional",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    return response.rstrip().endswith(\"</thought>\")\n"
                }
            ],
            "score": false
        },
        {
            "id": 5,
            "desc": "您可以在thought中使用python packages,但只能从以下packages列表中选择：",
            "other_info": {
                "available_packages": [
                    "math",
                    "time",
                    "stat",
                    "queue",
                    "itertools",
                    "statistics",
                    "random",
                    "collections",
                    "unicodedata"
                ]
            },
            "dimension": "unconditional",
            "type": "resource",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "提取下面文本中的所有的调用package，直接返回list格式结果，不要任何解释、前缀或赋值语句。\n输出格式要求：[\"package1\", \"package2\", ...]:\n\nHere is model response: {response}"
                },
                {
                    "type": "code",
                    "exec": "\nimport ast\n\ndef check_following(response: str) -> bool:\n    available_packages = ['math', 'time', 'stat', 'queue', 'itertools', 'statistics', 'random', 'collections', 'unicodedata']\n    try:\n        items = ast.literal_eval(response)\n        return isinstance(items, list) and all(item in available_packages for item in items)\n    except:\n        return False\n"
                }
            ],
            "score": false
        },
        {
            "id": 2,
            "desc": "开始 <thought>",
            "other_info": {
                "from": "system_para_1"
            },
            "dimension": "unconditional",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    return response.lstrip().startswith(\"<thought>\")\n"
                }
            ],
            "score": false
        },
        {
            "id": 3,
            "desc": "</thought> 结束",
            "other_info": {
                "from": "system_para_1"
            },
            "dimension": "unconditional",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    return response.rstrip().endswith(\"</thought>\")\n"
                }
            ],
            "score": false
        },
        {
            "id": 5,
            "desc": "您可以在thought中使用python packages,但只能从以下packages列表中选择：",
            "other_info": {
                "available_packages": [
                    "math",
                    "time",
                    "stat",
                    "queue",
                    "itertools",
                    "statistics",
                    "random",
                    "collections",
                    "unicodedata"
                ]
            },
            "dimension": "unconditional",
            "type": "resource",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "提取下面文本中的所有的调用package，直接返回list格式结果，不要任何解释、前缀或赋值语句。\n输出格式要求：[\"package1\", \"package2\", ...]:\n\nHere is model response: {response}"
                },
                {
                    "type": "code",
                    "exec": "\nimport ast\n\ndef check_following(response: str) -> bool:\n    available_packages = ['math', 'time', 'stat', 'queue', 'itertools', 'statistics', 'random', 'collections', 'unicodedata']\n    try:\n        items = ast.literal_eval(response)\n        return isinstance(items, list) and all(item in available_packages for item in items)\n    except:\n        return False\n"
                }
            ],
            "score": false
        },
        {
            "id": 2,
            "desc": "Connect them with appropriate mental health services while acknowledging barriers (cost, access, stigma).",
            "other_info": {
                "from": "system_para_0",
                "type_explanation": "The constraint focuses on content relevance and meaningfulness, specifying the inclusion of barriers (cost, access, stigma) while connecting individuals with mental health services. It emphasizes information completeness and logical clarity rather than formatting or resource limitations.",
                "meta_expalnation": "This constraint directly influences the content or output by specifying that appropriate mental health services should be connected while acknowledging certain barriers (cost, access, stigma). It does not define strategies for managing multiple constraints, like selection, prioritization, ignoring, deduplication, or composition, which are the defining traits of a Meta Constraint.",
                "evaluation_type": {
                    "constraint_type": "llm",
                    "explanation": "The constraint requires semantic understanding to identify and acknowledge barriers (cost, access, stigma), which goes beyond direct logic and involves subjective assessment of context and user needs."
                },
                "evaluation_generation_success": true
            },
            "dimension": "unconditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Does the model response connect the user with appropriate mental health services while explicitly acknowledging barriers? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "state": "modified",
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 4,
            "desc": "Suggest specific support communities with details on how to join.",
            "other_info": {
                "from": "system_para_0",
                "type_explanation": "The constraint focuses on ensuring the content is meaningful and complete by requiring specific details about communities and how to join them. This aligns with semantic requirements related to content accuracy, relevance, and inclusivity of key information.",
                "meta_expalnation": "The given constraint directly instructs the model to suggest support communities with details on how to join. This impacts the content of the output rather than managing or governing other constraints, which is the defining characteristic of a Meta Constraint.",
                "evaluation_type": {
                    "constraint_type": "llm",
                    "explanation": "Determining and suggesting appropriate support communities requires semantic understanding of the user's situation and preferences, which involves open-ended reasoning and context awareness. Joining details may vary greatly across communities and cannot be consistently verified purely via code logic."
                },
                "evaluation_generation_success": true
            },
            "dimension": "unconditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Does the model's response suggest specific support communities and provide clear details on how to join them? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "state": "success",
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 8,
            "desc": "Please **first** provide  a 2-3 sentence **summary** of your ideas on the action based on the context provided.",
            "other_info": {
                "from": "system_para_2",
                "type_explanation": "The constraint specifies the structure and presentation of the output by requiring it to be a '2-3 sentence summary,' which governs the format and length of the response rather than its content or resource limitations.",
                "meta_expalnation": "The given constraint directly governs the model's output by specifying content requirements (a 2-3 sentence summary of assessment ideas). It does not include any rules about managing or prioritizing multiple constraints, nor does it concern high-level strategies for selecting, ignoring, or combining constraints. Therefore, it is not a Meta Constraint.",
                "evaluation_type": {
                    "constraint_type": "llm",
                    "explanation": "The constraint is subjective and requires semantic interpretation of whether the assessment summary correctly addresses the context provided. This involves open-ended understanding and cannot be validated directly or through extracting structured elements via code."
                },
                "evaluation_generation_success": false
            },
            "dimension": "unconditional",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Does the model response provide a summary firstly? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "state": "modified",
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 16,
            "desc": "Please first provide a **2-3 sentence summary of your ideas on the action based on the context provided**.",
            "other_info": {
                "from": "system_para_2",
                "type_explanation": "The constraint specifies the structure and presentation of the output by requiring it to be a '2-3 sentence summary,' which governs the format and length of the response rather than its content or resource limitations.",
                "meta_expalnation": "The given constraint directly governs the model's output by specifying content requirements (a 2-3 sentence summary of assessment ideas). It does not include any rules about managing or prioritizing multiple constraints, nor does it concern high-level strategies for selecting, ignoring, or combining constraints. Therefore, it is not a Meta Constraint.",
                "evaluation_type": {
                    "constraint_type": "llm_assisted_code",
                    "explanation": "human_modified"
                },
                "evaluation_generation_success": false
            },
            "dimension": "unconditional",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Extract the summary where the response author presents their ideas on the action based on the given context. Return the extracted content verbatim from the response. If multiple segments are found, return them as a Python-style list of strings. If nothing is found, return an empty string (\"\").\n\nHere is the model response: {response}"
                },
                {
                    "type": "code",
                    "exec": "import re\n\ndef check_following(response: str) -> bool:\n    first_part = response.split('\\n\\n')[0] if '\\n\\n' in response else response\n    sentences = re.split('[.!?]', first_part)\n    sentences = [s.strip() for s in sentences if s.strip()]\n    return 2 <= len(sentences) <= 3"
                }
            ],
            "state": "modified",
            "score": false
        },
        {
            "id": 3,
            "desc": "始终为functions使用正确的参数",
            "other_info": {
                "available_functions": [
                    {
                        "name": "get_sub_company_info",
                        "takes_inputs": {
                            "identifier": {
                                "description": "被投资的公司名称",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_company_register_name",
                        "takes_inputs": {
                            "identifier": {
                                "description": "统一社会信用代码",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_company_info",
                        "takes_inputs": {
                            "identifier": {
                                "description": "公司名称、公司简称或公司代码",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_company_register",
                        "takes_inputs": {
                            "identifier": {
                                "description": "公司名称",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_sub_company_info_list",
                        "takes_inputs": {
                            "identifier": {
                                "description": "母公司的名称",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    }
                ],
                "from": "system_para_3"
            },
            "dimension": "unconditional",
            "type": "resource",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "提取下面文本中的所有函数调用表达式，直接返回list格式结果，不要任何解释、前缀或赋值语句。\n输出格式要求：[\"function(arg1=value1)\", \"function2(arg2=value2)\", ...]\n\n示例1：\n输入文本:\nresult = search(query=\"test\")\n\n输出函数列表:\n[\"search(query=\"test\")\"]\n\n示例2:\n输入文本:\ndata = get_data(id=123)\nprocessed = process(data)\nprint(\"done\")\n\n输出函数列表:\n[\"get_data(id=123)\", \"process(data)\"]\n\n现在请处理以下文本：\n{response}\n\n输出函数列表:"
                },
                {
                    "type": "code",
                    "exec": "\nimport re\nimport ast\n\ndef check_following(response):\n\n\n    def extract_tool(text):\n        pattern = r'\\[([^]]+)\\]'\n        matches = re.findall(pattern, text)\n        if not matches:\n            return []\n        try:\n            tools = eval(f\"[{matches[-1]}]\")\n            return tools\n        except Exception as e:\n            # print(\"error in extract_tool:\", e)\n            return []\n\n    def validate_function_call(call_str, func_def):\n        \"\"\"\n        验证函数调用是否符合函数定义\n        \n        :param call_str: 函数调用字符串，如 'search(query=\"test\", recency_days=7)'\n        :param func_def: 函数定义字典，包含name和takes_inputs\n        :return: (is_valid, error_msg) 元组，表示是否有效及错误信息\n        \"\"\"\n        # 1. 提取函数名和参数部分\n        try:\n            # 使用AST安全解析函数调用\n            module = ast.parse(call_str.strip())\n            if not isinstance(module, ast.Module) or len(module.body) != 1:\n                return False, \"Invalid statement\"\n            \n            call_node = module.body[0]\n            if not isinstance(call_node, ast.Expr) or not isinstance(call_node.value, ast.Call):\n                return False, \"Not a function call\"\n            \n            # 获取函数名\n            called_func_name = None\n            if isinstance(call_node.value.func, ast.Name):\n                called_func_name = call_node.value.func.id\n            \n            # 2. 检查函数名是否匹配\n            if called_func_name != func_def['name']:\n                return False, f\"Function name mismatch. Expected '{func_def['name']}', got '{called_func_name}'\"\n            \n            # 3. 提取调用参数\n            provided_args = {}\n            for kw in call_node.value.keywords:\n                arg_name = kw.arg\n                # 获取参数值（处理基本类型的字面量）\n                if isinstance(kw.value, ast.Constant):\n                    arg_value = kw.value.value\n                elif isinstance(kw.value, ast.Num):  # Python < 3.8\n                    arg_value = kw.value.n\n                elif isinstance(kw.value, ast.Str):  # Python < 3.8\n                    arg_value = kw.value.s\n                elif isinstance(kw.value, ast.NameConstant):  # Python < 3.8\n                    arg_value = kw.value.value\n                else:\n                    arg_value = None  # 复杂表达式暂不处理\n                \n                provided_args[arg_name] = arg_value\n            \n            # 4. 验证参数\n            expected_args = func_def['takes_inputs']\n            errors = []\n            \n            # 检查必填参数\n            for arg_name, arg_def in expected_args.items():\n                if arg_def['required'] == 'True' and arg_name not in provided_args:\n                    errors.append(f\"Missing required argument: '{arg_name}'\")\n            \n            # 检查未知参数\n            for arg_name in provided_args:\n                if arg_name not in expected_args:\n                    errors.append(f\"Unexpected argument: '{arg_name}'\")\n            \n            # 检查参数类型\n            for arg_name, arg_value in provided_args.items():\n                if arg_name in expected_args:\n                    expected_type = expected_args[arg_name]['type']\n                    actual_type = type(arg_value).__name__\n                    \n                    # 简单类型检查\n                    if expected_type == 'int' and not isinstance(arg_value, int):\n                        errors.append(f\"Argument '{arg_name}' should be {expected_type}, got {actual_type}\")\n                    elif expected_type == 'str' and not isinstance(arg_value, str):\n                        errors.append(f\"Argument '{arg_name}' should be {expected_type}, got {actual_type}\")\n            \n            if errors:\n                return False, \" | \".join(errors)\n            \n            return True, \"Valid call\"\n        \n        except Exception as e:\n            return False, f\"Parsing error: {str(e)}\"\n\n    available_functions = [{'name': 'get_sub_company_info', 'takes_inputs': {'identifier': {'description': '被投资的公司名称', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_company_register_name', 'takes_inputs': {'identifier': {'description': '统一社会信用代码', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_company_info', 'takes_inputs': {'identifier': {'description': '公司名称、公司简称或公司代码', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_company_register', 'takes_inputs': {'identifier': {'description': '公司名称', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_sub_company_info_list', 'takes_inputs': {'identifier': {'description': '母公司的名称', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}]\n    test_calls =  extract_tool(response)\n    # print(test_calls)\n    for call in test_calls:\n        Flag = False\n        for function_def in available_functions:\n            is_valid, msg = validate_function_call(call, function_def)\n            # print(function_def[\"name\"], msg)\n            if is_valid == True:\n                Flag = True\n                break\n        if Flag == False:\n            return False\n    return True\n"
                }
            ],
            "score": false
        },
        {
            "id": 6,
            "desc": "立即使用</code>结束本次编码",
            "other_info": {},
            "dimension": "unconditional",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "required_keys": [
                        "response"
                    ],
                    "exec": "检查文本中每个 print 语句后是否存在 </code>（允许中间有换行或空格）。直接回答 YES 或 NO。\n文本：{response}\n回答："
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 3,
            "desc": "始终为functions使用正确的参数",
            "other_info": {
                "available_functions": [
                    {
                        "name": "get_sub_company_info_list",
                        "takes_inputs": {
                            "identifier": {
                                "description": "母公司的名称",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_sub_company_info",
                        "takes_inputs": {
                            "identifier": {
                                "description": "被投资的公司名称",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_company_info",
                        "takes_inputs": {
                            "identifier": {
                                "description": "公司名称、公司简称或公司代码",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_finalized_case_company_list",
                        "takes_inputs": {
                            "identifier": {
                                "description": "终本公司名称",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_company_register",
                        "takes_inputs": {
                            "identifier": {
                                "description": "公司名称",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    }
                ],
                "from": "system_para_3"
            },
            "dimension": "unconditional",
            "type": "resource",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "提取下面文本中的所有函数调用表达式，直接返回list格式结果，不要任何解释、前缀或赋值语句。\n输出格式要求：[\"function(arg1=value1)\", \"function2(arg2=value2)\", ...]\n\n示例1：\n输入文本:\nresult = search(query=\"test\")\n\n输出函数列表:\n[\"search(query=\"test\")\"]\n\n示例2:\n输入文本:\ndata = get_data(id=123)\nprocessed = process(data)\nprint(\"done\")\n\n输出函数列表:\n[\"get_data(id=123)\", \"process(data)\"]\n\n现在请处理以下文本：\n{response}\n\n输出函数列表:"
                },
                {
                    "type": "code",
                    "exec": "\nimport re\nimport ast\n\ndef check_following(response):\n\n\n    def extract_tool(text):\n        pattern = r'\\[([^]]+)\\]'\n        matches = re.findall(pattern, text)\n        if not matches:\n            return []\n        try:\n            tools = eval(f\"[{matches[-1]}]\")\n            return tools\n        except Exception as e:\n            # print(\"error in extract_tool:\", e)\n            return []\n\n    def validate_function_call(call_str, func_def):\n        \"\"\"\n        验证函数调用是否符合函数定义\n        \n        :param call_str: 函数调用字符串，如 'search(query=\"test\", recency_days=7)'\n        :param func_def: 函数定义字典，包含name和takes_inputs\n        :return: (is_valid, error_msg) 元组，表示是否有效及错误信息\n        \"\"\"\n        # 1. 提取函数名和参数部分\n        try:\n            # 使用AST安全解析函数调用\n            module = ast.parse(call_str.strip())\n            if not isinstance(module, ast.Module) or len(module.body) != 1:\n                return False, \"Invalid statement\"\n            \n            call_node = module.body[0]\n            if not isinstance(call_node, ast.Expr) or not isinstance(call_node.value, ast.Call):\n                return False, \"Not a function call\"\n            \n            # 获取函数名\n            called_func_name = None\n            if isinstance(call_node.value.func, ast.Name):\n                called_func_name = call_node.value.func.id\n            \n            # 2. 检查函数名是否匹配\n            if called_func_name != func_def['name']:\n                return False, f\"Function name mismatch. Expected '{func_def['name']}', got '{called_func_name}'\"\n            \n            # 3. 提取调用参数\n            provided_args = {}\n            for kw in call_node.value.keywords:\n                arg_name = kw.arg\n                # 获取参数值（处理基本类型的字面量）\n                if isinstance(kw.value, ast.Constant):\n                    arg_value = kw.value.value\n                elif isinstance(kw.value, ast.Num):  # Python < 3.8\n                    arg_value = kw.value.n\n                elif isinstance(kw.value, ast.Str):  # Python < 3.8\n                    arg_value = kw.value.s\n                elif isinstance(kw.value, ast.NameConstant):  # Python < 3.8\n                    arg_value = kw.value.value\n                else:\n                    arg_value = None  # 复杂表达式暂不处理\n                \n                provided_args[arg_name] = arg_value\n            \n            # 4. 验证参数\n            expected_args = func_def['takes_inputs']\n            errors = []\n            \n            # 检查必填参数\n            for arg_name, arg_def in expected_args.items():\n                if arg_def['required'] == 'True' and arg_name not in provided_args:\n                    errors.append(f\"Missing required argument: '{arg_name}'\")\n            \n            # 检查未知参数\n            for arg_name in provided_args:\n                if arg_name not in expected_args:\n                    errors.append(f\"Unexpected argument: '{arg_name}'\")\n            \n            # 检查参数类型\n            for arg_name, arg_value in provided_args.items():\n                if arg_name in expected_args:\n                    expected_type = expected_args[arg_name]['type']\n                    actual_type = type(arg_value).__name__\n                    \n                    # 简单类型检查\n                    if expected_type == 'int' and not isinstance(arg_value, int):\n                        errors.append(f\"Argument '{arg_name}' should be {expected_type}, got {actual_type}\")\n                    elif expected_type == 'str' and not isinstance(arg_value, str):\n                        errors.append(f\"Argument '{arg_name}' should be {expected_type}, got {actual_type}\")\n            \n            if errors:\n                return False, \" | \".join(errors)\n            \n            return True, \"Valid call\"\n        \n        except Exception as e:\n            return False, f\"Parsing error: {str(e)}\"\n\n    available_functions = [{'name': 'get_sub_company_info_list', 'takes_inputs': {'identifier': {'description': '母公司的名称', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_sub_company_info', 'takes_inputs': {'identifier': {'description': '被投资的公司名称', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_company_info', 'takes_inputs': {'identifier': {'description': '公司名称、公司简称或公司代码', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_finalized_case_company_list', 'takes_inputs': {'identifier': {'description': '终本公司名称', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_company_register', 'takes_inputs': {'identifier': {'description': '公司名称', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}]\n    test_calls =  extract_tool(response)\n    # print(test_calls)\n    for call in test_calls:\n        Flag = False\n        for function_def in available_functions:\n            is_valid, msg = validate_function_call(call, function_def)\n            # print(function_def[\"name\"], msg)\n            if is_valid == True:\n                Flag = True\n                break\n        if Flag == False:\n            return False\n    return True\n"
                }
            ],
            "score": false
        },
        {
            "id": 6,
            "desc": "立即使用</code>结束本次编码",
            "other_info": {},
            "dimension": "unconditional",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "required_keys": [
                        "response"
                    ],
                    "exec": "检查文本中每个 print 语句后是否存在 </code>（允许中间有换行或空格）。直接回答 YES 或 NO。\n文本：{response}\n回答："
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 8,
            "desc": "您可以在代码中使用import，但只能从以下模块列表中导入",
            "other_info": {
                "available_packages": [
                    "math",
                    "time",
                    "stat",
                    "queue",
                    "itertools",
                    "statistics",
                    "random",
                    "collections",
                    "unicodedata"
                ],
                "from": "system_para_1"
            },
            "dimension": "unconditional",
            "type": "resource",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "提取下面文本中的所有的调用package，直接返回list格式结果，不要任何解释、前缀或赋值语句。\n输出格式要求：[\"package1\", \"package2\", ...]:\n\nHere is model response: {response}"
                },
                {
                    "type": "code",
                    "exec": "\nimport ast\n\ndef check_following(response: str) -> bool:\n    available_packages = ['math', 'time', 'stat', 'queue', 'itertools', 'statistics', 'random', 'collections', 'unicodedata']\n    try:\n        items = ast.literal_eval(response)\n        return isinstance(items, list) and all(item in available_packages for item in items)\n    except:\n        return False\n"
                }
            ],
            "score": false
        },
        {
            "id": 3,
            "desc": "始终为functions使用正确的参数",
            "other_info": {
                "available_functions": [
                    {
                        "name": "get_sub_company_info_list",
                        "takes_inputs": {
                            "identifier": {
                                "description": "母公司的名称",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_sub_company_info",
                        "takes_inputs": {
                            "identifier": {
                                "description": "被投资的公司名称",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_company_register",
                        "takes_inputs": {
                            "identifier": {
                                "description": "公司名称",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_company_info",
                        "takes_inputs": {
                            "identifier": {
                                "description": "公司名称、公司简称或公司代码",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_finalized_case_company_list",
                        "takes_inputs": {
                            "identifier": {
                                "description": "终本公司名称",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    }
                ],
                "from": "system_para_3"
            },
            "dimension": "unconditional",
            "type": "resource",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "提取下面文本中的所有函数调用表达式，直接返回list格式结果，不要任何解释、前缀或赋值语句。\n输出格式要求：[\"function(arg1=value1)\", \"function2(arg2=value2)\", ...]\n\n示例1：\n输入文本:\nresult = search(query=\"test\")\n\n输出函数列表:\n[\"search(query=\"test\")\"]\n\n示例2:\n输入文本:\ndata = get_data(id=123)\nprocessed = process(data)\nprint(\"done\")\n\n输出函数列表:\n[\"get_data(id=123)\", \"process(data)\"]\n\n现在请处理以下文本：\n{response}\n\n输出函数列表:"
                },
                {
                    "type": "code",
                    "exec": "\nimport re\nimport ast\n\ndef check_following(response):\n\n\n    def extract_tool(text):\n        pattern = r'\\[([^]]+)\\]'\n        matches = re.findall(pattern, text)\n        if not matches:\n            return []\n        try:\n            tools = eval(f\"[{matches[-1]}]\")\n            return tools\n        except Exception as e:\n            # print(\"error in extract_tool:\", e)\n            return []\n\n    def validate_function_call(call_str, func_def):\n        \"\"\"\n        验证函数调用是否符合函数定义\n        \n        :param call_str: 函数调用字符串，如 'search(query=\"test\", recency_days=7)'\n        :param func_def: 函数定义字典，包含name和takes_inputs\n        :return: (is_valid, error_msg) 元组，表示是否有效及错误信息\n        \"\"\"\n        # 1. 提取函数名和参数部分\n        try:\n            # 使用AST安全解析函数调用\n            module = ast.parse(call_str.strip())\n            if not isinstance(module, ast.Module) or len(module.body) != 1:\n                return False, \"Invalid statement\"\n            \n            call_node = module.body[0]\n            if not isinstance(call_node, ast.Expr) or not isinstance(call_node.value, ast.Call):\n                return False, \"Not a function call\"\n            \n            # 获取函数名\n            called_func_name = None\n            if isinstance(call_node.value.func, ast.Name):\n                called_func_name = call_node.value.func.id\n            \n            # 2. 检查函数名是否匹配\n            if called_func_name != func_def['name']:\n                return False, f\"Function name mismatch. Expected '{func_def['name']}', got '{called_func_name}'\"\n            \n            # 3. 提取调用参数\n            provided_args = {}\n            for kw in call_node.value.keywords:\n                arg_name = kw.arg\n                # 获取参数值（处理基本类型的字面量）\n                if isinstance(kw.value, ast.Constant):\n                    arg_value = kw.value.value\n                elif isinstance(kw.value, ast.Num):  # Python < 3.8\n                    arg_value = kw.value.n\n                elif isinstance(kw.value, ast.Str):  # Python < 3.8\n                    arg_value = kw.value.s\n                elif isinstance(kw.value, ast.NameConstant):  # Python < 3.8\n                    arg_value = kw.value.value\n                else:\n                    arg_value = None  # 复杂表达式暂不处理\n                \n                provided_args[arg_name] = arg_value\n            \n            # 4. 验证参数\n            expected_args = func_def['takes_inputs']\n            errors = []\n            \n            # 检查必填参数\n            for arg_name, arg_def in expected_args.items():\n                if arg_def['required'] == 'True' and arg_name not in provided_args:\n                    errors.append(f\"Missing required argument: '{arg_name}'\")\n            \n            # 检查未知参数\n            for arg_name in provided_args:\n                if arg_name not in expected_args:\n                    errors.append(f\"Unexpected argument: '{arg_name}'\")\n            \n            # 检查参数类型\n            for arg_name, arg_value in provided_args.items():\n                if arg_name in expected_args:\n                    expected_type = expected_args[arg_name]['type']\n                    actual_type = type(arg_value).__name__\n                    \n                    # 简单类型检查\n                    if expected_type == 'int' and not isinstance(arg_value, int):\n                        errors.append(f\"Argument '{arg_name}' should be {expected_type}, got {actual_type}\")\n                    elif expected_type == 'str' and not isinstance(arg_value, str):\n                        errors.append(f\"Argument '{arg_name}' should be {expected_type}, got {actual_type}\")\n            \n            if errors:\n                return False, \" | \".join(errors)\n            \n            return True, \"Valid call\"\n        \n        except Exception as e:\n            return False, f\"Parsing error: {str(e)}\"\n\n    available_functions = [{'name': 'get_sub_company_info_list', 'takes_inputs': {'identifier': {'description': '母公司的名称', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_sub_company_info', 'takes_inputs': {'identifier': {'description': '被投资的公司名称', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_company_register', 'takes_inputs': {'identifier': {'description': '公司名称', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_company_info', 'takes_inputs': {'identifier': {'description': '公司名称、公司简称或公司代码', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_finalized_case_company_list', 'takes_inputs': {'identifier': {'description': '终本公司名称', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}]\n    test_calls =  extract_tool(response)\n    # print(test_calls)\n    for call in test_calls:\n        Flag = False\n        for function_def in available_functions:\n            is_valid, msg = validate_function_call(call, function_def)\n            # print(function_def[\"name\"], msg)\n            if is_valid == True:\n                Flag = True\n                break\n        if Flag == False:\n            return False\n    return True\n"
                }
            ],
            "score": false
        },
        {
            "id": 6,
            "desc": "立即使用</code>结束本次编码",
            "other_info": {},
            "dimension": "unconditional",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "required_keys": [
                        "response"
                    ],
                    "exec": "检查文本中每个 print 语句后是否存在 </code>（允许中间有换行或空格）。直接回答 YES 或 NO。\n文本：{response}\n回答："
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 3,
            "desc": "始终为functions使用正确的参数",
            "other_info": {
                "available_functions": [
                    {
                        "name": "get_sub_company_info_list",
                        "takes_inputs": {
                            "identifier": {
                                "description": "母公司的名称",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_sub_company_info",
                        "takes_inputs": {
                            "identifier": {
                                "description": "被投资的公司名称",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_company_register",
                        "takes_inputs": {
                            "identifier": {
                                "description": "公司名称",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_company_info",
                        "takes_inputs": {
                            "identifier": {
                                "description": "公司名称、公司简称或公司代码",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    },
                    {
                        "name": "get_finalized_case_company_list",
                        "takes_inputs": {
                            "identifier": {
                                "description": "终本公司名称",
                                "type": "str",
                                "required": "True"
                            },
                            "columns": {
                                "description": "需要返回的列名列表",
                                "type": "list",
                                "required": "False"
                            }
                        }
                    }
                ],
                "from": "system_para_3"
            },
            "dimension": "unconditional",
            "type": "resource",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "提取下面文本中的所有函数调用表达式，直接返回list格式结果，不要任何解释、前缀或赋值语句。\n输出格式要求：[\"function(arg1=value1)\", \"function2(arg2=value2)\", ...]\n\n示例1：\n输入文本:\nresult = search(query=\"test\")\n\n输出函数列表:\n[\"search(query=\"test\")\"]\n\n示例2:\n输入文本:\ndata = get_data(id=123)\nprocessed = process(data)\nprint(\"done\")\n\n输出函数列表:\n[\"get_data(id=123)\", \"process(data)\"]\n\n现在请处理以下文本：\n{response}\n\n输出函数列表:"
                },
                {
                    "type": "code",
                    "exec": "\nimport re\nimport ast\n\ndef check_following(response):\n\n\n    def extract_tool(text):\n        pattern = r'\\[([^]]+)\\]'\n        matches = re.findall(pattern, text)\n        if not matches:\n            return []\n        try:\n            tools = eval(f\"[{matches[-1]}]\")\n            return tools\n        except Exception as e:\n            # print(\"error in extract_tool:\", e)\n            return []\n\n    def validate_function_call(call_str, func_def):\n        \"\"\"\n        验证函数调用是否符合函数定义\n        \n        :param call_str: 函数调用字符串，如 'search(query=\"test\", recency_days=7)'\n        :param func_def: 函数定义字典，包含name和takes_inputs\n        :return: (is_valid, error_msg) 元组，表示是否有效及错误信息\n        \"\"\"\n        # 1. 提取函数名和参数部分\n        try:\n            # 使用AST安全解析函数调用\n            module = ast.parse(call_str.strip())\n            if not isinstance(module, ast.Module) or len(module.body) != 1:\n                return False, \"Invalid statement\"\n            \n            call_node = module.body[0]\n            if not isinstance(call_node, ast.Expr) or not isinstance(call_node.value, ast.Call):\n                return False, \"Not a function call\"\n            \n            # 获取函数名\n            called_func_name = None\n            if isinstance(call_node.value.func, ast.Name):\n                called_func_name = call_node.value.func.id\n            \n            # 2. 检查函数名是否匹配\n            if called_func_name != func_def['name']:\n                return False, f\"Function name mismatch. Expected '{func_def['name']}', got '{called_func_name}'\"\n            \n            # 3. 提取调用参数\n            provided_args = {}\n            for kw in call_node.value.keywords:\n                arg_name = kw.arg\n                # 获取参数值（处理基本类型的字面量）\n                if isinstance(kw.value, ast.Constant):\n                    arg_value = kw.value.value\n                elif isinstance(kw.value, ast.Num):  # Python < 3.8\n                    arg_value = kw.value.n\n                elif isinstance(kw.value, ast.Str):  # Python < 3.8\n                    arg_value = kw.value.s\n                elif isinstance(kw.value, ast.NameConstant):  # Python < 3.8\n                    arg_value = kw.value.value\n                else:\n                    arg_value = None  # 复杂表达式暂不处理\n                \n                provided_args[arg_name] = arg_value\n            \n            # 4. 验证参数\n            expected_args = func_def['takes_inputs']\n            errors = []\n            \n            # 检查必填参数\n            for arg_name, arg_def in expected_args.items():\n                if arg_def['required'] == 'True' and arg_name not in provided_args:\n                    errors.append(f\"Missing required argument: '{arg_name}'\")\n            \n            # 检查未知参数\n            for arg_name in provided_args:\n                if arg_name not in expected_args:\n                    errors.append(f\"Unexpected argument: '{arg_name}'\")\n            \n            # 检查参数类型\n            for arg_name, arg_value in provided_args.items():\n                if arg_name in expected_args:\n                    expected_type = expected_args[arg_name]['type']\n                    actual_type = type(arg_value).__name__\n                    \n                    # 简单类型检查\n                    if expected_type == 'int' and not isinstance(arg_value, int):\n                        errors.append(f\"Argument '{arg_name}' should be {expected_type}, got {actual_type}\")\n                    elif expected_type == 'str' and not isinstance(arg_value, str):\n                        errors.append(f\"Argument '{arg_name}' should be {expected_type}, got {actual_type}\")\n            \n            if errors:\n                return False, \" | \".join(errors)\n            \n            return True, \"Valid call\"\n        \n        except Exception as e:\n            return False, f\"Parsing error: {str(e)}\"\n\n    available_functions = [{'name': 'get_sub_company_info_list', 'takes_inputs': {'identifier': {'description': '母公司的名称', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_sub_company_info', 'takes_inputs': {'identifier': {'description': '被投资的公司名称', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_company_register', 'takes_inputs': {'identifier': {'description': '公司名称', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_company_info', 'takes_inputs': {'identifier': {'description': '公司名称、公司简称或公司代码', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}, {'name': 'get_finalized_case_company_list', 'takes_inputs': {'identifier': {'description': '终本公司名称', 'type': 'str', 'required': 'True'}, 'columns': {'description': '需要返回的列名列表', 'type': 'list', 'required': 'False'}}}]\n    test_calls =  extract_tool(response)\n    # print(test_calls)\n    for call in test_calls:\n        Flag = False\n        for function_def in available_functions:\n            is_valid, msg = validate_function_call(call, function_def)\n            # print(function_def[\"name\"], msg)\n            if is_valid == True:\n                Flag = True\n                break\n        if Flag == False:\n            return False\n    return True\n"
                }
            ],
            "score": false
        },
        {
            "id": 6,
            "desc": "立即使用</code>结束本次编码",
            "other_info": {},
            "dimension": "unconditional",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "required_keys": [
                        "response"
                    ],
                    "exec": "检查文本中每个 print 语句后是否存在 </code>（允许中间有换行或空格）。直接回答 YES 或 NO。\n文本：{response}\n回答："
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 3,
            "desc": "</thought> 结束",
            "other_info": {
                "from": "system_para_1"
            },
            "dimension": "unconditional",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    return response.rstrip().endswith(\"</thought>\")\n"
                }
            ],
            "score": false
        },
        {
            "id": 5,
            "desc": "您可以在thought中使用python packages,但只能从以下packages列表中选择：",
            "other_info": {
                "available_packages": [
                    "math",
                    "time",
                    "stat",
                    "queue",
                    "itertools",
                    "statistics",
                    "random",
                    "collections",
                    "unicodedata"
                ]
            },
            "dimension": "unconditional",
            "type": "resource",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "提取下面文本中的所有的调用package，直接返回list格式结果，不要任何解释、前缀或赋值语句。\n输出格式要求：[\"package1\", \"package2\", ...]:\n\nHere is model response: {response}"
                },
                {
                    "type": "code",
                    "exec": "\nimport ast\n\ndef check_following(response: str) -> bool:\n    available_packages = ['math', 'time', 'stat', 'queue', 'itertools', 'statistics', 'random', 'collections', 'unicodedata']\n    try:\n        items = ast.literal_eval(response)\n        return isinstance(items, list) and all(item in available_packages for item in items)\n    except:\n        return False\n"
                }
            ],
            "score": false
        },
        {
            "id": 3,
            "desc": "</thought> 结束",
            "other_info": {
                "from": "system_para_1"
            },
            "dimension": "unconditional",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    return response.rstrip().endswith(\"</thought>\")\n"
                }
            ],
            "score": false
        },
        {
            "id": 3,
            "desc": "</thought> 结束",
            "other_info": {
                "from": "system_para_1"
            },
            "dimension": "unconditional",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    return response.rstrip().endswith(\"</thought>\")\n"
                }
            ],
            "score": false
        }
    ],
    "example_driven": [
        {
            "id": 0,
            "desc": "开始 <code>",
            "other_info": {
                "from": "system_para_6"
            },
            "dimension": "example_driven",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    return response.lstrip().startswith(\"<code>\")\n"
                }
            ],
            "score": false
        },
        {
            "id": 1,
            "desc": "</code> 结束",
            "other_info": {
                "from": "system_para_6"
            },
            "dimension": "example_driven",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    return response.rstrip().endswith(\"</code>\")\n"
                }
            ],
            "score": false
        },
        {
            "id": 0,
            "desc": "开始 <code>",
            "other_info": {
                "from": "system_para_6"
            },
            "dimension": "example_driven",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    return response.lstrip().startswith(\"<code>\")\n"
                }
            ],
            "score": false
        },
        {
            "id": 1,
            "desc": "</code> 结束",
            "other_info": {
                "from": "system_para_6"
            },
            "dimension": "example_driven",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    return response.rstrip().endswith(\"</code>\")\n"
                }
            ],
            "score": false
        },
        {
            "id": 0,
            "desc": "开始 <code>",
            "other_info": {
                "from": "system_para_6"
            },
            "dimension": "example_driven",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    return response.lstrip().startswith(\"<code>\")\n"
                }
            ],
            "score": false
        },
        {
            "id": 1,
            "desc": "</code> 结束",
            "other_info": {
                "from": "system_para_6"
            },
            "dimension": "example_driven",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    return response.rstrip().endswith(\"</code>\")\n"
                }
            ],
            "score": false
        },
        {
            "id": 0,
            "desc": "开始 <code>",
            "other_info": {
                "from": "system_para_6"
            },
            "dimension": "example_driven",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    return response.lstrip().startswith(\"<code>\")\n"
                }
            ],
            "score": false
        },
        {
            "id": 1,
            "desc": "</code> 结束",
            "other_info": {
                "from": "system_para_6"
            },
            "dimension": "example_driven",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    return response.rstrip().endswith(\"</code>\")\n"
                }
            ],
            "score": false
        },
        {
            "id": 0,
            "desc": "开始 <code>",
            "other_info": {
                "from": "system_para_6"
            },
            "dimension": "example_driven",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    return response.lstrip().startswith(\"<code>\")\n"
                }
            ],
            "score": false
        },
        {
            "id": 1,
            "desc": "</code> 结束",
            "other_info": {
                "from": "system_para_6"
            },
            "dimension": "example_driven",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    return response.rstrip().endswith(\"</code>\")\n"
                }
            ],
            "score": false
        },
        {
            "id": 0,
            "desc": "开始 <code>",
            "other_info": {
                "from": "system_para_6"
            },
            "dimension": "example_driven",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    return response.lstrip().startswith(\"<code>\")\n"
                }
            ],
            "score": false
        },
        {
            "id": 1,
            "desc": "</code> 结束",
            "other_info": {
                "from": "system_para_6"
            },
            "dimension": "example_driven",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    return response.rstrip().endswith(\"</code>\")\n"
                }
            ],
            "score": false
        },
        {
            "id": 0,
            "desc": "开始 <code>",
            "other_info": {
                "from": "system_para_6"
            },
            "dimension": "example_driven",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    return response.lstrip().startswith(\"<code>\")\n"
                }
            ],
            "score": false
        },
        {
            "id": 1,
            "desc": "</code> 结束",
            "other_info": {
                "from": "system_para_6"
            },
            "dimension": "example_driven",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    return response.rstrip().endswith(\"</code>\")\n"
                }
            ],
            "score": false
        },
        {
            "id": 0,
            "desc": "开始 <code>",
            "other_info": {
                "from": "system_para_6"
            },
            "dimension": "example_driven",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    return response.lstrip().startswith(\"<code>\")\n"
                }
            ],
            "score": false
        },
        {
            "id": 1,
            "desc": "</code> 结束",
            "other_info": {
                "from": "system_para_6"
            },
            "dimension": "example_driven",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    return response.rstrip().endswith(\"</code>\")\n"
                }
            ],
            "score": false
        },
        {
            "id": 0,
            "desc": "开始 <code>",
            "other_info": {
                "from": "system_para_6"
            },
            "dimension": "example_driven",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    return response.lstrip().startswith(\"<code>\")\n"
                }
            ],
            "score": false
        },
        {
            "id": 1,
            "desc": "</code> 结束",
            "other_info": {
                "from": "system_para_6"
            },
            "dimension": "example_driven",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    return response.rstrip().endswith(\"</code>\")\n"
                }
            ],
            "score": false
        },
        {
            "id": 0,
            "desc": "开始 <code>",
            "other_info": {
                "from": "system_para_6"
            },
            "dimension": "example_driven",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    return response.lstrip().startswith(\"<code>\")\n"
                }
            ],
            "score": false
        },
        {
            "id": 1,
            "desc": "</code> 结束",
            "other_info": {
                "from": "system_para_6"
            },
            "dimension": "example_driven",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    return response.rstrip().endswith(\"</code>\")\n"
                }
            ],
            "score": false
        },
        {
            "id": 0,
            "desc": "开始 <code>",
            "other_info": {
                "from": "system_para_6"
            },
            "dimension": "example_driven",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    return response.lstrip().startswith(\"<code>\")\n"
                }
            ],
            "score": false
        },
        {
            "id": 1,
            "desc": "</code> 结束",
            "other_info": {
                "from": "system_para_6"
            },
            "dimension": "example_driven",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    return response.rstrip().endswith(\"</code>\")\n"
                }
            ],
            "score": false
        },
        {
            "id": 0,
            "desc": "开始 <code>",
            "other_info": {
                "from": "system_para_6"
            },
            "dimension": "example_driven",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    return response.lstrip().startswith(\"<code>\")\n"
                }
            ],
            "score": false
        },
        {
            "id": 1,
            "desc": "</code> 结束",
            "other_info": {
                "from": "system_para_6"
            },
            "dimension": "example_driven",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    return response.rstrip().endswith(\"</code>\")\n"
                }
            ],
            "score": false
        },
        {
            "id": 0,
            "desc": "开始 <code>",
            "other_info": {
                "from": "system_para_6"
            },
            "dimension": "example_driven",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    return response.lstrip().startswith(\"<code>\")\n"
                }
            ],
            "score": false
        },
        {
            "id": 1,
            "desc": "</code> 结束",
            "other_info": {
                "from": "system_para_6"
            },
            "dimension": "example_driven",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    return response.rstrip().endswith(\"</code>\")\n"
                }
            ],
            "score": false
        },
        {
            "id": 0,
            "desc": "开始 <code>",
            "other_info": {
                "from": "system_para_6"
            },
            "dimension": "example_driven",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    return response.lstrip().startswith(\"<code>\")\n"
                }
            ],
            "score": false
        },
        {
            "id": 1,
            "desc": "</code> 结束",
            "other_info": {
                "from": "system_para_6"
            },
            "dimension": "example_driven",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    return response.rstrip().endswith(\"</code>\")\n"
                }
            ],
            "score": false
        },
        {
            "id": 0,
            "desc": "开始 <code>",
            "other_info": {
                "from": "system_para_6"
            },
            "dimension": "example_driven",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    return response.lstrip().startswith(\"<code>\")\n"
                }
            ],
            "score": false
        },
        {
            "id": 1,
            "desc": "</code> 结束",
            "other_info": {
                "from": "system_para_6"
            },
            "dimension": "example_driven",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    return response.rstrip().endswith(\"</code>\")\n"
                }
            ],
            "score": false
        },
        {
            "id": 0,
            "desc": "开始 <code>",
            "other_info": {
                "from": "system_para_6"
            },
            "dimension": "example_driven",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    return response.lstrip().startswith(\"<code>\")\n"
                }
            ],
            "score": false
        },
        {
            "id": 1,
            "desc": "</code> 结束",
            "other_info": {
                "from": "system_para_6"
            },
            "dimension": "example_driven",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    return response.rstrip().endswith(\"</code>\")\n"
                }
            ],
            "score": false
        },
        {
            "id": 0,
            "desc": "开始 <code>",
            "other_info": {
                "from": "system_para_6"
            },
            "dimension": "example_driven",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    return response.lstrip().startswith(\"<code>\")\n"
                }
            ],
            "score": false
        },
        {
            "id": 1,
            "desc": "</code> 结束",
            "other_info": {
                "from": "system_para_6"
            },
            "dimension": "example_driven",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    return response.rstrip().endswith(\"</code>\")\n"
                }
            ],
            "score": false
        },
        {
            "id": 0,
            "desc": "开始 <code>",
            "other_info": {
                "from": "system_para_6"
            },
            "dimension": "example_driven",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    return response.lstrip().startswith(\"<code>\")\n"
                }
            ],
            "score": false
        },
        {
            "id": 1,
            "desc": "</code> 结束",
            "other_info": {
                "from": "system_para_6"
            },
            "dimension": "example_driven",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    return response.rstrip().endswith(\"</code>\")\n"
                }
            ],
            "score": false
        },
        {
            "id": 0,
            "desc": "开始 <code>",
            "other_info": {
                "from": "system_para_6"
            },
            "dimension": "example_driven",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    return response.lstrip().startswith(\"<code>\")\n"
                }
            ],
            "score": false
        },
        {
            "id": 1,
            "desc": "</code> 结束",
            "other_info": {
                "from": "system_para_6"
            },
            "dimension": "example_driven",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    return response.rstrip().endswith(\"</code>\")\n"
                }
            ],
            "score": false
        },
        {
            "id": 0,
            "desc": "开始 <code>",
            "other_info": {
                "from": "system_para_6"
            },
            "dimension": "example_driven",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    return response.lstrip().startswith(\"<code>\")\n"
                }
            ],
            "score": false
        },
        {
            "id": 1,
            "desc": "</code> 结束",
            "other_info": {
                "from": "system_para_6"
            },
            "dimension": "example_driven",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    return response.rstrip().endswith(\"</code>\")\n"
                }
            ],
            "score": false
        },
        {
            "id": 0,
            "desc": "开始 <code>",
            "other_info": {
                "from": "system_para_6"
            },
            "dimension": "example_driven",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    return response.lstrip().startswith(\"<code>\")\n"
                }
            ],
            "score": false
        },
        {
            "id": 1,
            "desc": "</code> 结束",
            "other_info": {
                "from": "system_para_6"
            },
            "dimension": "example_driven",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response):\n    return response.rstrip().endswith(\"</code>\")\n"
                }
            ],
            "score": false
        },
        {
            "id": -1,
            "desc": "In case (2), write this sub-question with an [END] mark as a leaf node.",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "If case (2), write this sub-question with an [END] mark as a leaf node.",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm_conditional_check",
                    "exec": "Does there exist sub-questions that satisfy case (2), which can be 'directly answered by analyzing the answers of at least two previously answered questions, such as comparing, judging, intersecting, or counting'? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                },
                {
                    "type": "llm",
                    "exec": "In the model response, are all sub-questions that satisfy case (2) written with an [END] mark as leaf nodes? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "In case (2), write this sub-question with an [END] mark as a leaf node.",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "If case (2), write this sub-question with an [END] mark as a leaf node.",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm_conditional_check",
                    "exec": "Does there exist sub-questions that satisfy case (2), which can be 'directly answered by analyzing the answers of at least two previously answered questions, such as comparing, judging, intersecting, or counting'? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                },
                {
                    "type": "llm",
                    "exec": "In the model response, are all sub-questions that satisfy case (2) written with an [END] mark as leaf nodes? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "In case (2), write this sub-question with an [END] mark as a leaf node.",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "If case (2), write this sub-question with an [END] mark as a leaf node.",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm_conditional_check",
                    "exec": "Does there exist sub-questions that satisfy case (2), which can be 'directly answered by analyzing the answers of at least two previously answered questions, such as comparing, judging, intersecting, or counting'? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                },
                {
                    "type": "llm",
                    "exec": "In the model response, are all sub-questions that satisfy case (2) written with an [END] mark as leaf nodes? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "use escape quotes \"\" to enclose work titles and function parameters",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Did the model use escape quotes \"\" to enclose all work titles and function parameters? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "In case (2), write this sub-question with an [END] mark as a leaf node.",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "If case (2), write this sub-question with an [END] mark as a leaf node.",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm_conditional_check",
                    "exec": "Does there exist sub-questions that satisfy case (2), which can be 'directly answered by analyzing the answers of at least two previously answered questions, such as comparing, judging, intersecting, or counting'? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                },
                {
                    "type": "llm",
                    "exec": "In the model response, are all sub-questions that satisfy case (2) written with an [END] mark as leaf nodes? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "In case (2), write this sub-question with an [END] mark as a leaf node.",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "If case (2), write this sub-question with an [END] mark as a leaf node.",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm_conditional_check",
                    "exec": "Does there exist sub-questions that satisfy case (2), which can be 'directly answered by analyzing the answers of at least two previously answered questions, such as comparing, judging, intersecting, or counting'? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                },
                {
                    "type": "llm",
                    "exec": "In the model response, are all sub-questions that satisfy case (2) written with an [END] mark as leaf nodes? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "For function leaf nodes, do not write nested functions such as Filter(Search(...))",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "If function leaf nodes, do not write nested functions such as Filter(Search(...)).",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm_conditional_check",
                    "exec": "Are there function leaf nodes in the question decomposition tree? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                },
                {
                    "type": "llm",
                    "exec": "For all function leaf nodes in the model response, did the model avoid from writing any nested functions such as Filter(Search(...))? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "use escape quotes \"\" to enclose work titles and function parameters",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Did the model use escape quotes \"\" to enclose all work titles and function parameters? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "In case (2), write this sub-question with an [END] mark as a leaf node.",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "If case (2), write this sub-question with an [END] mark as a leaf node.",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm_conditional_check",
                    "exec": "Does there exist sub-questions that satisfy case (2), which can be 'directly answered by analyzing the answers of at least two previously answered questions, such as comparing, judging, intersecting, or counting'? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                },
                {
                    "type": "llm",
                    "exec": "In the model response, are all sub-questions that satisfy case (2) written with an [END] mark as leaf nodes? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "use escape quotes \"\" to enclose work titles and function parameters",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Did the model use escape quotes \"\" to enclose all work titles and function parameters? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "In case (2), write this sub-question with an [END] mark as a leaf node.",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "If case (2), write this sub-question with an [END] mark as a leaf node.",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm_conditional_check",
                    "exec": "Does there exist sub-questions that satisfy case (2), which can be 'directly answered by analyzing the answers of at least two previously answered questions, such as comparing, judging, intersecting, or counting'? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                },
                {
                    "type": "llm",
                    "exec": "In the model response, are all sub-questions that satisfy case (2) written with an [END] mark as leaf nodes? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "use double quotes to enclose sub-questions and functions",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Did the model use double quotes to enclose all sub-questions and functions? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "use escape quotes \"\" to enclose work titles and function parameters",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Did the model use escape quotes \"\" to enclose all work titles and function parameters? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "In case (2), write this sub-question with an [END] mark as a leaf node.",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "If case (2), write this sub-question with an [END] mark as a leaf node.",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm_conditional_check",
                    "exec": "Does there exist sub-questions that satisfy case (2), which can be 'directly answered by analyzing the answers of at least two previously answered questions, such as comparing, judging, intersecting, or counting'? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                },
                {
                    "type": "llm",
                    "exec": "In the model response, are all sub-questions that satisfy case (2) written with an [END] mark as leaf nodes? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "For [END] leaf questions, format your question as 'Given answers of [q_idx_1] and [q_idx_2], ...', where [q_idx_1] and [q_idx_2] are question indices of the previously answered questions required to answer this [END] question.",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "If [END] leaf questions, format your question as 'Given answers of [q_idx_1] and [q_idx_2], ...', where [q_idx_1] and [q_idx_2] are question indices of the previously answered questions required to answer this [END] question..",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm_conditional_check",
                    "exec": "Are there [END] leaf questions in the tree? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                },
                {
                    "type": "llm",
                    "exec": "Are all [END] leaf questions formatted as 'Given answers of [q_idx_1] and [q_idx_2], ...', where [q_idx_1] and [q_idx_2] are question indices of previously answered questions required to answer this [END] question? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "use escape quotes \"\" to enclose work titles and function parameters",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Did the model use escape quotes \"\" to enclose all work titles and function parameters? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "In case (2), write this sub-question with an [END] mark as a leaf node.",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "If case (2), write this sub-question with an [END] mark as a leaf node.",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm_conditional_check",
                    "exec": "Does there exist sub-questions that satisfy case (2), which can be 'directly answered by analyzing the answers of at least two previously answered questions, such as comparing, judging, intersecting, or counting'? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                },
                {
                    "type": "llm",
                    "exec": "In the model response, are all sub-questions that satisfy case (2) written with an [END] mark as leaf nodes? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "use escape quotes \"\" to enclose work titles and function parameters",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Did the model use escape quotes \"\" to enclose all work titles and function parameters? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "In case (1), write this sub-question with its corresponding function call as a leaf node.",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "If case (1), write this sub-question with its corresponding function call as a leaf node.",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm_conditional_check",
                    "exec": "Does there exist sub-questions that satisfy case (1), which can be 'directly answered by calling one of the three atomic functions Search(), Relate(), Filter()'? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                },
                {
                    "type": "llm",
                    "exec": "In the model response, are all sub-questions that satisfy case (1) written with their corresponding function calls as leaf nodes? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "In case (2), write this sub-question with an [END] mark as a leaf node.",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "If case (2), write this sub-question with an [END] mark as a leaf node.",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm_conditional_check",
                    "exec": "Does there exist sub-questions that satisfy case (2), which can be 'directly answered by analyzing the answers of at least two previously answered questions, such as comparing, judging, intersecting, or counting'? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                },
                {
                    "type": "llm",
                    "exec": "In the model response, are all sub-questions that satisfy case (2) written with an [END] mark as leaf nodes? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "For function leaf nodes, do not write nested functions such as Filter(Search(...))",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "If function leaf nodes, do not write nested functions such as Filter(Search(...)).",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm_conditional_check",
                    "exec": "Are there function leaf nodes in the question decomposition tree? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                },
                {
                    "type": "llm",
                    "exec": "For all function leaf nodes in the model response, did the model avoid from writing any nested functions such as Filter(Search(...))? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "use double quotes to enclose sub-questions and functions",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Did the model use double quotes to enclose all sub-questions and functions? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "use escape quotes \"\" to enclose work titles and function parameters",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Did the model use escape quotes \"\" to enclose all work titles and function parameters? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "In case (2), write this sub-question with an [END] mark as a leaf node.",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "If case (2), write this sub-question with an [END] mark as a leaf node.",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm_conditional_check",
                    "exec": "Does there exist sub-questions that satisfy case (2), which can be 'directly answered by analyzing the answers of at least two previously answered questions, such as comparing, judging, intersecting, or counting'? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                },
                {
                    "type": "llm",
                    "exec": "In the model response, are all sub-questions that satisfy case (2) written with an [END] mark as leaf nodes? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "use escape quotes \"\" to enclose work titles and function parameters",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Did the model use escape quotes \"\" to enclose all work titles and function parameters? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "In case (2), write this sub-question with an [END] mark as a leaf node.",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "If case (2), write this sub-question with an [END] mark as a leaf node.",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm_conditional_check",
                    "exec": "Does there exist sub-questions that satisfy case (2), which can be 'directly answered by analyzing the answers of at least two previously answered questions, such as comparing, judging, intersecting, or counting'? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                },
                {
                    "type": "llm",
                    "exec": "In the model response, are all sub-questions that satisfy case (2) written with an [END] mark as leaf nodes? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "use escape quotes \"\" to enclose work titles and function parameters",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Did the model use escape quotes \"\" to enclose all work titles and function parameters? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "In case (2), write this sub-question with an [END] mark as a leaf node.",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "If case (2), write this sub-question with an [END] mark as a leaf node.",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm_conditional_check",
                    "exec": "Does there exist sub-questions that satisfy case (2), which can be 'directly answered by analyzing the answers of at least two previously answered questions, such as comparing, judging, intersecting, or counting'? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                },
                {
                    "type": "llm",
                    "exec": "In the model response, are all sub-questions that satisfy case (2) written with an [END] mark as leaf nodes? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "In case (2), write this sub-question with an [END] mark as a leaf node.",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "If case (2), write this sub-question with an [END] mark as a leaf node.",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm_conditional_check",
                    "exec": "Does there exist sub-questions that satisfy case (2), which can be 'directly answered by analyzing the answers of at least two previously answered questions, such as comparing, judging, intersecting, or counting'? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                },
                {
                    "type": "llm",
                    "exec": "In the model response, are all sub-questions that satisfy case (2) written with an [END] mark as leaf nodes? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "For function leaf nodes, do not write nested functions such as Filter(Search(...))",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "If function leaf nodes, do not write nested functions such as Filter(Search(...)).",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm_conditional_check",
                    "exec": "Are there function leaf nodes in the question decomposition tree? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                },
                {
                    "type": "llm",
                    "exec": "For all function leaf nodes in the model response, did the model avoid from writing any nested functions such as Filter(Search(...))? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "use escape quotes \"\" to enclose work titles and function parameters",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Did the model use escape quotes \"\" to enclose all work titles and function parameters? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "In case (2), write this sub-question with an [END] mark as a leaf node.",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "If case (2), write this sub-question with an [END] mark as a leaf node.",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm_conditional_check",
                    "exec": "Does there exist sub-questions that satisfy case (2), which can be 'directly answered by analyzing the answers of at least two previously answered questions, such as comparing, judging, intersecting, or counting'? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                },
                {
                    "type": "llm",
                    "exec": "In the model response, are all sub-questions that satisfy case (2) written with an [END] mark as leaf nodes? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "use double quotes to enclose sub-questions and functions",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Did the model use double quotes to enclose all sub-questions and functions? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "use escape quotes \"\" to enclose work titles and function parameters",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Did the model use escape quotes \"\" to enclose all work titles and function parameters? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "In case (2), write this sub-question with an [END] mark as a leaf node.",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "If case (2), write this sub-question with an [END] mark as a leaf node.",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm_conditional_check",
                    "exec": "Does there exist sub-questions that satisfy case (2), which can be 'directly answered by analyzing the answers of at least two previously answered questions, such as comparing, judging, intersecting, or counting'? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                },
                {
                    "type": "llm",
                    "exec": "In the model response, are all sub-questions that satisfy case (2) written with an [END] mark as leaf nodes? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "use escape quotes \"\" to enclose work titles and function parameters",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Did the model use escape quotes \"\" to enclose all work titles and function parameters? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "In case (2), write this sub-question with an [END] mark as a leaf node.",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "If case (2), write this sub-question with an [END] mark as a leaf node.",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm_conditional_check",
                    "exec": "Does there exist sub-questions that satisfy case (2), which can be 'directly answered by analyzing the answers of at least two previously answered questions, such as comparing, judging, intersecting, or counting'? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                },
                {
                    "type": "llm",
                    "exec": "In the model response, are all sub-questions that satisfy case (2) written with an [END] mark as leaf nodes? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "use escape quotes \"\" to enclose work titles and function parameters",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Did the model use escape quotes \"\" to enclose all work titles and function parameters? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "use escape quotes \"\" to enclose work titles and function parameters",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Did the model use escape quotes \"\" to enclose all work titles and function parameters? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "In case (2), write this sub-question with an [END] mark as a leaf node.",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "If case (2), write this sub-question with an [END] mark as a leaf node.",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm_conditional_check",
                    "exec": "Does there exist sub-questions that satisfy case (2), which can be 'directly answered by analyzing the answers of at least two previously answered questions, such as comparing, judging, intersecting, or counting'? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                },
                {
                    "type": "llm",
                    "exec": "In the model response, are all sub-questions that satisfy case (2) written with an [END] mark as leaf nodes? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "use double quotes to enclose sub-questions and functions",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Did the model use double quotes to enclose all sub-questions and functions? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "use double quotes to enclose sub-questions and functions",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Did the model use double quotes to enclose all sub-questions and functions? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "In case (2), write this sub-question with an [END] mark as a leaf node.",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "If case (2), write this sub-question with an [END] mark as a leaf node.",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm_conditional_check",
                    "exec": "Does there exist sub-questions that satisfy case (2), which can be 'directly answered by analyzing the answers of at least two previously answered questions, such as comparing, judging, intersecting, or counting'? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                },
                {
                    "type": "llm",
                    "exec": "In the model response, are all sub-questions that satisfy case (2) written with an [END] mark as leaf nodes? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "use escape quotes \"\" to enclose work titles and function parameters",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Did the model use escape quotes \"\" to enclose all work titles and function parameters? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "use escape quotes \"\" to enclose work titles and function parameters",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Did the model use escape quotes \"\" to enclose all work titles and function parameters? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "In case (2), write this sub-question with an [END] mark as a leaf node.",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "If case (2), write this sub-question with an [END] mark as a leaf node.",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm_conditional_check",
                    "exec": "Does there exist sub-questions that satisfy case (2), which can be 'directly answered by analyzing the answers of at least two previously answered questions, such as comparing, judging, intersecting, or counting'? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                },
                {
                    "type": "llm",
                    "exec": "In the model response, are all sub-questions that satisfy case (2) written with an [END] mark as leaf nodes? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "use escape quotes \"\" to enclose work titles and function parameters",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Did the model use escape quotes \"\" to enclose all work titles and function parameters? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "use escape quotes \"\" to enclose work titles and function parameters",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Did the model use escape quotes \"\" to enclose all work titles and function parameters? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "In case (2), write this sub-question with an [END] mark as a leaf node.",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "If case (2), write this sub-question with an [END] mark as a leaf node.",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm_conditional_check",
                    "exec": "Does there exist sub-questions that satisfy case (2), which can be 'directly answered by analyzing the answers of at least two previously answered questions, such as comparing, judging, intersecting, or counting'? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                },
                {
                    "type": "llm",
                    "exec": "In the model response, are all sub-questions that satisfy case (2) written with an [END] mark as leaf nodes? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "use escape quotes \"\" to enclose work titles and function parameters",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Did the model use escape quotes \"\" to enclose all work titles and function parameters? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "In case (2), write this sub-question with an [END] mark as a leaf node.",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "If case (2), write this sub-question with an [END] mark as a leaf node.",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm_conditional_check",
                    "exec": "Does there exist sub-questions that satisfy case (2), which can be 'directly answered by analyzing the answers of at least two previously answered questions, such as comparing, judging, intersecting, or counting'? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                },
                {
                    "type": "llm",
                    "exec": "In the model response, are all sub-questions that satisfy case (2) written with an [END] mark as leaf nodes? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "use double quotes to enclose sub-questions and functions",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Did the model use double quotes to enclose all sub-questions and functions? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "use escape quotes \"\" to enclose work titles and function parameters",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Did the model use escape quotes \"\" to enclose all work titles and function parameters? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "In case (2), write this sub-question with an [END] mark as a leaf node.",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "If case (2), write this sub-question with an [END] mark as a leaf node.",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm_conditional_check",
                    "exec": "Does there exist sub-questions that satisfy case (2), which can be 'directly answered by analyzing the answers of at least two previously answered questions, such as comparing, judging, intersecting, or counting'? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                },
                {
                    "type": "llm",
                    "exec": "In the model response, are all sub-questions that satisfy case (2) written with an [END] mark as leaf nodes? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "use escape quotes \"\" to enclose work titles and function parameters",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Did the model use escape quotes \"\" to enclose all work titles and function parameters? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "use escape quotes \"\" to enclose work titles and function parameters",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Did the model use escape quotes \"\" to enclose all work titles and function parameters? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "In case (2), write this sub-question with an [END] mark as a leaf node.",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "If case (2), write this sub-question with an [END] mark as a leaf node.",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm_conditional_check",
                    "exec": "Does there exist sub-questions that satisfy case (2), which can be 'directly answered by analyzing the answers of at least two previously answered questions, such as comparing, judging, intersecting, or counting'? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                },
                {
                    "type": "llm",
                    "exec": "In the model response, are all sub-questions that satisfy case (2) written with an [END] mark as leaf nodes? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "use escape quotes \"\" to enclose work titles and function parameters",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "",
                "complete_instruction_para": []
            },
            "dimension": "example_driven",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Did the model use escape quotes \"\" to enclose all work titles and function parameters? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        }
    ],
    "conditional": [
        {
            "id": 2,
            "desc": "在写代码时你需要综合考虑最终目标<final_goal></final_goal>和本轮的任务<task></task>。",
            "other_info": {
                "from": "system_para_1",
                "first_task": "我将使用这些信息来回答用户的问题"
            },
            "is_meta": true,
            "type": "semantic",
            "dimension": "conditional",
            "dependency": [],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "检查 response 是否严格按 task 要求执行核心操作（允许 print 辅助输出），没有执行任何其他额外操作，直接回答 YES 或 NO。\n\ntask: 我将使用这些信息来回答用户的问题\n\nresponse:{response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 9,
            "desc": "finish函数将在用于返回用户的final_goal的结果时结束推理流程",
            "other_info": {},
            "dimension": "conditional",
            "type": "formatting",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response: str) -> bool:\n    if \"finish()\" in response:\n        return True\n    else:\n        return False\n"
                }
            ],
            "score": false
        },
        {
            "id": 2,
            "desc": "在写代码时你需要综合考虑最终目标<final_goal></final_goal>和本轮的任务<task></task>。",
            "other_info": {
                "from": "system_para_1",
                "first_task": "根据查询结果，我已经获得了江苏易智供应链管理有限公司的统一社会信用代码"
            },
            "is_meta": true,
            "type": "semantic",
            "dimension": "conditional",
            "dependency": [],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "检查 response 是否严格按 task 要求执行核心操作（允许 print 辅助输出），没有执行任何其他额外操作，直接回答 YES 或 NO。\n\ntask: 根据查询结果，我已经获得了江苏易智供应链管理有限公司的统一社会信用代码\n\nresponse:{response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 2,
            "desc": "在写代码时你需要综合考虑最终目标<final_goal></final_goal>和本轮的任务<task></task>。",
            "other_info": {
                "from": "system_para_1",
                "first_task": "调用get_company_register找到该公司的'注册资本'"
            },
            "is_meta": true,
            "type": "semantic",
            "dimension": "conditional",
            "dependency": [],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "检查 response 是否严格按 task 要求执行核心操作（允许 print 辅助输出），没有执行任何其他额外操作，直接回答 YES 或 NO。\n\ntask: 调用get_company_register找到该公司的'注册资本'\n\nresponse:{response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 2,
            "desc": "在写代码时你需要综合考虑最终目标<final_goal></final_goal>和本轮的任务<task></task>。",
            "other_info": {
                "from": "system_para_1",
                "first_task": "我需要使用get_company_register来通过公司名称查找公司的注册信息"
            },
            "is_meta": true,
            "type": "semantic",
            "dimension": "conditional",
            "dependency": [],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "检查 response 是否严格按 task 要求执行核心操作（允许 print 辅助输出），没有执行任何其他额外操作，直接回答 YES 或 NO。\n\ntask: 我需要使用get_company_register来通过公司名称查找公司的注册信息\n\nresponse:{response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 2,
            "desc": "在写代码时你需要综合考虑最终目标<final_goal></final_goal>和本轮的任务<task></task>。",
            "other_info": {
                "from": "system_para_1",
                "first_task": "将1800万元转换为亿元，并精确到小数点后两位"
            },
            "is_meta": true,
            "type": "semantic",
            "dimension": "conditional",
            "dependency": [],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "检查 response 是否严格按 task 要求执行核心操作（允许 print 辅助输出），没有执行任何其他额外操作，直接回答 YES 或 NO。\n\ntask: 将1800万元转换为亿元，并精确到小数点后两位\n\nresponse:{response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 2,
            "desc": "在写代码时你需要综合考虑最终目标<final_goal></final_goal>和本轮的任务<task></task>。",
            "other_info": {
                "from": "system_para_1",
                "first_task": "根据查询结果，安徽中能电源有限公司的母公司是天能电池集团股份有限公司。"
            },
            "is_meta": true,
            "type": "semantic",
            "dimension": "conditional",
            "dependency": [],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "检查 response 是否严格按 task 要求执行核心操作（允许 print 辅助输出），没有执行任何其他额外操作，直接回答 YES 或 NO。\n\ntask: 根据查询结果，安徽中能电源有限公司的母公司是天能电池集团股份有限公司。\n\nresponse:{response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 2,
            "desc": "在写代码时你需要综合考虑最终目标<final_goal></final_goal>和本轮的任务<task></task>。",
            "other_info": {
                "from": "system_para_1",
                "first_task": "根据查询结果，上海家化的全称是“上海家化联合股份有限公司”。"
            },
            "is_meta": true,
            "type": "semantic",
            "dimension": "conditional",
            "dependency": [],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "检查 response 是否严格按 task 要求执行核心操作（允许 print 辅助输出），没有执行任何其他额外操作，直接回答 YES 或 NO。\n\ntask: 根据查询结果，上海家化的全称是“上海家化联合股份有限公司”。\n\nresponse:{response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 2,
            "desc": "在写代码时你需要综合考虑最终目标<final_goal></final_goal>和本轮的任务<task></task>。",
            "other_info": {
                "from": "system_para_1",
                "first_task": "筛选出参股比例为'100.0'且投资金额超过12.5亿元的子公司信息"
            },
            "is_meta": true,
            "type": "semantic",
            "dimension": "conditional",
            "dependency": [],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "检查 response 是否严格按 task 要求执行核心操作（允许 print 辅助输出），没有执行任何其他额外操作，直接回答 YES 或 NO。\n\ntask: 筛选出参股比例为'100.0'且投资金额超过12.5亿元的子公司信息\n\nresponse:{response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 2,
            "desc": "在写代码时你需要综合考虑最终目标<final_goal></final_goal>和本轮的任务<task></task>。",
            "other_info": {
                "from": "system_para_1",
                "first_task": "将9000万元转换为亿元，并保留两位小数。"
            },
            "is_meta": true,
            "type": "semantic",
            "dimension": "conditional",
            "dependency": [],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "检查 response 是否严格按 task 要求执行核心操作（允许 print 辅助输出），没有执行任何其他额外操作，直接回答 YES 或 NO。\n\ntask: 将9000万元转换为亿元，并保留两位小数。\n\nresponse:{response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 2,
            "desc": "在写代码时你需要综合考虑最终目标<final_goal></final_goal>和本轮的任务<task></task>。",
            "other_info": {
                "from": "system_para_1",
                "first_task": "根据查询结果，案件编号为（2019）京0105民初9223号的原告是北京市桃李食品有限公司，被告是北京全时叁陆伍连锁便利店有限公司。"
            },
            "is_meta": true,
            "type": "semantic",
            "dimension": "conditional",
            "dependency": [],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "检查 response 是否严格按 task 要求执行核心操作（允许 print 辅助输出），没有执行任何其他额外操作，直接回答 YES 或 NO。\n\ntask: 根据查询结果，案件编号为（2019）京0105民初9223号的原告是北京市桃李食品有限公司，被告是北京全时叁陆伍连锁便利店有限公司。\n\nresponse:{response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 2,
            "desc": "在写代码时你需要综合考虑最终目标<final_goal></final_goal>和本轮的任务<task></task>。",
            "other_info": {
                "from": "system_para_1",
                "first_task": "根据获取的案件信息，原告是中国建设银行股份有限公司常州经济开发区支行，被告包括常州市互联涂料有限公司、江苏新互盛电缆有限公司、常州格林电力机械制造有限公司、周某某、任某某、常州市神猴焊丝有限公司。"
            },
            "is_meta": true,
            "type": "semantic",
            "dimension": "conditional",
            "dependency": [],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "检查 response 是否严格按 task 要求执行核心操作（允许 print 辅助输出），没有执行任何其他额外操作，直接回答 YES 或 NO。\n\ntask: 根据获取的案件信息，原告是中国建设银行股份有限公司常州经济开发区支行，被告包括常州市互联涂料有限公司、江苏新互盛电缆有限公司、常州格林电力机械制造有限公司、周某某、任某某、常州市神猴焊丝有限公司。\n\nresponse:{response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 2,
            "desc": "在写代码时你需要综合考虑最终目标<final_goal></final_goal>和本轮的任务<task></task>。",
            "other_info": {
                "from": "system_para_1",
                "first_task": "我需要使用 get_company_info 函数获取华灿光电股份有限公司的相关信息。"
            },
            "is_meta": true,
            "type": "semantic",
            "dimension": "conditional",
            "dependency": [],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "检查 response 是否严格按 task 要求执行核心操作（允许 print 辅助输出），没有执行任何其他额外操作，直接回答 YES 或 NO。\n\ntask: 我需要使用 get_company_info 函数获取华灿光电股份有限公司的相关信息。\n\nresponse:{response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 2,
            "desc": "在写代码时你需要综合考虑最终目标<final_goal></final_goal>和本轮的任务<task></task>。",
            "other_info": {
                "from": "system_para_1",
                "first_task": "将投资金额1.31亿转换为亿元并保留2位小数"
            },
            "is_meta": true,
            "type": "semantic",
            "dimension": "conditional",
            "dependency": [],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "检查 response 是否严格按 task 要求执行核心操作（允许 print 辅助输出），没有执行任何其他额外操作，直接回答 YES 或 NO。\n\ntask: 将投资金额1.31亿转换为亿元并保留2位小数\n\nresponse:{response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 2,
            "desc": "在写代码时你需要综合考虑最终目标<final_goal></final_goal>和本轮的任务<task></task>。",
            "other_info": {
                "from": "system_para_1",
                "first_task": "根据查询结果，唐山港的全称是“唐山港集团股份有限公司”。"
            },
            "is_meta": true,
            "type": "semantic",
            "dimension": "conditional",
            "dependency": [],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "检查 response 是否严格按 task 要求执行核心操作（允许 print 辅助输出），没有执行任何其他额外操作，直接回答 YES 或 NO。\n\ntask: 根据查询结果，唐山港的全称是“唐山港集团股份有限公司”。\n\nresponse:{response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 5,
            "desc": "必须使用get_lawfirm_info查看律师事务所的负责人，不要直接使用案件的\"被告律师事务所\"",
            "other_info": {
                "available_packages": [
                    "math",
                    "time",
                    "stat",
                    "queue",
                    "itertools",
                    "statistics",
                    "random",
                    "collections",
                    "unicodedata"
                ]
            },
            "dimension": "conditional",
            "type": "resource",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm_conditional_check",
                    "exec": "response是否提及律师事务所。直接回答 YES 或 NO。\n\nresponse: {response}"
                },
                {
                    "type": "code",
                    "exec": "\ndef check_following(response: str) -> bool:\n    if \"get_lawfirm_info\" in response:\n        return True\n    else:\n        return False\n"
                }
            ],
            "score": false
        },
        {
            "id": 2,
            "desc": "在写代码时你需要综合考虑最终目标<final_goal></final_goal>和本轮的任务<task></task>。",
            "other_info": {
                "from": "system_para_1",
                "first_task": "使用get_company_info来获取甘肃陇神戎发药业股份有限公司的公司全称"
            },
            "is_meta": true,
            "type": "semantic",
            "dimension": "conditional",
            "dependency": [],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "检查 response 是否严格按 task 要求执行核心操作（允许 print 辅助输出），没有执行任何其他额外操作，直接回答 YES 或 NO。\n\ntask: 使用get_company_info来获取甘肃陇神戎发药业股份有限公司的公司全称\n\nresponse:{response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 2,
            "desc": "在写代码时你需要综合考虑最终目标<final_goal></final_goal>和本轮的任务<task></task>。",
            "other_info": {
                "from": "system_para_1",
                "first_task": "我需要使用get_sub_company_info函数来查询安徽中能电源有限公司的母公司信息"
            },
            "is_meta": true,
            "type": "semantic",
            "dimension": "conditional",
            "dependency": [],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "检查 response 是否严格按 task 要求执行核心操作（允许 print 辅助输出），没有执行任何其他额外操作，直接回答 YES 或 NO。\n\ntask: 我需要使用get_sub_company_info函数来查询安徽中能电源有限公司的母公司信息\n\nresponse:{response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 2,
            "desc": "在写代码时你需要综合考虑最终目标<final_goal></final_goal>和本轮的任务<task></task>。",
            "other_info": {
                "from": "system_para_1",
                "first_task": "使用 get_company_info 函数获取金宏气体股份有限公司的代码、法人代表以及董秘的详细信息"
            },
            "is_meta": true,
            "type": "semantic",
            "dimension": "conditional",
            "dependency": [],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "检查 response 是否严格按 task 要求执行核心操作（允许 print 辅助输出），没有执行任何其他额外操作，直接回答 YES 或 NO。\n\ntask: 使用 get_company_info 函数获取金宏气体股份有限公司的代码、法人代表以及董秘的详细信息\n\nresponse:{response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 2,
            "desc": "在写代码时你需要综合考虑最终目标<final_goal></final_goal>和本轮的任务<task></task>。",
            "other_info": {
                "from": "system_para_1",
                "first_task": "根据获取的信息，华灿光电股份有限公司的股票代码是300323。"
            },
            "is_meta": true,
            "type": "semantic",
            "dimension": "conditional",
            "dependency": [],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "检查 response 是否严格按 task 要求执行核心操作（允许 print 辅助输出），没有执行任何其他额外操作，直接回答 YES 或 NO。\n\ntask: 根据获取的信息，华灿光电股份有限公司的股票代码是300323。\n\nresponse:{response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 2,
            "desc": "在写代码时你需要综合考虑最终目标<final_goal></final_goal>和本轮的任务<task></task>。",
            "other_info": {
                "from": "system_para_1",
                "first_task": "根据获取的信息，金宏气体股份有限公司的公司代码是688106，法人代表是金向华，董秘是陈莹。"
            },
            "is_meta": true,
            "type": "semantic",
            "dimension": "conditional",
            "dependency": [],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "检查 response 是否严格按 task 要求执行核心操作（允许 print 辅助输出），没有执行任何其他额外操作，直接回答 YES 或 NO。\n\ntask: 根据获取的信息，金宏气体股份有限公司的公司代码是688106，法人代表是金向华，董秘是陈莹。\n\nresponse:{response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 2,
            "desc": "在写代码时你需要综合考虑最终目标<final_goal></final_goal>和本轮的任务<task></task>。",
            "other_info": {
                "from": "system_para_1",
                "first_task": "我需要使用get_legal_document来获取（2020）苏0492民初2530号案件的文档内容。"
            },
            "is_meta": true,
            "type": "semantic",
            "dimension": "conditional",
            "dependency": [],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "检查 response 是否严格按 task 要求执行核心操作（允许 print 辅助输出），没有执行任何其他额外操作，直接回答 YES 或 NO。\n\ntask: 我需要使用get_legal_document来获取（2020）苏0492民初2530号案件的文档内容。\n\nresponse:{response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 2,
            "desc": "在写代码时你需要综合考虑最终目标<final_goal></final_goal>和本轮的任务<task></task>。",
            "other_info": {
                "from": "system_para_1",
                "first_task": "使用get_company_register来通过公司名称查找公司的注册信息"
            },
            "is_meta": true,
            "type": "semantic",
            "dimension": "conditional",
            "dependency": [],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "检查 response 是否严格按 task 要求执行核心操作（允许 print 辅助输出），没有执行任何其他额外操作，直接回答 YES 或 NO。\n\ntask: 使用get_company_register来通过公司名称查找公司的注册信息\n\nresponse:{response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 2,
            "desc": "在写代码时你需要综合考虑最终目标<final_goal></final_goal>和本轮的任务<task></task>。",
            "other_info": {
                "from": "system_para_1",
                "first_task": "我需要使用get_company_info函数来查询唐山港的完整名称"
            },
            "is_meta": true,
            "type": "semantic",
            "dimension": "conditional",
            "dependency": [],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "检查 response 是否严格按 task 要求执行核心操作（允许 print 辅助输出），没有执行任何其他额外操作，直接回答 YES 或 NO。\n\ntask: 我需要使用get_company_info函数来查询唐山港的完整名称\n\nresponse:{response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 2,
            "desc": "在写代码时你需要综合考虑最终目标<final_goal></final_goal>和本轮的任务<task></task>。",
            "other_info": {
                "from": "system_para_1",
                "first_task": "我将使用get_legal_document函数获取案件编号为（2019）京0105民初9223号的法律文书内容"
            },
            "is_meta": true,
            "type": "semantic",
            "dimension": "conditional",
            "dependency": [],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "检查 response 是否严格按 task 要求执行核心操作（允许 print 辅助输出），没有执行任何其他额外操作，直接回答 YES 或 NO。\n\ntask: 我将使用get_legal_document函数获取案件编号为（2019）京0105民初9223号的法律文书内容\n\nresponse:{response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 2,
            "desc": "在写代码时你需要综合考虑最终目标<final_goal></final_goal>和本轮的任务<task></task>。",
            "other_info": {
                "from": "system_para_1",
                "first_task": "我需要获取金诚信矿业管理股份有限公司投资的子公司信息。"
            },
            "is_meta": true,
            "type": "semantic",
            "dimension": "conditional",
            "dependency": [],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "检查 response 是否严格按 task 要求执行核心操作（允许 print 辅助输出），没有执行任何其他额外操作，直接回答 YES 或 NO。\n\ntask: 我需要获取金诚信矿业管理股份有限公司投资的子公司信息。\n\nresponse:{response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 5,
            "desc": "必须使用get_address_info获得企业地址具体的省份，城市，区县。",
            "other_info": {
                "available_packages": [
                    "math",
                    "time",
                    "stat",
                    "queue",
                    "itertools",
                    "statistics",
                    "random",
                    "collections",
                    "unicodedata"
                ]
            },
            "dimension": "conditional",
            "type": "resource",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm_conditional_check",
                    "exec": "response是否希望获得企业地址具体的省份或者城市或者区县。直接回答 YES 或 NO。\n\nresponse: {response}"
                },
                {
                    "type": "code",
                    "exec": "\ndef check_following(response: str) -> bool:\n    if \"get_address_info\" in response:\n        return True\n    else:\n        return False\n"
                }
            ],
            "score": false
        },
        {
            "id": 5,
            "desc": "必须使用get_address_info获得企业地址具体的省份，城市，区县。",
            "other_info": {
                "available_packages": [
                    "math",
                    "time",
                    "stat",
                    "queue",
                    "itertools",
                    "statistics",
                    "random",
                    "collections",
                    "unicodedata"
                ]
            },
            "dimension": "conditional",
            "type": "resource",
            "is_meta": false,
            "evaluation": [
                {
                    "type": "llm_conditional_check",
                    "exec": "response是否希望获得企业地址具体的省份或者城市或者区县。直接回答 YES 或 NO。\n\nresponse: {response}"
                },
                {
                    "type": "code",
                    "exec": "\ndef check_following(response: str) -> bool:\n    if \"get_address_info\" in response:\n        return True\n    else:\n        return False\n"
                }
            ],
            "score": false
        },
        {
            "id": 3,
            "desc": "If using LaTeX, use double $$ as a delimiter instead of single $.",
            "other_info": {
                "from": "system_para_0",
                "type_explanation": "This constraint controls the structure and presentation format of the output, specifically the syntax norms of LaTeX by requiring double $$ delimiters rather than single $. This aligns directly with formatting requirements.",
                "meta_expalnation": "The given constraint directly governs the output format in terms of how LaTeX is delimited. It specifies a content-related output rule ('use double $$ instead of single $') based on a condition ('if using LaTeX'). Therefore, it is not a high-level rule managing other constraints, and does not qualify as a Meta Constraint.",
                "evaluation_type": {
                    "constraint_type": "code",
                    "explanation": "The constraint pertains to a specific formatting rule (using double $$ instead of single $), which can be validated through simple pattern matching and logic in code."
                },
                "evaluation_generation_success": true
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\nimport re\n\ndef check_following(response: str) -> bool:\n    if re.search(r'(?<!\\$)\\$(?!\\$)', response):\n        return False\n    count = response.count('$$')\n    if count % 2 != 0:\n        return False\n    return True\n"
                }
            ],
            "state": "modified",
            "score": false
        },
        {
            "id": 4,
            "desc": "Add visual elements descriptions (charts, diagrams, infographics).",
            "other_info": {
                "from": "system_para_1",
                "type_explanation": "The constraint focuses on enhancing the meaningfulness and completeness of the output by including descriptions of visual elements such as charts, diagrams, and infographics. This aligns with the semantic category, as it emphasizes the need for content that conveys information effectively and meets specific requirements in terms of completeness.",
                "meta_expalnation": "The given constraint directly specifies an output requirement (adding visual element descriptions) rather than defining how to manage other constraints. It directly constrains the model’s content and format rather than operating as a high-level rule for handling multiple constraints.",
                "evaluation_type": {
                    "constraint_type": "llm",
                    "explanation": "The task requires understanding the visual elements' content and generating descriptive text that complements them, which involves semantic understanding and subjective assessment of relevance and accuracy, making it suited for LLM-based validation."
                },
                "evaluation_generation_success": true
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Does the model's response explicitly include descriptions for visual elements such as charts, diagrams, or infographics? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "state": "success",
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 4,
            "desc": "Design multiplayer interactions if applicable.",
            "other_info": {
                "from": "system_para_0",
                "type_explanation": "This constraint relates to the meaning and content of the output, as it specifies the inclusion of multiplayer interactions in the design if the context allows for it. The use of 'if applicable' indicates a conditional requirement for semantic completeness and relevance rather than formatting or resource-based restrictions.",
                "meta_expalnation": "The constraint 'Design multiplayer interactions if applicable' directly provides a conditional requirement related to the content of an output (designing multiplayer interactions). It does not specify how to manage or manipulate other constraints (e.g., prioritization, selection, composition, etc.), so it is not a meta constraint.",
                "evaluation_type": {
                    "constraint_type": "llm",
                    "explanation": "Designing multiplayer interactions is a semantic task requiring subjective understanding of game design principles, player engagement, and context-specific needs. It involves conceptualizing how players interact, which goes beyond code-level or extraction-based validation."
                },
                "evaluation_generation_success": true
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Does the model's response include a clear and specific design for multiplayer interactions in the game? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "state": "success",
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 4,
            "desc": "Add visual elements descriptions (charts, diagrams, infographics).",
            "other_info": {
                "from": "system_para_1",
                "type_explanation": "The constraint focuses on enhancing the meaningfulness and completeness of the output by including descriptions of visual elements such as charts, diagrams, and infographics. This aligns with the semantic category, as it emphasizes the need for content that conveys information effectively and meets specific requirements in terms of completeness.",
                "meta_expalnation": "The given constraint directly specifies an output requirement (adding visual element descriptions) rather than defining how to manage other constraints. It directly constrains the model’s content and format rather than operating as a high-level rule for handling multiple constraints.",
                "evaluation_type": {
                    "constraint_type": "llm",
                    "explanation": "The task requires understanding the visual elements' content and generating descriptive text that complements them, which involves semantic understanding and subjective assessment of relevance and accuracy, making it suited for LLM-based validation."
                },
                "evaluation_generation_success": true
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Does the model's response explicitly include descriptions for visual elements such as charts, diagrams, or infographics? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "state": "success",
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 4,
            "desc": "Design multiplayer interactions if applicable.",
            "other_info": {
                "from": "system_para_0",
                "type_explanation": "This constraint relates to the meaning and content of the output, as it specifies the inclusion of multiplayer interactions in the design if the context allows for it. The use of 'if applicable' indicates a conditional requirement for semantic completeness and relevance rather than formatting or resource-based restrictions.",
                "meta_expalnation": "The constraint 'Design multiplayer interactions if applicable' directly provides a conditional requirement related to the content of an output (designing multiplayer interactions). It does not specify how to manage or manipulate other constraints (e.g., prioritization, selection, composition, etc.), so it is not a meta constraint.",
                "evaluation_type": {
                    "constraint_type": "llm",
                    "explanation": "Designing multiplayer interactions is a semantic task requiring subjective understanding of game design principles, player engagement, and context-specific needs. It involves conceptualizing how players interact, which goes beyond code-level or extraction-based validation."
                },
                "evaluation_generation_success": true
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Does the model's response include a clear and specific design for multiplayer interactions in the game? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "state": "success",
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 4,
            "desc": "Design multiplayer interactions if applicable.",
            "other_info": {
                "from": "system_para_0",
                "type_explanation": "This constraint relates to the meaning and content of the output, as it specifies the inclusion of multiplayer interactions in the design if the context allows for it. The use of 'if applicable' indicates a conditional requirement for semantic completeness and relevance rather than formatting or resource-based restrictions.",
                "meta_expalnation": "The constraint 'Design multiplayer interactions if applicable' directly provides a conditional requirement related to the content of an output (designing multiplayer interactions). It does not specify how to manage or manipulate other constraints (e.g., prioritization, selection, composition, etc.), so it is not a meta constraint.",
                "evaluation_type": {
                    "constraint_type": "llm",
                    "explanation": "Designing multiplayer interactions is a semantic task requiring subjective understanding of game design principles, player engagement, and context-specific needs. It involves conceptualizing how players interact, which goes beyond code-level or extraction-based validation."
                },
                "evaluation_generation_success": true
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Does the model's response include a clear and specific design for multiplayer interactions in the game? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "state": "success",
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 6,
            "desc": "Plan for multiplayer infrastructure if applicable.",
            "other_info": {
                "from": "system_para_0",
                "type_explanation": "The constraint focuses on the content's logic and completeness by requiring consideration of multiplayer infrastructure, but only if it is applicable. This relates to ensuring meaningful and accurate content based on situational conditions (conditional logic). The emphasis is on the conceptual requirement rather than formatting or resource usage.",
                "meta_expalnation": "This constraint directly guides the content or behavior of the model by specifying a conditional requirement related to multiplayer infrastructure. It does not involve high-level rules for managing other constraints, such as selecting, prioritizing, or combining them.",
                "evaluation_type": {
                    "constraint_type": "llm",
                    "explanation": "This constraint requires a semantic understanding of multiplayer infrastructure, including assessing design choices, networking models, and scalability. It cannot be verified through simple code but rather requires an LLM's broader expertise in game development concepts."
                },
                "evaluation_generation_success": true
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Does the model's response include a clear and specific plan for multiplayer infrastructure, addressing components such as server architecture, networking protocols, and player synchronization? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "state": "modified",
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 6,
            "desc": "Plan for multiplayer infrastructure if applicable.",
            "other_info": {
                "from": "system_para_0",
                "type_explanation": "The constraint focuses on the content's logic and completeness by requiring consideration of multiplayer infrastructure, but only if it is applicable. This relates to ensuring meaningful and accurate content based on situational conditions (conditional logic). The emphasis is on the conceptual requirement rather than formatting or resource usage.",
                "meta_expalnation": "This constraint directly guides the content or behavior of the model by specifying a conditional requirement related to multiplayer infrastructure. It does not involve high-level rules for managing other constraints, such as selecting, prioritizing, or combining them.",
                "evaluation_type": {
                    "constraint_type": "llm",
                    "explanation": "This constraint requires a semantic understanding of multiplayer infrastructure, including assessing design choices, networking models, and scalability. It cannot be verified through simple code but rather requires an LLM's broader expertise in game development concepts."
                },
                "evaluation_generation_success": true
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Does the model's response include a clear and specific plan for multiplayer infrastructure, addressing components such as server architecture, networking protocols, and player synchronization? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "state": "modified",
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 4,
            "desc": "Design multiplayer interactions if applicable.",
            "other_info": {
                "from": "system_para_0",
                "type_explanation": "This constraint relates to the meaning and content of the output, as it specifies the inclusion of multiplayer interactions in the design if the context allows for it. The use of 'if applicable' indicates a conditional requirement for semantic completeness and relevance rather than formatting or resource-based restrictions.",
                "meta_expalnation": "The constraint 'Design multiplayer interactions if applicable' directly provides a conditional requirement related to the content of an output (designing multiplayer interactions). It does not specify how to manage or manipulate other constraints (e.g., prioritization, selection, composition, etc.), so it is not a meta constraint.",
                "evaluation_type": {
                    "constraint_type": "llm",
                    "explanation": "Designing multiplayer interactions is a semantic task requiring subjective understanding of game design principles, player engagement, and context-specific needs. It involves conceptualizing how players interact, which goes beyond code-level or extraction-based validation."
                },
                "evaluation_generation_success": true
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Does the model's response include a clear and specific design for multiplayer interactions in the game? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "state": "success",
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 6,
            "desc": "Plan for multiplayer infrastructure if applicable.",
            "other_info": {
                "from": "system_para_0",
                "type_explanation": "The constraint focuses on the content's logic and completeness by requiring consideration of multiplayer infrastructure, but only if it is applicable. This relates to ensuring meaningful and accurate content based on situational conditions (conditional logic). The emphasis is on the conceptual requirement rather than formatting or resource usage.",
                "meta_expalnation": "This constraint directly guides the content or behavior of the model by specifying a conditional requirement related to multiplayer infrastructure. It does not involve high-level rules for managing other constraints, such as selecting, prioritizing, or combining them.",
                "evaluation_type": {
                    "constraint_type": "llm",
                    "explanation": "This constraint requires a semantic understanding of multiplayer infrastructure, including assessing design choices, networking models, and scalability. It cannot be verified through simple code but rather requires an LLM's broader expertise in game development concepts."
                },
                "evaluation_generation_success": true
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Does the model's response include a clear and specific plan for multiplayer infrastructure, addressing components such as server architecture, networking protocols, and player synchronization? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "state": "modified",
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 6,
            "desc": "Plan for multiplayer infrastructure if applicable.",
            "other_info": {
                "from": "system_para_0",
                "type_explanation": "The constraint focuses on the content's logic and completeness by requiring consideration of multiplayer infrastructure, but only if it is applicable. This relates to ensuring meaningful and accurate content based on situational conditions (conditional logic). The emphasis is on the conceptual requirement rather than formatting or resource usage.",
                "meta_expalnation": "This constraint directly guides the content or behavior of the model by specifying a conditional requirement related to multiplayer infrastructure. It does not involve high-level rules for managing other constraints, such as selecting, prioritizing, or combining them.",
                "evaluation_type": {
                    "constraint_type": "llm",
                    "explanation": "This constraint requires a semantic understanding of multiplayer infrastructure, including assessing design choices, networking models, and scalability. It cannot be verified through simple code but rather requires an LLM's broader expertise in game development concepts."
                },
                "evaluation_generation_success": true
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Does the model's response include a clear and specific plan for multiplayer infrastructure, addressing components such as server architecture, networking protocols, and player synchronization? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "state": "modified",
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 4,
            "desc": "Design multiplayer interactions if applicable.",
            "other_info": {
                "from": "system_para_0",
                "type_explanation": "This constraint relates to the meaning and content of the output, as it specifies the inclusion of multiplayer interactions in the design if the context allows for it. The use of 'if applicable' indicates a conditional requirement for semantic completeness and relevance rather than formatting or resource-based restrictions.",
                "meta_expalnation": "The constraint 'Design multiplayer interactions if applicable' directly provides a conditional requirement related to the content of an output (designing multiplayer interactions). It does not specify how to manage or manipulate other constraints (e.g., prioritization, selection, composition, etc.), so it is not a meta constraint.",
                "evaluation_type": {
                    "constraint_type": "llm",
                    "explanation": "Designing multiplayer interactions is a semantic task requiring subjective understanding of game design principles, player engagement, and context-specific needs. It involves conceptualizing how players interact, which goes beyond code-level or extraction-based validation."
                },
                "evaluation_generation_success": true
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Does the model's response include a clear and specific design for multiplayer interactions in the game? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "state": "success",
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 6,
            "desc": "Plan for multiplayer infrastructure if applicable.",
            "other_info": {
                "from": "system_para_0",
                "type_explanation": "The constraint focuses on the content's logic and completeness by requiring consideration of multiplayer infrastructure, but only if it is applicable. This relates to ensuring meaningful and accurate content based on situational conditions (conditional logic). The emphasis is on the conceptual requirement rather than formatting or resource usage.",
                "meta_expalnation": "This constraint directly guides the content or behavior of the model by specifying a conditional requirement related to multiplayer infrastructure. It does not involve high-level rules for managing other constraints, such as selecting, prioritizing, or combining them.",
                "evaluation_type": {
                    "constraint_type": "llm",
                    "explanation": "This constraint requires a semantic understanding of multiplayer infrastructure, including assessing design choices, networking models, and scalability. It cannot be verified through simple code but rather requires an LLM's broader expertise in game development concepts."
                },
                "evaluation_generation_success": true
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Does the model's response include a clear and specific plan for multiplayer infrastructure, addressing components such as server architecture, networking protocols, and player synchronization? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "state": "modified",
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 6,
            "desc": "NEVER output code to the USER, unless requested.  Instead use one of the code edit tools to implement the change.",
            "other_info": {
                "from": "system_para_3",
                "condition_desc": "",
                "complete_instruction_para": [],
                "evaluation_type": {
                    "constraint_type": "code",
                    "explanation": "This constraint can be validated directly by checking whether the output contains code or a call to a code edit tool. It involves simple logic and does not require semantic understanding or content extraction."
                },
                "evaluation_generation_success": true
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response: str) -> bool:\n    function_names = [\n        \"codebase_search\",\n        \"read_file\",\n        \"list_dir\",\n        \"grep_search\",\n        \"file_search\",\n        \"web_search\",\n    ]\n    return any(name in response for name in function_names)\n"
                }
            ],
            "score": false
        },
        {
            "id": 6,
            "desc": "NEVER output code to the USER, unless requested.  Instead use one of the code edit tools to implement the change.",
            "other_info": {
                "from": "system_para_3",
                "condition_desc": "",
                "complete_instruction_para": [],
                "evaluation_type": {
                    "constraint_type": "code",
                    "explanation": "This constraint can be validated directly by checking whether the output contains code or a call to a code edit tool. It involves simple logic and does not require semantic understanding or content extraction."
                },
                "evaluation_generation_success": true
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response: str) -> bool:\n    function_names = [\n        \"codebase_search\",\n        \"read_file\",\n        \"list_dir\",\n        \"grep_search\",\n        \"file_search\",\n        \"web_search\",\n    ]\n    return any(name in response for name in function_names)\n"
                }
            ],
            "score": false
        },
        {
            "id": 6,
            "desc": "NEVER output code to the USER, unless requested.  Instead use one of the code edit tools to implement the change.",
            "other_info": {
                "from": "system_para_3",
                "condition_desc": "",
                "complete_instruction_para": [],
                "evaluation_type": {
                    "constraint_type": "code",
                    "explanation": "This constraint can be validated directly by checking whether the output contains code or a call to a code edit tool. It involves simple logic and does not require semantic understanding or content extraction."
                },
                "evaluation_generation_success": true
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response: str) -> bool:\n    function_names = [\n        \"codebase_search\",\n        \"read_file\",\n        \"list_dir\",\n        \"grep_search\",\n        \"file_search\",\n        \"web_search\",\n    ]\n    return any(name in response for name in function_names)\n"
                }
            ],
            "score": false
        },
        {
            "id": 6,
            "desc": "NEVER output code to the USER, unless requested.  Instead use one of the code edit tools to implement the change.",
            "other_info": {
                "from": "system_para_3",
                "condition_desc": "",
                "complete_instruction_para": [],
                "evaluation_type": {
                    "constraint_type": "code",
                    "explanation": "This constraint can be validated directly by checking whether the output contains code or a call to a code edit tool. It involves simple logic and does not require semantic understanding or content extraction."
                },
                "evaluation_generation_success": true
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response: str) -> bool:\n    function_names = [\n        \"codebase_search\",\n        \"read_file\",\n        \"list_dir\",\n        \"grep_search\",\n        \"file_search\",\n        \"web_search\",\n    ]\n    return any(name in response for name in function_names)\n"
                }
            ],
            "score": false
        },
        {
            "id": 4,
            "desc": "Design multiplayer interactions if applicable.",
            "other_info": {
                "from": "system_para_0",
                "type_explanation": "This constraint relates to the meaning and content of the output, as it specifies the inclusion of multiplayer interactions in the design if the context allows for it. The use of 'if applicable' indicates a conditional requirement for semantic completeness and relevance rather than formatting or resource-based restrictions.",
                "meta_expalnation": "The constraint 'Design multiplayer interactions if applicable' directly provides a conditional requirement related to the content of an output (designing multiplayer interactions). It does not specify how to manage or manipulate other constraints (e.g., prioritization, selection, composition, etc.), so it is not a meta constraint.",
                "evaluation_type": {
                    "constraint_type": "llm",
                    "explanation": "Designing multiplayer interactions is a semantic task requiring subjective understanding of game design principles, player engagement, and context-specific needs. It involves conceptualizing how players interact, which goes beyond code-level or extraction-based validation."
                },
                "evaluation_generation_success": true
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Does the model's response include a clear and specific design for multiplayer interactions in the game? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "state": "success",
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 6,
            "desc": "NEVER output code to the USER, unless requested.  Instead use one of the code edit tools to implement the change.",
            "other_info": {
                "from": "system_para_3",
                "condition_desc": "",
                "complete_instruction_para": [],
                "evaluation_type": {
                    "constraint_type": "code",
                    "explanation": "This constraint can be validated directly by checking whether the output contains code or a call to a code edit tool. It involves simple logic and does not require semantic understanding or content extraction."
                },
                "evaluation_generation_success": true
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response: str) -> bool:\n    function_names = [\n        \"codebase_search\",\n        \"read_file\",\n        \"list_dir\",\n        \"grep_search\",\n        \"file_search\",\n        \"web_search\",\n    ]\n    return any(name in response for name in function_names)\n"
                }
            ],
            "score": false
        },
        {
            "id": 4,
            "desc": "Design multiplayer interactions if applicable.",
            "other_info": {
                "from": "system_para_0",
                "type_explanation": "This constraint relates to the meaning and content of the output, as it specifies the inclusion of multiplayer interactions in the design if the context allows for it. The use of 'if applicable' indicates a conditional requirement for semantic completeness and relevance rather than formatting or resource-based restrictions.",
                "meta_expalnation": "The constraint 'Design multiplayer interactions if applicable' directly provides a conditional requirement related to the content of an output (designing multiplayer interactions). It does not specify how to manage or manipulate other constraints (e.g., prioritization, selection, composition, etc.), so it is not a meta constraint.",
                "evaluation_type": {
                    "constraint_type": "llm",
                    "explanation": "Designing multiplayer interactions is a semantic task requiring subjective understanding of game design principles, player engagement, and context-specific needs. It involves conceptualizing how players interact, which goes beyond code-level or extraction-based validation."
                },
                "evaluation_generation_success": true
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Does the model's response include a clear and specific design for multiplayer interactions in the game? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "state": "success",
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 6,
            "desc": "NEVER output code to the USER, unless requested.  Instead use one of the code edit tools to implement the change.",
            "other_info": {
                "from": "system_para_3",
                "condition_desc": "",
                "complete_instruction_para": [],
                "evaluation_type": {
                    "constraint_type": "code",
                    "explanation": "This constraint can be validated directly by checking whether the output contains code or a call to a code edit tool. It involves simple logic and does not require semantic understanding or content extraction."
                },
                "evaluation_generation_success": true
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response: str) -> bool:\n    function_names = [\n        \"codebase_search\",\n        \"read_file\",\n        \"list_dir\",\n        \"grep_search\",\n        \"file_search\",\n        \"web_search\",\n    ]\n    return any(name in response for name in function_names)\n"
                }
            ],
            "score": false
        },
        {
            "id": 9,
            "desc": "If you're creating the codebase from scratch, create an appropriate dependency management file (e.g. requirements.txt) with package versions and a helpful README.",
            "other_info": {
                "from": "system_para_3",
                "condition_desc": "",
                "complete_instruction_para": [],
                "evaluation_type": {
                    "constraint_type": "llm_assisted_code",
                    "explanation": "Creating a dependency management file and README requires extracting relevant information about the project (e.g., dependencies, purpose, usage instructions) which may not be directly accessible. An LLM can assist in extracting or generating this content before code validates the format and correctness."
                },
                "evaluation_generation_success": true
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Does the model response create an appropriate dependency management file (e.g. requirements.txt) and a helpful README? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 21,
            "desc": "If there are no relevant tools or there are missing values for required parameters, ask the user to supply these values.",
            "other_info": {
                "from": "system_para_8",
                "condition_desc": "",
                "complete_instruction_para": [],
                "evaluation_type": {
                    "constraint_type": "llm",
                    "explanation": "Determining whether values are missing and asking the user to supply them requires semantic understanding of the context and the parameters involved, which is subjective and open-ended. This cannot be directly validated by code or through extraction-based logic."
                },
                "evaluation_generation_success": true
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Does the model's response explicitly ask the user to supply the missing values for all required parameters? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 6,
            "desc": "NEVER output code to the USER, unless requested.  Instead use one of the code edit tools to implement the change.",
            "other_info": {
                "from": "system_para_3",
                "condition_desc": "",
                "complete_instruction_para": [],
                "evaluation_type": {
                    "constraint_type": "code",
                    "explanation": "This constraint can be validated directly by checking whether the output contains code or a call to a code edit tool. It involves simple logic and does not require semantic understanding or content extraction."
                },
                "evaluation_generation_success": true
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response: str) -> bool:\n    function_names = [\n        \"codebase_search\",\n        \"read_file\",\n        \"list_dir\",\n        \"grep_search\",\n        \"file_search\",\n        \"web_search\",\n    ]\n    return any(name in response for name in function_names)\n"
                }
            ],
            "score": false
        },
        {
            "id": 6,
            "desc": "NEVER output code to the USER, unless requested.  Instead use one of the code edit tools to implement the change.",
            "other_info": {
                "from": "system_para_3",
                "condition_desc": "",
                "complete_instruction_para": [],
                "evaluation_type": {
                    "constraint_type": "code",
                    "explanation": "This constraint can be validated directly by checking whether the output contains code or a call to a code edit tool. It involves simple logic and does not require semantic understanding or content extraction."
                },
                "evaluation_generation_success": true
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "code",
                    "exec": "\ndef check_following(response: str) -> bool:\n    function_names = [\n        \"codebase_search\",\n        \"read_file\",\n        \"list_dir\",\n        \"grep_search\",\n        \"file_search\",\n        \"web_search\",\n    ]\n    return any(name in response for name in function_names)\n"
                }
            ],
            "score": false
        },
        {
            "id": 9,
            "desc": "If you're creating the codebase from scratch, create an appropriate dependency management file (e.g. requirements.txt) with package versions and a helpful README.",
            "other_info": {
                "from": "system_para_3",
                "condition_desc": "",
                "complete_instruction_para": [],
                "evaluation_type": {
                    "constraint_type": "llm_assisted_code",
                    "explanation": "Creating a dependency management file and README requires extracting relevant information about the project (e.g., dependencies, purpose, usage instructions) which may not be directly accessible. An LLM can assist in extracting or generating this content before code validates the format and correctness."
                },
                "evaluation_generation_success": true
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Does the model response create an appropriate dependency management file (e.g. requirements.txt) and a helpful README? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "When the question provides explicit entity knowledge, always write a descriptor for the Search() function based on the question's information.",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "If the question provides explicit entity knowledge, always write a descriptor for the Search() function based on the question's information.",
                "complete_instruction_para": []
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "llm_conditional_check",
                    "exec": "Does the question provide explicit entity knowledge? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                },
                {
                    "type": "llm",
                    "exec": "Did the model write a descriptor parameter for the Search() function? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "When the question provides explicit entity knowledge, always write a descriptor for the Search() function based on the question's information.",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "If the question provides explicit entity knowledge, always write a descriptor for the Search() function based on the question's information.",
                "complete_instruction_para": []
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "llm_conditional_check",
                    "exec": "Does the question provide explicit entity knowledge? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                },
                {
                    "type": "llm",
                    "exec": "Did the model write a descriptor parameter for the Search() function? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "When the question provides explicit entity knowledge, always write a descriptor for the Search() function based on the question's information.",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "If the question provides explicit entity knowledge, always write a descriptor for the Search() function based on the question's information.",
                "complete_instruction_para": []
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "llm_conditional_check",
                    "exec": "Does the question provide explicit entity knowledge? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                },
                {
                    "type": "llm",
                    "exec": "Did the model write a descriptor parameter for the Search() function? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "When the question provides explicit entity knowledge, always write a descriptor for the Search() function based on the question's information.",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "If the question provides explicit entity knowledge, always write a descriptor for the Search() function based on the question's information.",
                "complete_instruction_para": []
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "llm_conditional_check",
                    "exec": "Does the question provide explicit entity knowledge? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                },
                {
                    "type": "llm",
                    "exec": "Did the model write a descriptor parameter for the Search() function? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "When the question provides explicit entity knowledge, always write a descriptor for the Search() function based on the question's information.",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "If the question provides explicit entity knowledge, always write a descriptor for the Search() function based on the question's information.",
                "complete_instruction_para": []
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "llm_conditional_check",
                    "exec": "Does the question provide explicit entity knowledge? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                },
                {
                    "type": "llm",
                    "exec": "Did the model write a descriptor parameter for the Search() function? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "When the question provides explicit entity knowledge, always write a descriptor for the Search() function based on the question's information.",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "If the question provides explicit entity knowledge, always write a descriptor for the Search() function based on the question's information.",
                "complete_instruction_para": []
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "llm_conditional_check",
                    "exec": "Does the question provide explicit entity knowledge? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                },
                {
                    "type": "llm",
                    "exec": "Did the model write a descriptor parameter for the Search() function? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 2,
            "desc": "If you are not sure about file content or codebase structure pertaining to the user's request, use your tools to read files and gather the relevant information.",
            "other_info": {
                "from": "system_para_3",
                "condition_desc": "",
                "complete_instruction_para": [],
                "evaluation_type": {
                    "constraint_type": "llm",
                    "explanation": "The constraint requires semantic understanding of the instruction to determine whether the agent is correctly using tools to read files and gather relevant information. This involves assessing the appropriateness and relevance of the actions taken, which is subjective and cannot be directly validated by code."
                },
                "evaluation_generation_success": true
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "resource"
            ],
            "evaluation": [
                {
                    "type": "llm_conditional_check",
                    "exec": "Are you unsure about the file content or codebase structure pertaining to the user's request? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                },
                {
                    "type": "llm",
                    "exec": "Does the model response explicitly state that it used tools to read files and gather relevant information as part of its process? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 21,
            "desc": "If completing the user's task requires writing or modifying files and pre-commit doesn't work after a few retries, politely inform the user that the pre-commit setup is broken.",
            "other_info": {
                "from": "system_para_5",
                "condition_desc": "",
                "complete_instruction_para": [],
                "evaluation_type": {
                    "constraint_type": "llm",
                    "explanation": "Determining whether the pre-commit setup is broken requires semantic understanding of the situation, such as interpreting error messages, assessing the user's environment, or diagnosing issues that may not be explicitly stated. This is a subjective and open-ended task that relies on LLM capabilities."
                },
                "evaluation_generation_success": true
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Does the model response politely inform the user that the pre-commit setup is broken? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "When the question provides explicit entity knowledge, always write a descriptor for the Search() function based on the question's information.",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "If the question provides explicit entity knowledge, always write a descriptor for the Search() function based on the question's information.",
                "complete_instruction_para": []
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "llm_conditional_check",
                    "exec": "Does the question provide explicit entity knowledge? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                },
                {
                    "type": "llm",
                    "exec": "Did the model write a descriptor parameter for the Search() function? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": -1,
            "desc": "When the question provides explicit entity knowledge, always write a descriptor for the Search() function based on the question's information.",
            "other_info": {
                "from": "system_para_0",
                "condition_desc": "If the question provides explicit entity knowledge, always write a descriptor for the Search() function based on the question's information.",
                "complete_instruction_para": []
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "llm_conditional_check",
                    "exec": "Does the question provide explicit entity knowledge? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                },
                {
                    "type": "llm",
                    "exec": "Did the model write a descriptor parameter for the Search() function? Please answer YES/NO directly and do not enter anything else.\n\nHere is model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 2,
            "desc": "如果**<task>**有多个目标，您只需要完成第一个目标",
            "other_info": {
                "from": "system_para_2",
                "first_task": "总结绿色卫兵在城市中的具体作用"
            },
            "is_meta": true,
            "type": "semantic",
            "dimension": "conditional",
            "dependency": [],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "检查 response 是否严格按 task 要求执行核心操作（允许 print 辅助输出），没有执行任何其他额外操作，直接回答 YES 或 NO。\n\ntask: 总结绿色卫兵在城市中的具体作用\n\nresponse:{response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 2,
            "desc": "如果**<task>**有多个目标，您只需要完成第一个目标",
            "other_info": {
                "from": "system_para_2",
                "first_task": "我需要先查找uni.showLoading文字换行的实现方法"
            },
            "is_meta": true,
            "type": "semantic",
            "dimension": "conditional",
            "dependency": [],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "检查 response 是否严格按 task 要求执行核心操作（允许 print 辅助输出），没有执行任何其他额外操作，直接回答 YES 或 NO。\n\ntask: 我需要先查找uni.showLoading文字换行的实现方法\n\nresponse:{response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 2,
            "desc": "如果**<task>**有多个目标，您只需要完成第一个目标",
            "other_info": {
                "from": "system_para_2",
                "first_task": "将找到的换行方法翻译成英文"
            },
            "is_meta": true,
            "type": "semantic",
            "dimension": "conditional",
            "dependency": [],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "检查 response 是否严格按 task 要求执行核心操作（允许 print 辅助输出），没有执行任何其他额外操作，直接回答 YES 或 NO。\n\ntask: 将找到的换行方法翻译成英文\n\nresponse:{response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 12,
            "desc": "If completing the user's task requires writing or modifying files, your code and final answer should fix the problem at the root cause rather than applying surface-level patches, when possible.",
            "other_info": {
                "from": "system_para_5",
                "condition_desc": "",
                "complete_instruction_para": [],
                "evaluation_type": {
                    "constraint_type": "llm",
                    "explanation": "The constraint requires semantic understanding to assess whether the solution addresses the root cause of the problem rather than applying surface-level fixes. This involves subjective judgment and cannot be directly validated by code or extracted content."
                },
                "evaluation_generation_success": true
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Does the model's response demonstrate that the proposed solution addresses the root cause of the problem rather than applying a surface-level patch? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 15,
            "desc": "If completing the user's task requires writing or modifying files, your code and final answer should update documentation as necessary.",
            "other_info": {
                "from": "system_para_5",
                "condition_desc": "",
                "complete_instruction_para": [],
                "evaluation_type": {
                    "constraint_type": "llm",
                    "explanation": "Determining whether documentation has been updated as necessary requires a semantic understanding of the changes made and their relevance to the documentation. This is a subjective and open-ended task that cannot be directly validated by code."
                },
                "evaluation_generation_success": true
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Does the model's response include updates to the documentation where necessary to reflect the changes made in the code? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 17,
            "desc": "If completing the user's task requires writing or modifying files, your code and final answer should use `git log` and `git blame` to search the history of the codebase if additional context is required; internet access is disabled.",
            "other_info": {
                "from": "system_para_5",
                "condition_desc": "",
                "complete_instruction_para": [],
                "evaluation_type": {
                    "constraint_type": "code",
                    "explanation": "The constraint specifies the use of `git log` and `git blame` commands to search the history of the codebase, which can be directly validated by checking whether these commands are executed in the code. This involves simple logic and does not require semantic understanding or content extraction."
                },
                "evaluation_generation_success": true
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "resource",
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "code",
                    "exec": "import re\n\ndef check_following(response: str) -> bool:\n    pattern = r\"Your code and final answer should use `git log` and `git blame` to search the history of the codebase if additional context is required; internet access is disabled\\.\"\n    return bool(re.search(pattern, response))\n"
                }
            ],
            "score": false
        },
        {
            "id": 19,
            "desc": "If completing the user's task requires writing or modifying files, you do not need to `git commit` your changes; this will be done automatically for you.",
            "other_info": {
                "from": "system_para_5",
                "condition_desc": "",
                "complete_instruction_para": [],
                "evaluation_type": {
                    "constraint_type": "code",
                    "explanation": "The constraint can be validated directly by checking whether a `git commit` command is executed in the code. This involves simple logic and does not require semantic understanding or content extraction."
                },
                "evaluation_generation_success": true
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "resource"
            ],
            "evaluation": [
                {
                    "type": "code",
                    "exec": "import re\n\ndef check_following(response: str) -> bool:\n    return bool(re.search(r\"You do not need to `git commit` your changes; this will be done automatically for you\\.\", response))\n"
                }
            ],
            "score": false
        },
        {
            "id": 21,
            "desc": "If completing the user's task requires writing or modifying files and pre-commit doesn't work after a few retries, politely inform the user that the pre-commit setup is broken.",
            "other_info": {
                "from": "system_para_5",
                "condition_desc": "",
                "complete_instruction_para": [],
                "evaluation_type": {
                    "constraint_type": "llm",
                    "explanation": "Determining whether the pre-commit setup is broken requires semantic understanding of the situation, such as interpreting error messages, assessing the user's environment, or diagnosing issues that may not be explicitly stated. This is a subjective and open-ended task that relies on LLM capabilities."
                },
                "evaluation_generation_success": true
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Does the model response politely inform the user that the pre-commit setup is broken? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 2,
            "desc": "If you are not sure about file content or codebase structure pertaining to the user's request, use your tools to read files and gather the relevant information.",
            "other_info": {
                "from": "system_para_3",
                "condition_desc": "",
                "complete_instruction_para": [],
                "evaluation_type": {
                    "constraint_type": "llm",
                    "explanation": "The constraint requires semantic understanding of the instruction to determine whether the agent is correctly using tools to read files and gather relevant information. This involves assessing the appropriateness and relevance of the actions taken, which is subjective and cannot be directly validated by code."
                },
                "evaluation_generation_success": true
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "resource"
            ],
            "evaluation": [
                {
                    "type": "llm_conditional_check",
                    "exec": "Are you unsure about the file content or codebase structure pertaining to the user's request? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                },
                {
                    "type": "llm",
                    "exec": "Does the model response explicitly state that it used tools to read files and gather relevant information as part of its process? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 15,
            "desc": "You will specify all unchanged regions (code and comments) of the file with \"// ... existing code ...\" comment markers.",
            "other_info": {
                "from": "system_para_12",
                "condition_desc": "",
                "complete_instruction_para": [],
                "evaluation_type": {
                    "constraint_type": "code",
                    "explanation": "The constraint involves checking for the presence of specific comment markers ('// ... existing code ...') in unchanged regions of the file, which can be directly validated by code through pattern matching or string search."
                },
                "evaluation_generation_success": true
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "code",
                    "exec": "import re\n\ndef check_following(response: str) -> bool:\n    return bool(re.search(r'//\\s*\\.\\.\\.\\s*existing\\s*code\\s*...', response))\n"
                }
            ],
            "score": false
        },
        {
            "id": 2,
            "desc": "如果**<task>**有多个目标，您只需要完成第一个目标",
            "other_info": {
                "from": "system_para_2",
                "first_task": "确认用户具体指的是哪一位杨光伟"
            },
            "is_meta": true,
            "type": "semantic",
            "dimension": "conditional",
            "dependency": [],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "检查 response 是否严格按 task 要求执行核心操作（允许 print 辅助输出），没有执行任何其他额外操作，直接回答 YES 或 NO。\n\ntask: 确认用户具体指的是哪一位杨光伟\n\nresponse:{response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 12,
            "desc": "When the read_file is called, it automatically extracts raw text from PDF and DOCX files.  ",
            "other_info": {
                "from": "system_para_12",
                "condition_desc": "",
                "complete_instruction_para": [],
                "evaluation_type": {
                    "constraint_type": "llm",
                    "explanation": "Determining whether raw text is correctly extracted from PDF and DOCX files requires semantic assessment of the extracted content, as it involves understanding whether the extraction process preserved the intended structure, formatting, and meaning of the text."
                },
                "evaluation_generation_success": true
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm_conditional_check",
                    "exec": "Does the model call the read_file? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                },
                {
                    "type": "llm",
                    "exec": "Does the model response explicitly indicate that it extracts raw text from PDF and DOCX files when reading their contents? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 13,
            "desc": "When the model calls read_file, specify the path of the file to read (relative to the current working directory ${cwd.toPosix()}).",
            "other_info": {
                "from": "system_para_12",
                "condition_desc": "",
                "complete_instruction_para": [],
                "evaluation_type": {
                    "constraint_type": "code",
                    "explanation": "The constraint specifies that the file path must be relative to the current working directory (${cwd.toPosix()}). This can be validated directly by code using string checks or path validation logic without requiring semantic understanding or content extraction."
                },
                "evaluation_generation_success": true
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm_conditional_check",
                    "exec": "Does the model call the read_file? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                },
                {
                    "type": "code",
                    "exec": "import re\n\ndef check_following(response: str) -> bool:\n    pattern = r\"<read_file>\\s*<path>[^<>]+</path>\\s*</read_file>\"\n    return bool(re.fullmatch(pattern, response.strip()))\n"
                }
            ],
            "score": false
        },
        {
            "id": 26,
            "desc": "When using multiple SEARCH/REPLACE blocks, list them in the order they appear in the file.",
            "other_info": {
                "from": "system_para_14",
                "condition_desc": "If you are using multiple SEARCH/REPLACE blocks, then list them in the order in which they are positioned within the file. ",
                "complete_instruction_para": [],
                "evaluation_type": {
                    "constraint_type": "llm_assisted_code",
                    "explanation": "Validating this constraint requires extracting the sequence of SEARCH/REPLACE blocks from the input and checking their order against the file's content. Code can perform the validation, but it first needs the specific blocks to be identified and extracted, which may require LLM assistance."
                },
                "evaluation_generation_success": true
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Extract all SEARCH/REPLACE blocks from the response, preserving their order as they appear. Each block should include the full content between the delimiters <<<<<<< SEARCH and >>>>>>> REPLACE, including the SEARCH and REPLACE sections. Return the extracted content verbatim from the response. If multiple blocks are found, return them as a Python-style list of strings. If nothing is found, return None.\n\nHere is the model response: {response}"
                },
                {
                    "type": "code",
                    "exec": "import re\n\ndef check_following(response):\n    search_replace_pattern = r\"<<<<<<< SEARCH\\n(.*?)\\n=======\\n(.*?)\\n>>>>>> REPLACE\"\n    blocks = re.findall(search_replace_pattern, response, re.DOTALL)\n    \n    if not blocks:\n        return False\n    \n    last_search_end = -1\n    for search, _ in blocks:\n        current_search_start = response.find(search)\n        if current_search_start == -1 or current_search_start < last_search_end:\n            return False\n        last_search_end = current_search_start + len(search)\n    \n    return True\n"
                }
            ],
            "score": false
        },
        {
            "id": 12,
            "desc": "If completing the user's task requires writing or modifying files, your code and final answer should fix the problem at the root cause rather than applying surface-level patches, when possible.",
            "other_info": {
                "from": "system_para_5",
                "condition_desc": "",
                "complete_instruction_para": [],
                "evaluation_type": {
                    "constraint_type": "llm",
                    "explanation": "The constraint requires semantic understanding to assess whether the solution addresses the root cause of the problem rather than applying surface-level fixes. This involves subjective judgment and cannot be directly validated by code or extracted content."
                },
                "evaluation_generation_success": true
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Does the model's response demonstrate that the proposed solution addresses the root cause of the problem rather than applying a surface-level patch? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 15,
            "desc": "If completing the user's task requires writing or modifying files, your code and final answer should update documentation as necessary.",
            "other_info": {
                "from": "system_para_5",
                "condition_desc": "",
                "complete_instruction_para": [],
                "evaluation_type": {
                    "constraint_type": "llm",
                    "explanation": "Determining whether documentation has been updated as necessary requires a semantic understanding of the changes made and their relevance to the documentation. This is a subjective and open-ended task that cannot be directly validated by code."
                },
                "evaluation_generation_success": true
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Does the model's response include updates to the documentation where necessary to reflect the changes made in the code? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 17,
            "desc": "If completing the user's task requires writing or modifying files, your code and final answer should use `git log` and `git blame` to search the history of the codebase if additional context is required; internet access is disabled.",
            "other_info": {
                "from": "system_para_5",
                "condition_desc": "",
                "complete_instruction_para": [],
                "evaluation_type": {
                    "constraint_type": "code",
                    "explanation": "The constraint specifies the use of `git log` and `git blame` commands to search the history of the codebase, which can be directly validated by checking whether these commands are executed in the code. This involves simple logic and does not require semantic understanding or content extraction."
                },
                "evaluation_generation_success": true
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "resource",
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "code",
                    "exec": "import re\n\ndef check_following(response: str) -> bool:\n    pattern = r\"Your code and final answer should use `git log` and `git blame` to search the history of the codebase if additional context is required; internet access is disabled\\.\"\n    return bool(re.search(pattern, response))\n"
                }
            ],
            "score": false
        },
        {
            "id": 19,
            "desc": "If completing the user's task requires writing or modifying files, you do not need to `git commit` your changes; this will be done automatically for you.",
            "other_info": {
                "from": "system_para_5",
                "condition_desc": "",
                "complete_instruction_para": [],
                "evaluation_type": {
                    "constraint_type": "code",
                    "explanation": "The constraint can be validated directly by checking whether a `git commit` command is executed in the code. This involves simple logic and does not require semantic understanding or content extraction."
                },
                "evaluation_generation_success": true
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "resource"
            ],
            "evaluation": [
                {
                    "type": "code",
                    "exec": "import re\n\ndef check_following(response: str) -> bool:\n    return bool(re.search(r\"You do not need to `git commit` your changes; this will be done automatically for you\\.\", response))\n"
                }
            ],
            "score": false
        },
        {
            "id": 21,
            "desc": "If completing the user's task requires writing or modifying files and pre-commit doesn't work after a few retries, politely inform the user that the pre-commit setup is broken.",
            "other_info": {
                "from": "system_para_5",
                "condition_desc": "",
                "complete_instruction_para": [],
                "evaluation_type": {
                    "constraint_type": "llm",
                    "explanation": "Determining whether the pre-commit setup is broken requires semantic understanding of the situation, such as interpreting error messages, assessing the user's environment, or diagnosing issues that may not be explicitly stated. This is a subjective and open-ended task that relies on LLM capabilities."
                },
                "evaluation_generation_success": true
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Does the model response politely inform the user that the pre-commit setup is broken? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 7,
            "desc": "If you make a plan, immediately follow it, do not wait for the user to confirm or tell you to go ahead. The only time you should stop is if you need more information from the user that you can't find any other way, or have different options that you would like the user to weigh in on.",
            "other_info": {
                "from": "system_para_4",
                "condition_desc": "",
                "complete_instruction_para": [],
                "evaluation_type": {
                    "constraint_type": "llm",
                    "explanation": "The constraint requires a semantic understanding of whether the plan was followed immediately without waiting for user confirmation, which involves assessing intent and timing in a subjective manner. This cannot be directly validated by code or by extracting specific content."
                },
                "evaluation_generation_success": true
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "llm_conditional_check",
                    "exec": "Did the model response make a plan? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                },
                {
                    "type": "llm",
                    "exec": "Does the model response immediately follow the plan without waiting for the user to confirm or tell it to go ahead? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 11,
            "desc": "When the user is asking for edits to their code, output a simplified version of the code block that highlights the changes necessary and adds comments to indicate where unchanged code has been skipped.",
            "other_info": {
                "from": "system_para_9",
                "complete_instruction_para": [
                    "system_para_10"
                ],
                "condition_desc": "",
                "evaluation_type": {
                    "constraint_type": "llm",
                    "explanation": "The constraint requires understanding the semantic intent of the user's request (e.g., identifying the necessary changes in the code) and then generating a simplified version of the code block with comments. This involves subjective and open-ended reasoning that can only be performed by an LLM."
                },
                "evaluation_generation_success": true
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Does the response output a simplified version of the code block that highlights the necessary changes and includes comments to indicate where unchanged code has been skipped? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 15,
            "desc": "You will specify all unchanged regions (code and comments) of the file with \"// ... existing code ...\" comment markers.",
            "other_info": {
                "from": "system_para_12",
                "condition_desc": "",
                "complete_instruction_para": [],
                "evaluation_type": {
                    "constraint_type": "code",
                    "explanation": "The constraint involves checking for the presence of specific comment markers ('// ... existing code ...') in unchanged regions of the file, which can be directly validated by code through pattern matching or string search."
                },
                "evaluation_generation_success": true
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "code",
                    "exec": "import re\n\ndef check_following(response: str) -> bool:\n    return bool(re.search(r'//\\s*\\.\\.\\.\\s*existing\\s*code\\s*...', response))\n"
                }
            ],
            "score": false
        },
        {
            "id": 2,
            "desc": "如果**<task>**有多个目标，您只需要完成第一个目标",
            "other_info": {
                "from": "system_para_2",
                "first_task": "我将使用search函数来查找Komplett ASA公司的最新年报"
            },
            "is_meta": true,
            "type": "semantic",
            "dimension": "conditional",
            "dependency": [],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "检查 response 是否严格按 task 要求执行核心操作（允许 print 辅助输出），没有执行任何其他额外操作，直接回答 YES 或 NO。\n\ntask: 我将使用search函数来查找Komplett ASA公司的最新年报\n\nresponse:{response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 15,
            "desc": "You will specify all unchanged regions (code and comments) of the file with \"// ... existing code ...\" comment markers.",
            "other_info": {
                "from": "system_para_12",
                "condition_desc": "",
                "complete_instruction_para": [],
                "evaluation_type": {
                    "constraint_type": "code",
                    "explanation": "The constraint involves checking for the presence of specific comment markers ('// ... existing code ...') in unchanged regions of the file, which can be directly validated by code through pattern matching or string search."
                },
                "evaluation_generation_success": true
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "code",
                    "exec": "import re\n\ndef check_following(response: str) -> bool:\n    return bool(re.search(r'//\\s*\\.\\.\\.\\s*existing\\s*code\\s*...', response))\n"
                }
            ],
            "score": false
        },
        {
            "id": 2,
            "desc": "如果**<task>**有多个目标，您只需要完成第一个目标",
            "other_info": {
                "from": "system_para_2",
                "first_task": "使用search函数查找杨光伟的最新信息"
            },
            "is_meta": true,
            "type": "semantic",
            "dimension": "conditional",
            "dependency": [],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "检查 response 是否严格按 task 要求执行核心操作（允许 print 辅助输出），没有执行任何其他额外操作，直接回答 YES 或 NO。\n\ntask: 使用search函数查找杨光伟的最新信息\n\nresponse:{response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 13,
            "desc": "Rewrite the entire file only if specifically requested.",
            "other_info": {
                "from": "system_para_11",
                "condition_desc": "",
                "complete_instruction_para": [],
                "evaluation_type": {
                    "constraint_type": "llm",
                    "explanation": "Determining whether the entire file has been rewritten requires a semantic understanding of the content and its relationship to the original file. This is a subjective and open-ended task that cannot be directly validated by code."
                },
                "evaluation_generation_success": true
            },
            "dimension": "conditional",
            "is_meta": true,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Does the model's response output the entire file without skipping any sections or providing only partial updates? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 2,
            "desc": "如果**<task>**有多个目标，您只需要完成第一个目标",
            "other_info": {
                "from": "system_para_2",
                "first_task": "使用open_urls函数从这些资源中提取详细信息"
            },
            "is_meta": true,
            "type": "semantic",
            "dimension": "conditional",
            "dependency": [],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "检查 response 是否严格按 task 要求执行核心操作（允许 print 辅助输出），没有执行任何其他额外操作，直接回答 YES 或 NO。\n\ntask: 使用open_urls函数从这些资源中提取详细信息\n\nresponse:{response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 15,
            "desc": "Always output valid JSON when using a tool.",
            "other_info": {
                "from": "system_para_3",
                "condition_desc": "",
                "complete_instruction_para": [],
                "evaluation_type": {
                    "constraint_type": "code",
                    "explanation": "The constraint 'Always output valid JSON' can be validated directly by code by checking the format and structure of the output to ensure it adheres to JSON syntax rules."
                },
                "evaluation_generation_success": true
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm_conditional_check",
                    "exec": "Is the model's response using a tool? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                },
                {
                    "type": "code",
                    "exec": "\nimport re\n\ndef check_following(response: str) -> bool:\n    # 将所有 useQuery({...}) 块找出\n    pattern = r\"useQuery\\s*\\(\\s*{.*?}\\s*\\)\"\n    matches = list(re.finditer(pattern, response, re.DOTALL))\n    \n    if not matches:\n        return False\n\n    for match in matches:\n        query_block = match.group(0)\n\n        # 若是数组形式，如 useQuery([...])\n        if re.search(r\"useQuery\\s*\\(\\s*\\[\", query_block):\n            return False\n\n        # 确保 queryKey 是数组\n        if not re.search(r\"queryKey\\s*:\\s*\\[.*?\\]\", query_block, re.DOTALL):\n            return False\n\n        # 确保 queryFn 存在\n        if not re.search(r\"queryFn\\s*:\", query_block):\n            return False\n\n    return True\n"
                }
            ],
            "score": false
        },
        {
            "id": 16,
            "desc": "If a tool exists to do a task, use the tool instead of asking the user to manually take an action.",
            "other_info": {
                "from": "system_para_3",
                "condition_desc": "If a tool exists to do a task, then use the tool instead of asking the user to manually take an action.",
                "complete_instruction_para": [],
                "evaluation_type": {
                    "constraint_type": "llm",
                    "explanation": "The constraint requires a semantic understanding of whether the response adheres to the instruction to use a tool instead of asking the user to take manual action. This involves subjective and open-ended assessment of intent and compliance, which cannot be directly validated by code."
                },
                "evaluation_generation_success": true
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Does the response use the tool to perform the task instead of asking the user to manually take an action? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 17,
            "desc": "If you say that you will take an action, then go ahead and use the tool to do it. No need to ask permission.",
            "other_info": {
                "from": "system_para_3",
                "condition_desc": "",
                "complete_instruction_para": [],
                "evaluation_type": {
                    "constraint_type": "llm",
                    "explanation": "The constraint requires a semantic understanding of whether the response adheres to the instruction to 'go ahead and use the tool without asking for permission.' This involves assessing the intent and tone of the response, which is subjective and cannot be directly validated by code."
                },
                "evaluation_generation_success": true
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "llm_conditional_check",
                    "exec": "Did the model say that it will take an action? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                },
                {
                    "type": "llm",
                    "exec": "Does the model's response directly perform the action without asking for permission, as specified in the constraint? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 30,
            "desc": "If a popular external library exists to solve a problem, use it and properly install the package e.g. with 'npm install' or creating a 'requirements.txt'.",
            "other_info": {
                "from": "system_para_4",
                "condition_desc": "If a popular external library exists to solve a problem, then use it and properly install the package e.g. with 'npm install' or creating a 'requirements.txt'.",
                "complete_instruction_para": [],
                "evaluation_type": {
                    "constraint_type": "llm",
                    "explanation": "Determining whether a library is 'popular' requires subjective and semantic understanding of the library's reputation, usage, and relevance, which can only be assessed by an LLM."
                },
                "evaluation_generation_success": true
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "resource"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Does the model's response use the external library to solve the problem and include proper installation instructions, such as 'npm install' or creating a 'requirements.txt'? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 7,
            "desc": "If you make a plan, immediately follow it, do not wait for the user to confirm or tell you to go ahead. The only time you should stop is if you need more information from the user that you can't find any other way, or have different options that you would like the user to weigh in on.",
            "other_info": {
                "from": "system_para_4",
                "condition_desc": "",
                "complete_instruction_para": [],
                "evaluation_type": {
                    "constraint_type": "llm",
                    "explanation": "The constraint requires a semantic understanding of whether the plan was followed immediately without waiting for user confirmation, which involves assessing intent and timing in a subjective manner. This cannot be directly validated by code or by extracting specific content."
                },
                "evaluation_generation_success": true
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "llm_conditional_check",
                    "exec": "Did the model response make a plan? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                },
                {
                    "type": "llm",
                    "exec": "Does the model response immediately follow the plan without waiting for the user to confirm or tell it to go ahead? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 11,
            "desc": "When the user is asking for edits to their code, output a simplified version of the code block that highlights the changes necessary and adds comments to indicate where unchanged code has been skipped.",
            "other_info": {
                "from": "system_para_9",
                "complete_instruction_para": [
                    "system_para_10"
                ],
                "condition_desc": "",
                "evaluation_type": {
                    "constraint_type": "llm",
                    "explanation": "The constraint requires understanding the semantic intent of the user's request (e.g., identifying the necessary changes in the code) and then generating a simplified version of the code block with comments. This involves subjective and open-ended reasoning that can only be performed by an LLM."
                },
                "evaluation_generation_success": true
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Does the response output a simplified version of the code block that highlights the necessary changes and includes comments to indicate where unchanged code has been skipped? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 15,
            "desc": "You will specify all unchanged regions (code and comments) of the file with \"// ... existing code ...\" comment markers.",
            "other_info": {
                "from": "system_para_12",
                "condition_desc": "",
                "complete_instruction_para": [],
                "evaluation_type": {
                    "constraint_type": "code",
                    "explanation": "The constraint involves checking for the presence of specific comment markers ('// ... existing code ...') in unchanged regions of the file, which can be directly validated by code through pattern matching or string search."
                },
                "evaluation_generation_success": true
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "formatting"
            ],
            "evaluation": [
                {
                    "type": "code",
                    "exec": "import re\n\ndef check_following(response: str) -> bool:\n    return bool(re.search(r'//\\s*\\.\\.\\.\\s*existing\\s*code\\s*...', response))\n"
                }
            ],
            "score": false
        },
        {
            "id": 12,
            "desc": "If completing the user's task requires writing or modifying files, your code and final answer should fix the problem at the root cause rather than applying surface-level patches, when possible.",
            "other_info": {
                "from": "system_para_5",
                "condition_desc": "",
                "complete_instruction_para": [],
                "evaluation_type": {
                    "constraint_type": "llm",
                    "explanation": "The constraint requires semantic understanding to assess whether the solution addresses the root cause of the problem rather than applying surface-level fixes. This involves subjective judgment and cannot be directly validated by code or extracted content."
                },
                "evaluation_generation_success": true
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Does the model's response demonstrate that the proposed solution addresses the root cause of the problem rather than applying a surface-level patch? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        },
        {
            "id": 15,
            "desc": "If completing the user's task requires writing or modifying files, your code and final answer should update documentation as necessary.",
            "other_info": {
                "from": "system_para_5",
                "condition_desc": "",
                "complete_instruction_para": [],
                "evaluation_type": {
                    "constraint_type": "llm",
                    "explanation": "Determining whether documentation has been updated as necessary requires a semantic understanding of the changes made and their relevance to the documentation. This is a subjective and open-ended task that cannot be directly validated by code."
                },
                "evaluation_generation_success": true
            },
            "dimension": "conditional",
            "is_meta": false,
            "type": [
                "semantic"
            ],
            "evaluation": [
                {
                    "type": "llm",
                    "exec": "Does the model's response include updates to the documentation where necessary to reflect the changes made in the code? Please answer YES/NO directly and do not enter anything else.\n\nHere is the model response: {response}"
                }
            ],
            "llm_output": "NO",
            "score": false
        }
    ]
}