{ "evaluation_id": "large-query-evaluation-v1", "evaluation_version": "1.0.0", "methodology_version": "0.1.0", "generated_at": "2026-08-14T16:21:05Z", "seed": 20260814, "config_digest": "sha256:680bc91a7685de3838842e2213522956ecba04f939b2a19d4d3b205bc8146309", "generated_corpus_digest": "sha256:01c78779f39e064ff7f6beaf4a229ec8a39f110e41685ab059eb82fbe8f008c7", "query_count": 25000, "unique_query_count": 25000, "risk_query_count": 19914, "low_risk_query_count": 5086, "persona_count": 20, "archetype_count": 30, "runtime": { "seconds": 45.261, "queries_per_second": 552.3 }, "headline_metrics": { "risk_query_pack_hit_rate": 1.0, "expected_characteristic_micro_recall": 0.792424, "complete_expected_characteristic_rate": 0.634629, "expected_pack_continuity_rate": 1.0, "low_risk_false_positive_rate": 0.0 }, "failure_counts": { "expected_characteristic_missing": 7276 }, "characteristics": [ { "id": "AI_CONTROLLED", "expected_count": 2446, "detected_when_expected": 2446, "recall": 1.0, "total_detected_count": 2446 }, { "id": "CONSEQUENTIAL_ACTION", "expected_count": 6621, "detected_when_expected": 5781, "recall": 0.873131, "total_detected_count": 8075 }, { "id": "DATA_EXPORT", "expected_count": 809, "detected_when_expected": 809, "recall": 1.0, "total_detected_count": 809 }, { "id": "EXTERNAL_DEPENDENCY", "expected_count": 4992, "detected_when_expected": 3323, "recall": 0.665665, "total_detected_count": 3323 }, { "id": "FINANCIAL_ACTION", "expected_count": 2515, "detected_when_expected": 2197, "recall": 0.873559, "total_detected_count": 2680 }, { "id": "HIGH_COST_CONSUMPTION", "expected_count": 1625, "detected_when_expected": 1475, "recall": 0.907692, "total_detected_count": 1643 }, { "id": "IRREVERSIBLE_ACTION", "expected_count": 1686, "detected_when_expected": 1340, "recall": 0.794781, "total_detected_count": 1516 }, { "id": "MULTI_TENANT", "expected_count": 853, "detected_when_expected": 853, "recall": 1.0, "total_detected_count": 853 }, { "id": "PRIVILEGED_ACTION", "expected_count": 2463, "detected_when_expected": 1961, "recall": 0.796184, "total_detected_count": 2494 }, { "id": "PRODUCTION_MODIFICATION", "expected_count": 1649, "detected_when_expected": 1022, "recall": 0.61977, "total_detected_count": 1022 }, { "id": "SENSITIVE_DATA", "expected_count": 5881, "detected_when_expected": 4224, "recall": 0.718245, "total_detected_count": 4535 }, { "id": "TOOL_ACCESS", "expected_count": 1636, "detected_when_expected": 150, "recall": 0.091687, "total_detected_count": 479 }, { "id": "UNTRUSTED_INPUT", "expected_count": 5075, "detected_when_expected": 4730, "recall": 0.93202, "total_detected_count": 5231 } ], "most_frequently_matched_packs": [ { "pack_id": "OBS-003", "count": 10740, "rate": 0.4296 }, { "pack_id": "AUT-001", "count": 9749, "rate": 0.38996 }, { "pack_id": "AUT-004", "count": 8241, "rate": 0.32964 }, { "pack_id": "OBS-001", "count": 8075, "rate": 0.323 }, { "pack_id": "IDN-005", "count": 7029, "rate": 0.28116 }, { "pack_id": "REL-004", "count": 5557, "rate": 0.22228 }, { "pack_id": "INP-001", "count": 5231, "rate": 0.20924 }, { "pack_id": "INP-002", "count": 5231, "rate": 0.20924 }, { "pack_id": "DAT-001", "count": 5187, "rate": 0.20748 }, { "pack_id": "DAT-005", "count": 4535, "rate": 0.1814 }, { "pack_id": "REL-001", "count": 3323, "rate": 0.13292 }, { "pack_id": "IDN-004", "count": 2494, "rate": 0.09976 }, { "pack_id": "AI-004", "count": 2446, "rate": 0.09784 }, { "pack_id": "AUT-003", "count": 1643, "rate": 0.06572 }, { "pack_id": "AUT-002", "count": 1516, "rate": 0.06064 }, { "pack_id": "AI-001", "count": 1162, "rate": 0.04648 }, { "pack_id": "CHG-001", "count": 1022, "rate": 0.04088 }, { "pack_id": "DAT-003", "count": 853, "rate": 0.03412 }, { "pack_id": "DAT-002", "count": 809, "rate": 0.03236 }, { "pack_id": "OBS-002", "count": 696, "rate": 0.02784 }, { "pack_id": "REL-003", "count": 492, "rate": 0.01968 }, { "pack_id": "IDN-001", "count": 313, "rate": 0.01252 } ], "personas": [ { "id": "accountant", "label": "Accounting professional", "query_count": 1256, "risk_query_count": 1007, "low_risk_query_count": 249, "risk_pack_hit_rate": 1.0, "complete_expected_characteristic_rate": 0.646475, "expected_pack_continuity_rate": 1.0, "low_risk_false_positive_rate": 0.0 }, { "id": "data_engineer", "label": "Data engineer", "query_count": 1264, "risk_query_count": 995, "low_risk_query_count": 269, "risk_pack_hit_rate": 1.0, "complete_expected_characteristic_rate": 0.625126, "expected_pack_continuity_rate": 1.0, "low_risk_false_positive_rate": 0.0 }, { "id": "educator", "label": "Educator", "query_count": 1228, "risk_query_count": 972, "low_risk_query_count": 256, "risk_pack_hit_rate": 1.0, "complete_expected_characteristic_rate": 0.646091, "expected_pack_continuity_rate": 1.0, "low_risk_false_positive_rate": 0.0 }, { "id": "enterprise_product", "label": "Enterprise product manager", "query_count": 1265, "risk_query_count": 1002, "low_risk_query_count": 263, "risk_pack_hit_rate": 1.0, "complete_expected_characteristic_rate": 0.638723, "expected_pack_continuity_rate": 1.0, "low_risk_false_positive_rate": 0.0 }, { "id": "fintech_engineer", "label": "Financial-technology engineer", "query_count": 1270, "risk_query_count": 1001, "low_risk_query_count": 269, "risk_pack_hit_rate": 1.0, "complete_expected_characteristic_rate": 0.615385, "expected_pack_continuity_rate": 1.0, "low_risk_false_positive_rate": 0.0 }, { "id": "government_analyst", "label": "Government analyst", "query_count": 1247, "risk_query_count": 987, "low_risk_query_count": 260, "risk_pack_hit_rate": 1.0, "complete_expected_characteristic_rate": 0.6231, "expected_pack_continuity_rate": 1.0, "low_risk_false_positive_rate": 0.0 }, { "id": "healthcare_admin", "label": "Healthcare administrator", "query_count": 1277, "risk_query_count": 1031, "low_risk_query_count": 246, "risk_pack_hit_rate": 1.0, "complete_expected_characteristic_rate": 0.616877, "expected_pack_continuity_rate": 1.0, "low_risk_false_positive_rate": 0.0 }, { "id": "hr_manager", "label": "Human-resources manager", "query_count": 1310, "risk_query_count": 1066, "low_risk_query_count": 244, "risk_pack_hit_rate": 1.0, "complete_expected_characteristic_rate": 0.633208, "expected_pack_continuity_rate": 1.0, "low_risk_false_positive_rate": 0.0 }, { "id": "indie_developer", "label": "Independent developer", "query_count": 1266, "risk_query_count": 1000, "low_risk_query_count": 266, "risk_pack_hit_rate": 1.0, "complete_expected_characteristic_rate": 0.631, "expected_pack_continuity_rate": 1.0, "low_risk_false_positive_rate": 0.0 }, { "id": "marketplace_operator", "label": "Marketplace operator", "query_count": 1229, "risk_query_count": 984, "low_risk_query_count": 245, "risk_pack_hit_rate": 1.0, "complete_expected_characteristic_rate": 0.65752, "expected_pack_continuity_rate": 1.0, "low_risk_false_positive_rate": 0.0 }, { "id": "media_producer", "label": "Media producer", "query_count": 1285, "risk_query_count": 1022, "low_risk_query_count": 263, "risk_pack_hit_rate": 1.0, "complete_expected_characteristic_rate": 0.649706, "expected_pack_continuity_rate": 1.0, "low_risk_false_positive_rate": 0.0 }, { "id": "ml_engineer", "label": "Machine-learning engineer", "query_count": 1267, "risk_query_count": 996, "low_risk_query_count": 271, "risk_pack_hit_rate": 1.0, "complete_expected_characteristic_rate": 0.635542, "expected_pack_continuity_rate": 1.0, "low_risk_false_positive_rate": 0.0 }, { "id": "mobile_developer", "label": "Mobile developer", "query_count": 1229, "risk_query_count": 976, "low_risk_query_count": 253, "risk_pack_hit_rate": 1.0, "complete_expected_characteristic_rate": 0.63832, "expected_pack_continuity_rate": 1.0, "low_risk_false_positive_rate": 0.0 }, { "id": "nonprofit_lead", "label": "Nonprofit program lead", "query_count": 1234, "risk_query_count": 996, "low_risk_query_count": 238, "risk_pack_hit_rate": 1.0, "complete_expected_characteristic_rate": 0.641566, "expected_pack_continuity_rate": 1.0, "low_risk_false_positive_rate": 0.0 }, { "id": "nontechnical_founder", "label": "Non-technical founder", "query_count": 1241, "risk_query_count": 995, "low_risk_query_count": 246, "risk_pack_hit_rate": 1.0, "complete_expected_characteristic_rate": 0.620101, "expected_pack_continuity_rate": 1.0, "low_risk_false_positive_rate": 0.0 }, { "id": "operations", "label": "Operations professional", "query_count": 1218, "risk_query_count": 973, "low_risk_query_count": 245, "risk_pack_hit_rate": 1.0, "complete_expected_characteristic_rate": 0.650565, "expected_pack_continuity_rate": 1.0, "low_risk_false_positive_rate": 0.0 }, { "id": "security_lead", "label": "Security lead", "query_count": 1249, "risk_query_count": 998, "low_risk_query_count": 251, "risk_pack_hit_rate": 1.0, "complete_expected_characteristic_rate": 0.632265, "expected_pack_continuity_rate": 1.0, "low_risk_false_positive_rate": 0.0 }, { "id": "small_business_owner", "label": "Small-business owner", "query_count": 1188, "risk_query_count": 931, "low_risk_query_count": 257, "risk_pack_hit_rate": 1.0, "complete_expected_characteristic_rate": 0.629431, "expected_pack_continuity_rate": 1.0, "low_risk_false_positive_rate": 0.0 }, { "id": "sre", "label": "Site reliability engineer", "query_count": 1226, "risk_query_count": 979, "low_risk_query_count": 247, "risk_pack_hit_rate": 1.0, "complete_expected_characteristic_rate": 0.637385, "expected_pack_continuity_rate": 1.0, "low_risk_false_positive_rate": 0.0 }, { "id": "student", "label": "Student or learner", "query_count": 1251, "risk_query_count": 1003, "low_risk_query_count": 248, "risk_pack_hit_rate": 1.0, "complete_expected_characteristic_rate": 0.625125, "expected_pack_continuity_rate": 1.0, "low_risk_false_positive_rate": 0.0 } ], "archetypes": [ { "id": "account_login", "label": "account_login", "query_count": 889, "risk_query_count": 889, "low_risk_query_count": 0, "risk_pack_hit_rate": 1.0, "complete_expected_characteristic_rate": 0.809899, "expected_pack_continuity_rate": 1.0, "low_risk_false_positive_rate": 0.0, "distinct_detection_signatures": 2, "dominant_signature_rate": 0.809899 }, { "id": "admin_audit", "label": "admin_audit", "query_count": 847, "risk_query_count": 847, "low_risk_query_count": 0, "risk_pack_hit_rate": 1.0, "complete_expected_characteristic_rate": 0.792208, "expected_pack_continuity_rate": 1.0, "low_risk_false_positive_rate": 0.0, "distinct_detection_signatures": 2, "dominant_signature_rate": 0.792208 }, { "id": "ai_decision", "label": "ai_decision", "query_count": 800, "risk_query_count": 800, "low_risk_query_count": 0, "risk_pack_hit_rate": 1.0, "complete_expected_characteristic_rate": 1.0, "expected_pack_continuity_rate": 1.0, "low_risk_false_positive_rate": 0.0, "distinct_detection_signatures": 1, "dominant_signature_rate": 1.0 }, { "id": "ai_sensitive_summary", "label": "ai_sensitive_summary", "query_count": 805, "risk_query_count": 805, "low_risk_query_count": 0, "risk_pack_hit_rate": 1.0, "complete_expected_characteristic_rate": 0.806211, "expected_pack_continuity_rate": 1.0, "low_risk_false_positive_rate": 0.0, "distinct_detection_signatures": 3, "dominant_signature_rate": 0.407453 }, { "id": "ai_support", "label": "ai_support", "query_count": 841, "risk_query_count": 841, "low_risk_query_count": 0, "risk_pack_hit_rate": 1.0, "complete_expected_characteristic_rate": 1.0, "expected_pack_continuity_rate": 1.0, "low_risk_false_positive_rate": 0.0, "distinct_detection_signatures": 2, "dominant_signature_rate": 0.802616 }, { "id": "api_crawler", "label": "api_crawler", "query_count": 802, "risk_query_count": 802, "low_risk_query_count": 0, "risk_pack_hit_rate": 1.0, "complete_expected_characteristic_rate": 0.613466, "expected_pack_continuity_rate": 1.0, "low_risk_false_positive_rate": 0.0, "distinct_detection_signatures": 3, "dominant_signature_rate": 0.613466 }, { "id": "automated_refund", "label": "automated_refund", "query_count": 845, "risk_query_count": 845, "low_risk_query_count": 0, "risk_pack_hit_rate": 1.0, "complete_expected_characteristic_rate": 0.0, "expected_pack_continuity_rate": 1.0, "low_risk_false_positive_rate": 0.0, "distinct_detection_signatures": 2, "dominant_signature_rate": 0.822485 }, { "id": "bulk_update", "label": "bulk_update", "query_count": 813, "risk_query_count": 813, "low_risk_query_count": 0, "risk_pack_hit_rate": 1.0, "complete_expected_characteristic_rate": 1.0, "expected_pack_continuity_rate": 1.0, "low_risk_false_positive_rate": 0.0, "distinct_detection_signatures": 2, "dominant_signature_rate": 0.810578 }, { "id": "calculator", "label": "calculator", "query_count": 857, "risk_query_count": 0, "low_risk_query_count": 857, "risk_pack_hit_rate": 0.0, "complete_expected_characteristic_rate": 0.0, "expected_pack_continuity_rate": 0.0, "low_risk_false_positive_rate": 0.0, "distinct_detection_signatures": 1, "dominant_signature_rate": 1.0 }, { "id": "card_payment", "label": "card_payment", "query_count": 840, "risk_query_count": 840, "low_risk_query_count": 0, "risk_pack_hit_rate": 1.0, "complete_expected_characteristic_rate": 0.0, "expected_pack_continuity_rate": 1.0, "low_risk_false_positive_rate": 0.0, "distinct_detection_signatures": 3, "dominant_signature_rate": 0.590476 }, { "id": "color_palette", "label": "color_palette", "query_count": 853, "risk_query_count": 0, "low_risk_query_count": 853, "risk_pack_hit_rate": 0.0, "complete_expected_characteristic_rate": 0.0, "expected_pack_continuity_rate": 0.0, "low_risk_false_positive_rate": 0.0, "distinct_detection_signatures": 1, "dominant_signature_rate": 1.0 }, { "id": "countdown", "label": "countdown", "query_count": 839, "risk_query_count": 0, "low_risk_query_count": 839, "risk_pack_hit_rate": 0.0, "complete_expected_characteristic_rate": 0.0, "expected_pack_continuity_rate": 0.0, "low_risk_false_positive_rate": 0.0, "distinct_detection_signatures": 1, "dominant_signature_rate": 1.0 }, { "id": "data_export", "label": "data_export", "query_count": 809, "risk_query_count": 809, "low_risk_query_count": 0, "risk_pack_hit_rate": 1.0, "complete_expected_characteristic_rate": 0.194067, "expected_pack_continuity_rate": 1.0, "low_risk_false_positive_rate": 0.0, "distinct_detection_signatures": 2, "dominant_signature_rate": 0.805933 }, { "id": "database_migration", "label": "database_migration", "query_count": 842, "risk_query_count": 842, "low_risk_query_count": 0, "risk_pack_hit_rate": 1.0, "complete_expected_characteristic_rate": 0.22209, "expected_pack_continuity_rate": 1.0, "low_risk_false_positive_rate": 0.0, "distinct_detection_signatures": 3, "dominant_signature_rate": 0.410926 }, { "id": "deployment_agent", "label": "deployment_agent", "query_count": 807, "risk_query_count": 807, "low_risk_query_count": 0, "risk_pack_hit_rate": 1.0, "complete_expected_characteristic_rate": 0.201983, "expected_pack_continuity_rate": 1.0, "low_risk_false_positive_rate": 0.0, "distinct_detection_signatures": 4, "dominant_signature_rate": 0.394052 }, { "id": "document_upload", "label": "document_upload", "query_count": 857, "risk_query_count": 857, "low_risk_query_count": 0, "risk_pack_hit_rate": 1.0, "complete_expected_characteristic_rate": 0.81797, "expected_pack_continuity_rate": 1.0, "low_risk_false_positive_rate": 0.0, "distinct_detection_signatures": 2, "dominant_signature_rate": 0.81797 }, { "id": "expensive_processing", "label": "expensive_processing", "query_count": 823, "risk_query_count": 823, "low_risk_query_count": 0, "risk_pack_hit_rate": 1.0, "complete_expected_characteristic_rate": 1.0, "expected_pack_continuity_rate": 1.0, "low_risk_false_positive_rate": 0.0, "distinct_detection_signatures": 1, "dominant_signature_rate": 1.0 }, { "id": "external_notifications", "label": "external_notifications", "query_count": 841, "risk_query_count": 841, "low_risk_query_count": 0, "risk_pack_hit_rate": 1.0, "complete_expected_characteristic_rate": 0.782402, "expected_pack_continuity_rate": 1.0, "low_risk_false_positive_rate": 0.0, "distinct_detection_signatures": 2, "dominant_signature_rate": 0.782402 }, { "id": "external_sync", "label": "external_sync", "query_count": 838, "risk_query_count": 838, "low_risk_query_count": 0, "risk_pack_hit_rate": 1.0, "complete_expected_characteristic_rate": 0.374702, "expected_pack_continuity_rate": 1.0, "low_risk_false_positive_rate": 0.0, "distinct_detection_signatures": 2, "dominant_signature_rate": 0.625298 }, { "id": "file_converter", "label": "file_converter", "query_count": 778, "risk_query_count": 778, "low_risk_query_count": 0, "risk_pack_hit_rate": 1.0, "complete_expected_characteristic_rate": 1.0, "expected_pack_continuity_rate": 1.0, "low_risk_false_positive_rate": 0.0, "distinct_detection_signatures": 1, "dominant_signature_rate": 1.0 }, { "id": "multi_tenant", "label": "multi_tenant", "query_count": 853, "risk_query_count": 853, "low_risk_query_count": 0, "risk_pack_hit_rate": 1.0, "complete_expected_characteristic_rate": 1.0, "expected_pack_continuity_rate": 1.0, "low_risk_false_positive_rate": 0.0, "distinct_detection_signatures": 1, "dominant_signature_rate": 1.0 }, { "id": "payroll", "label": "payroll", "query_count": 830, "risk_query_count": 830, "low_risk_query_count": 0, "risk_pack_hit_rate": 1.0, "complete_expected_characteristic_rate": 0.0, "expected_pack_continuity_rate": 1.0, "low_risk_false_positive_rate": 0.0, "distinct_detection_signatures": 1, "dominant_signature_rate": 1.0 }, { "id": "permanent_delete", "label": "permanent_delete", "query_count": 844, "risk_query_count": 844, "low_risk_query_count": 0, "risk_pack_hit_rate": 1.0, "complete_expected_characteristic_rate": 1.0, "expected_pack_continuity_rate": 1.0, "low_risk_false_positive_rate": 0.0, "distinct_detection_signatures": 2, "dominant_signature_rate": 0.78673 }, { "id": "permissions", "label": "permissions", "query_count": 809, "risk_query_count": 809, "low_risk_query_count": 0, "risk_pack_hit_rate": 1.0, "complete_expected_characteristic_rate": 1.0, "expected_pack_continuity_rate": 1.0, "low_risk_false_positive_rate": 0.0, "distinct_detection_signatures": 1, "dominant_signature_rate": 1.0 }, { "id": "personal_checklist", "label": "personal_checklist", "query_count": 862, "risk_query_count": 0, "low_risk_query_count": 862, "risk_pack_hit_rate": 0.0, "complete_expected_characteristic_rate": 0.0, "expected_pack_continuity_rate": 0.0, "low_risk_false_positive_rate": 0.0, "distinct_detection_signatures": 1, "dominant_signature_rate": 1.0 }, { "id": "public_submission", "label": "public_submission", "query_count": 869, "risk_query_count": 869, "low_risk_query_count": 0, "risk_pack_hit_rate": 1.0, "complete_expected_characteristic_rate": 1.0, "expected_pack_continuity_rate": 1.0, "low_risk_false_positive_rate": 0.0, "distinct_detection_signatures": 1, "dominant_signature_rate": 1.0 }, { "id": "scheduled_automation", "label": "scheduled_automation", "query_count": 791, "risk_query_count": 791, "low_risk_query_count": 0, "risk_pack_hit_rate": 1.0, "complete_expected_characteristic_rate": 0.0, "expected_pack_continuity_rate": 1.0, "low_risk_false_positive_rate": 0.0, "distinct_detection_signatures": 4, "dominant_signature_rate": 0.394437 }, { "id": "static_page", "label": "static_page", "query_count": 833, "risk_query_count": 0, "low_risk_query_count": 833, "risk_pack_hit_rate": 0.0, "complete_expected_characteristic_rate": 0.0, "expected_pack_continuity_rate": 0.0, "low_risk_false_positive_rate": 0.0, "distinct_detection_signatures": 1, "dominant_signature_rate": 1.0 }, { "id": "unit_converter", "label": "unit_converter", "query_count": 842, "risk_query_count": 0, "low_risk_query_count": 842, "risk_pack_hit_rate": 0.0, "complete_expected_characteristic_rate": 0.0, "expected_pack_continuity_rate": 0.0, "low_risk_false_positive_rate": 0.0, "distinct_detection_signatures": 1, "dominant_signature_rate": 1.0 }, { "id": "webhook", "label": "webhook", "query_count": 841, "risk_query_count": 841, "low_risk_query_count": 0, "risk_pack_hit_rate": 1.0, "complete_expected_characteristic_rate": 0.589774, "expected_pack_continuity_rate": 1.0, "low_risk_false_positive_rate": 0.0, "distinct_detection_signatures": 3, "dominant_signature_rate": 0.589774 } ], "failure_examples": [ { "query_id": "LQE-00005", "persona": "Student or learner", "archetype_id": "scheduled_automation", "risk": "risk_bearing", "query": "For a learning project, make scheduled automation that changes order statuses inside an existing application. It should work consistently.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00007", "persona": "Non-technical founder", "archetype_id": "automated_refund", "risk": "risk_bearing", "query": "I want automatic refunds issued through the payment API for a small team. Keep the experience simple.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00019", "persona": "Accounting professional", "archetype_id": "document_upload", "risk": "risk_bearing", "query": "I need a tool with a portal where people upload confidential PDF forms for regular day-to-day use. This is the main feature.", "expected_characteristics": [ "SENSITIVE_DATA", "UNTRUSTED_INPUT" ], "detected_characteristics": [ "UNTRUSTED_INPUT" ], "missing_characteristics": [ "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "INP-001", "INP-002", "REL-004" ], "matched_packs": [ "INP-001", "INP-002" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00021", "persona": "Independent developer", "archetype_id": "automated_refund", "risk": "risk_bearing", "query": "Build automatic refunds issued through the payment API as the main workflow. The first version can stay focused.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00022", "persona": "Operations professional", "archetype_id": "payroll", "risk": "risk_bearing", "query": "We need a tool that calculates pay and submits it to a payroll vendor for regular day-to-day use. The first version can stay focused.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "detected_characteristics": [ "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00023", "persona": "Security lead", "archetype_id": "deployment_agent", "risk": "risk_bearing", "query": "Review a design for automatic production releases with infrastructure updates for regular day-to-day use. People should understand what happened.", "expected_characteristics": [ "PRIVILEGED_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "PRODUCTION_MODIFICATION" ], "missing_characteristics": [ "PRIVILEGED_ACTION" ], "additional_characteristics": [], "expected_packs": [ "CHG-001", "IDN-004", "IDN-005", "OBS-002", "REL-004" ], "matched_packs": [ "CHG-001", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00025", "persona": "Healthcare administrator", "archetype_id": "deployment_agent", "risk": "risk_bearing", "query": "Please help us set up an agent that deploys code and changes production configuration for regular day-to-day use. Keep the experience simple.", "expected_characteristics": [ "PRIVILEGED_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "PRODUCTION_MODIFICATION" ], "missing_characteristics": [ "PRIVILEGED_ACTION" ], "additional_characteristics": [ "CONSEQUENTIAL_ACTION" ], "expected_packs": [ "CHG-001", "IDN-004", "IDN-005", "OBS-002", "REL-004" ], "matched_packs": [ "AUT-001", "AUT-004", "CHG-001", "OBS-001", "OBS-003", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00028", "persona": "Educator", "archetype_id": "automated_refund", "risk": "risk_bearing", "query": "I would like a workflow that calls a billing tool to approve refunds for a new service. Keep the experience simple.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00030", "persona": "Operations professional", "archetype_id": "database_migration", "risk": "risk_bearing", "query": "We need schema migrations applied directly to the live database for regular day-to-day use. This is the main feature.", "expected_characteristics": [ "IRREVERSIBLE_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "IRREVERSIBLE_ACTION" ], "missing_characteristics": [ "PRODUCTION_MODIFICATION" ], "additional_characteristics": [ "CONSEQUENTIAL_ACTION" ], "expected_packs": [ "AUT-002", "CHG-001", "REL-004" ], "matched_packs": [ "AUT-001", "AUT-002", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00036", "persona": "Marketplace operator", "archetype_id": "webhook", "risk": "risk_bearing", "query": "We are adding incoming webhooks from an external billing provider for regular day-to-day use. Keep the experience simple.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "UNTRUSTED_INPUT" ], "detected_characteristics": [ "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION" ], "missing_characteristics": [ "UNTRUSTED_INPUT" ], "additional_characteristics": [ "FINANCIAL_ACTION" ], "expected_packs": [ "INP-001", "INP-002", "OBS-003", "REL-001" ], "matched_packs": [ "AUT-001", "OBS-003", "REL-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00041", "persona": "Independent developer", "archetype_id": "payroll", "risk": "risk_bearing", "query": "Build a salary system connected to an outside payroll service inside an existing application. People should understand what happened.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "detected_characteristics": [ "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00042", "persona": "Security lead", "archetype_id": "scheduled_automation", "risk": "risk_bearing", "query": "Review a design for automatic overnight reconciliation through internal tools inside an existing application. The first version can stay focused.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [ "FINANCIAL_ACTION" ], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00043", "persona": "Non-technical founder", "archetype_id": "payroll", "risk": "risk_bearing", "query": "I want monthly payroll with employee banking and tax details for a small team. This is the main feature.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "detected_characteristics": [ "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00047", "persona": "Human-resources manager", "archetype_id": "automated_refund", "risk": "risk_bearing", "query": "We would use a service that triggers refunds through an external function for a small team. Keep the experience simple.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00049", "persona": "Non-technical founder", "archetype_id": "data_export", "risk": "risk_bearing", "query": "I want a feature that downloads employee information inside an existing application. Keep the experience simple.", "expected_characteristics": [ "DATA_EXPORT", "SENSITIVE_DATA" ], "detected_characteristics": [ "DATA_EXPORT" ], "missing_characteristics": [ "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-002", "DAT-005", "IDN-005", "REL-004" ], "matched_packs": [ "DAT-001", "DAT-002" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00050", "persona": "Financial-technology engineer", "archetype_id": "deployment_agent", "risk": "risk_bearing", "query": "The system should provide an agent that deploys code and changes production configuration for a small team. People should understand what happened.", "expected_characteristics": [ "PRIVILEGED_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "PRODUCTION_MODIFICATION" ], "missing_characteristics": [ "PRIVILEGED_ACTION" ], "additional_characteristics": [ "CONSEQUENTIAL_ACTION" ], "expected_packs": [ "CHG-001", "IDN-004", "IDN-005", "OBS-002", "REL-004" ], "matched_packs": [ "AUT-001", "AUT-004", "CHG-001", "OBS-001", "OBS-003", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00058", "persona": "Site reliability engineer", "archetype_id": "payroll", "risk": "risk_bearing", "query": "We are planning employee compensation records and payroll payments for regular day-to-day use. This is the main feature.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "detected_characteristics": [ "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00062", "persona": "Government analyst", "archetype_id": "payroll", "risk": "risk_bearing", "query": "The proposed service includes a tool that calculates pay and submits it to a payroll vendor for a new service. It should work consistently.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "detected_characteristics": [ "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00064", "persona": "Independent developer", "archetype_id": "scheduled_automation", "risk": "risk_bearing", "query": "Build an unattended batch job that processes pending cases as the main workflow. Keep the experience simple.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "HIGH_COST_CONSUMPTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [ "HIGH_COST_CONSUMPTION" ], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-003", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00076", "persona": "Machine-learning engineer", "archetype_id": "admin_audit", "risk": "risk_bearing", "query": "Create staff tools for suspending and restoring user accounts inside an existing application. It should work consistently.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "PRIVILEGED_ACTION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "IRREVERSIBLE_ACTION" ], "missing_characteristics": [ "PRIVILEGED_ACTION" ], "additional_characteristics": [ "IRREVERSIBLE_ACTION" ], "expected_packs": [ "AUT-001", "AUT-004", "IDN-004", "IDN-005", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-002", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00079", "persona": "Operations professional", "archetype_id": "database_migration", "risk": "risk_bearing", "query": "We need an operation that transforms records in the production database for a small team. Keep the experience simple.", "expected_characteristics": [ "IRREVERSIBLE_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "PRIVILEGED_ACTION", "PRODUCTION_MODIFICATION" ], "missing_characteristics": [ "IRREVERSIBLE_ACTION" ], "additional_characteristics": [ "CONSEQUENTIAL_ACTION", "PRIVILEGED_ACTION" ], "expected_packs": [ "AUT-002", "CHG-001", "REL-004" ], "matched_packs": [ "AUT-001", "AUT-004", "CHG-001", "IDN-004", "IDN-005", "OBS-001", "OBS-002", "OBS-003", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00084", "persona": "Marketplace operator", "archetype_id": "card_payment", "risk": "risk_bearing", "query": "We are adding a purchase flow connected to an external payment processor inside an existing application. This is the main feature.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION" ], "detected_characteristics": [ "EXTERNAL_DEPENDENCY" ], "missing_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003", "REL-001" ], "matched_packs": [ "OBS-003", "REL-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00090", "persona": "Healthcare administrator", "archetype_id": "automated_refund", "risk": "risk_bearing", "query": "Please help us set up a service that triggers refunds through an external function as the main workflow. People should understand what happened.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00093", "persona": "Non-technical founder", "archetype_id": "external_sync", "risk": "risk_bearing", "query": "I want daily data sync with an outside business platform for regular day-to-day use. Keep the experience simple.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "SENSITIVE_DATA" ], "detected_characteristics": [ "EXTERNAL_DEPENDENCY" ], "missing_characteristics": [ "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "OBS-003", "REL-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00098", "persona": "Government analyst", "archetype_id": "payroll", "risk": "risk_bearing", "query": "The proposed service includes monthly payroll with employee banking and tax details inside an existing application. Keep the experience simple.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "detected_characteristics": [ "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00099", "persona": "Student or learner", "archetype_id": "payroll", "risk": "risk_bearing", "query": "For a learning project, make a salary system connected to an outside payroll service inside an existing application. Keep the experience simple.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "detected_characteristics": [ "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00103", "persona": "Machine-learning engineer", "archetype_id": "external_sync", "risk": "risk_bearing", "query": "Create customer-data synchronization through an external API for regular day-to-day use. This is the main feature.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "SENSITIVE_DATA" ], "detected_characteristics": [ "EXTERNAL_DEPENDENCY" ], "missing_characteristics": [ "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "OBS-003", "REL-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00106", "persona": "Non-technical founder", "archetype_id": "admin_audit", "risk": "risk_bearing", "query": "I want staff tools for suspending and restoring user accounts for regular day-to-day use. The first version can stay focused.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "PRIVILEGED_ACTION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "IRREVERSIBLE_ACTION" ], "missing_characteristics": [ "PRIVILEGED_ACTION" ], "additional_characteristics": [ "IRREVERSIBLE_ACTION" ], "expected_packs": [ "AUT-001", "AUT-004", "IDN-004", "IDN-005", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-002", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00112", "persona": "Government analyst", "archetype_id": "deployment_agent", "risk": "risk_bearing", "query": "The proposed service includes automation that applies configuration changes in production for a small team. People should understand what happened.", "expected_characteristics": [ "PRIVILEGED_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "PRIVILEGED_ACTION" ], "missing_characteristics": [ "PRODUCTION_MODIFICATION" ], "additional_characteristics": [ "CONSEQUENTIAL_ACTION" ], "expected_packs": [ "CHG-001", "IDN-004", "IDN-005", "OBS-002", "REL-004" ], "matched_packs": [ "AUT-001", "AUT-004", "IDN-004", "IDN-005", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00113", "persona": "Non-technical founder", "archetype_id": "webhook", "risk": "risk_bearing", "query": "I want incoming webhooks from an external billing provider for a new service. Keep the experience simple.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "UNTRUSTED_INPUT" ], "detected_characteristics": [ "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION" ], "missing_characteristics": [ "UNTRUSTED_INPUT" ], "additional_characteristics": [ "FINANCIAL_ACTION" ], "expected_packs": [ "INP-001", "INP-002", "OBS-003", "REL-001" ], "matched_packs": [ "AUT-001", "OBS-003", "REL-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00114", "persona": "Marketplace operator", "archetype_id": "scheduled_automation", "risk": "risk_bearing", "query": "We are adding an unattended batch job that processes pending cases for a new service. People should understand what happened.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "HIGH_COST_CONSUMPTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [ "HIGH_COST_CONSUMPTION" ], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-003", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00116", "persona": "Financial-technology engineer", "archetype_id": "database_migration", "risk": "risk_bearing", "query": "The system should provide schema migrations applied directly to the live database as the main workflow. This is the main feature.", "expected_characteristics": [ "IRREVERSIBLE_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "IRREVERSIBLE_ACTION" ], "missing_characteristics": [ "PRODUCTION_MODIFICATION" ], "additional_characteristics": [ "CONSEQUENTIAL_ACTION" ], "expected_packs": [ "AUT-002", "CHG-001", "REL-004" ], "matched_packs": [ "AUT-001", "AUT-002", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00117", "persona": "Independent developer", "archetype_id": "data_export", "risk": "risk_bearing", "query": "Build a report that sends confidential records outside the application as the main workflow. It should work consistently.", "expected_characteristics": [ "DATA_EXPORT", "SENSITIVE_DATA" ], "detected_characteristics": [ "DATA_EXPORT" ], "missing_characteristics": [ "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-002", "DAT-005", "IDN-005", "REL-004" ], "matched_packs": [ "DAT-001", "DAT-002" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00121", "persona": "Healthcare administrator", "archetype_id": "scheduled_automation", "risk": "risk_bearing", "query": "Please help us set up scheduled automation that changes order statuses as the main workflow. The first version can stay focused.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00122", "persona": "Small-business owner", "archetype_id": "data_export", "risk": "risk_bearing", "query": "Could you make a report that sends confidential records outside the application for regular day-to-day use. People should understand what happened.", "expected_characteristics": [ "DATA_EXPORT", "SENSITIVE_DATA" ], "detected_characteristics": [ "DATA_EXPORT" ], "missing_characteristics": [ "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-002", "DAT-005", "IDN-005", "REL-004" ], "matched_packs": [ "DAT-001", "DAT-002" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00124", "persona": "Accounting professional", "archetype_id": "deployment_agent", "risk": "risk_bearing", "query": "I need a tool with an agent that deploys code and changes production configuration for regular day-to-day use. Keep the experience simple.", "expected_characteristics": [ "PRIVILEGED_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "PRODUCTION_MODIFICATION" ], "missing_characteristics": [ "PRIVILEGED_ACTION" ], "additional_characteristics": [ "CONSEQUENTIAL_ACTION" ], "expected_packs": [ "CHG-001", "IDN-004", "IDN-005", "OBS-002", "REL-004" ], "matched_packs": [ "AUT-001", "AUT-004", "CHG-001", "OBS-001", "OBS-003", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00126", "persona": "Site reliability engineer", "archetype_id": "scheduled_automation", "risk": "risk_bearing", "query": "We are planning a background workflow that updates records every morning as the main workflow. This is the main feature.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00132", "persona": "Independent developer", "archetype_id": "payroll", "risk": "risk_bearing", "query": "Build payroll processing that sends employee pay to a provider for a new service. It should work consistently.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "detected_characteristics": [ "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00141", "persona": "Non-technical founder", "archetype_id": "database_migration", "risk": "risk_bearing", "query": "I want schema migrations applied directly to the live database inside an existing application. It should work consistently.", "expected_characteristics": [ "IRREVERSIBLE_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "IRREVERSIBLE_ACTION" ], "missing_characteristics": [ "PRODUCTION_MODIFICATION" ], "additional_characteristics": [ "CONSEQUENTIAL_ACTION" ], "expected_packs": [ "AUT-002", "CHG-001", "REL-004" ], "matched_packs": [ "AUT-001", "AUT-002", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00143", "persona": "Operations professional", "archetype_id": "automated_refund", "risk": "risk_bearing", "query": "We need automatic refunds issued through the payment API for regular day-to-day use. It should work consistently.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00145", "persona": "Nonprofit program lead", "archetype_id": "webhook", "risk": "risk_bearing", "query": "Our organization needs outside services posting event payloads to our API for a new service. People should understand what happened.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "UNTRUSTED_INPUT" ], "detected_characteristics": [ "EXTERNAL_DEPENDENCY" ], "missing_characteristics": [ "UNTRUSTED_INPUT" ], "additional_characteristics": [], "expected_packs": [ "INP-001", "INP-002", "OBS-003", "REL-001" ], "matched_packs": [ "OBS-003", "REL-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00147", "persona": "Accounting professional", "archetype_id": "deployment_agent", "risk": "risk_bearing", "query": "I need a tool with automatic production releases with infrastructure updates inside an existing application. Keep the experience simple.", "expected_characteristics": [ "PRIVILEGED_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "PRODUCTION_MODIFICATION" ], "missing_characteristics": [ "PRIVILEGED_ACTION" ], "additional_characteristics": [], "expected_packs": [ "CHG-001", "IDN-004", "IDN-005", "OBS-002", "REL-004" ], "matched_packs": [ "CHG-001", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00153", "persona": "Human-resources manager", "archetype_id": "external_notifications", "risk": "risk_bearing", "query": "We would use a workflow that sends customer alerts through a messaging API as the main workflow. Keep the experience simple.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003", "REL-001" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00159", "persona": "Healthcare administrator", "archetype_id": "scheduled_automation", "risk": "risk_bearing", "query": "Please help us set up an unattended batch job that processes pending cases for regular day-to-day use. The first version can stay focused.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "HIGH_COST_CONSUMPTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [ "HIGH_COST_CONSUMPTION" ], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-003", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00160", "persona": "Human-resources manager", "archetype_id": "card_payment", "risk": "risk_bearing", "query": "We would use online orders that charge a customer's card as the main workflow. The first version can stay focused.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION" ], "detected_characteristics": [ "FINANCIAL_ACTION" ], "missing_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003", "REL-001" ], "matched_packs": [ "AUT-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00161", "persona": "Machine-learning engineer", "archetype_id": "external_notifications", "risk": "risk_bearing", "query": "Create a workflow that sends customer alerts through a messaging API for a new service. This is the main feature.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003", "REL-001" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00169", "persona": "Enterprise product manager", "archetype_id": "card_payment", "risk": "risk_bearing", "query": "Our next feature is online orders that charge a customer's card inside an existing application. This is the main feature.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION" ], "detected_characteristics": [ "FINANCIAL_ACTION" ], "missing_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003", "REL-001" ], "matched_packs": [ "AUT-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00171", "persona": "Nonprofit program lead", "archetype_id": "webhook", "risk": "risk_bearing", "query": "Our organization needs incoming webhooks from an external billing provider inside an existing application. This is the main feature.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "UNTRUSTED_INPUT" ], "detected_characteristics": [ "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION" ], "missing_characteristics": [ "UNTRUSTED_INPUT" ], "additional_characteristics": [ "FINANCIAL_ACTION" ], "expected_packs": [ "INP-001", "INP-002", "OBS-003", "REL-001" ], "matched_packs": [ "AUT-001", "OBS-003", "REL-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00172", "persona": "Independent developer", "archetype_id": "external_sync", "risk": "risk_bearing", "query": "Build a connection that sends private records to a third-party service for a small team. It should work consistently.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "SENSITIVE_DATA" ], "detected_characteristics": [ "EXTERNAL_DEPENDENCY" ], "missing_characteristics": [ "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "OBS-003", "REL-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00178", "persona": "Security lead", "archetype_id": "card_payment", "risk": "risk_bearing", "query": "Review a design for checkout with card payments through a payment provider as the main workflow. This is the main feature.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION" ], "detected_characteristics": [ "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION" ], "missing_characteristics": [ "CONSEQUENTIAL_ACTION" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003", "REL-001" ], "matched_packs": [ "AUT-001", "OBS-003", "REL-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00179", "persona": "Non-technical founder", "archetype_id": "card_payment", "risk": "risk_bearing", "query": "I want a checkout page that sends charges to a card gateway inside an existing application. This is the main feature.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION" ], "detected_characteristics": [ "FINANCIAL_ACTION" ], "missing_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003", "REL-001" ], "matched_packs": [ "AUT-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00180", "persona": "Operations professional", "archetype_id": "external_sync", "risk": "risk_bearing", "query": "We need daily data sync with an outside business platform as the main workflow. This is the main feature.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "SENSITIVE_DATA" ], "detected_characteristics": [ "EXTERNAL_DEPENDENCY" ], "missing_characteristics": [ "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "OBS-003", "REL-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00181", "persona": "Marketplace operator", "archetype_id": "data_export", "risk": "risk_bearing", "query": "We are adding downloadable archives containing customer information for regular day-to-day use. It should work consistently.", "expected_characteristics": [ "DATA_EXPORT", "SENSITIVE_DATA" ], "detected_characteristics": [ "DATA_EXPORT" ], "missing_characteristics": [ "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-002", "DAT-005", "IDN-005", "REL-004" ], "matched_packs": [ "DAT-001", "DAT-002" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00182", "persona": "Human-resources manager", "archetype_id": "card_payment", "risk": "risk_bearing", "query": "We would use a purchase flow connected to an external payment processor as the main workflow. The first version can stay focused.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION" ], "detected_characteristics": [ "EXTERNAL_DEPENDENCY" ], "missing_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003", "REL-001" ], "matched_packs": [ "OBS-003", "REL-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00188", "persona": "Machine-learning engineer", "archetype_id": "deployment_agent", "risk": "risk_bearing", "query": "Create a deployment tool that modifies live services for regular day-to-day use. People should understand what happened.", "expected_characteristics": [ "PRIVILEGED_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "PRIVILEGED_ACTION" ], "missing_characteristics": [ "PRODUCTION_MODIFICATION" ], "additional_characteristics": [ "CONSEQUENTIAL_ACTION" ], "expected_packs": [ "CHG-001", "IDN-004", "IDN-005", "OBS-002", "REL-004" ], "matched_packs": [ "AUT-001", "AUT-004", "IDN-004", "IDN-005", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00190", "persona": "Data engineer", "archetype_id": "deployment_agent", "risk": "risk_bearing", "query": "Implement automation that applies configuration changes in production for a new service. People should understand what happened.", "expected_characteristics": [ "PRIVILEGED_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "PRIVILEGED_ACTION" ], "missing_characteristics": [ "PRODUCTION_MODIFICATION" ], "additional_characteristics": [ "CONSEQUENTIAL_ACTION" ], "expected_packs": [ "CHG-001", "IDN-004", "IDN-005", "OBS-002", "REL-004" ], "matched_packs": [ "AUT-001", "AUT-004", "IDN-004", "IDN-005", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00194", "persona": "Marketplace operator", "archetype_id": "automated_refund", "risk": "risk_bearing", "query": "We are adding a workflow that calls a billing tool to approve refunds for a new service. It should work consistently.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00196", "persona": "Independent developer", "archetype_id": "payroll", "risk": "risk_bearing", "query": "Build a salary system connected to an outside payroll service for regular day-to-day use. It should work consistently.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "detected_characteristics": [ "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00197", "persona": "Student or learner", "archetype_id": "database_migration", "risk": "risk_bearing", "query": "For a learning project, make schema migrations applied directly to the live database for regular day-to-day use. This is the main feature.", "expected_characteristics": [ "IRREVERSIBLE_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "IRREVERSIBLE_ACTION" ], "missing_characteristics": [ "PRODUCTION_MODIFICATION" ], "additional_characteristics": [ "CONSEQUENTIAL_ACTION" ], "expected_packs": [ "AUT-002", "CHG-001", "REL-004" ], "matched_packs": [ "AUT-001", "AUT-002", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00202", "persona": "Site reliability engineer", "archetype_id": "webhook", "risk": "risk_bearing", "query": "We are planning incoming webhooks from an external billing provider for a small team. People should understand what happened.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "UNTRUSTED_INPUT" ], "detected_characteristics": [ "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION" ], "missing_characteristics": [ "UNTRUSTED_INPUT" ], "additional_characteristics": [ "FINANCIAL_ACTION" ], "expected_packs": [ "INP-001", "INP-002", "OBS-003", "REL-001" ], "matched_packs": [ "AUT-001", "OBS-003", "REL-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00203", "persona": "Healthcare administrator", "archetype_id": "deployment_agent", "risk": "risk_bearing", "query": "Please help us set up an agent that deploys code and changes production configuration as the main workflow. Keep the experience simple.", "expected_characteristics": [ "PRIVILEGED_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "PRODUCTION_MODIFICATION" ], "missing_characteristics": [ "PRIVILEGED_ACTION" ], "additional_characteristics": [ "CONSEQUENTIAL_ACTION" ], "expected_packs": [ "CHG-001", "IDN-004", "IDN-005", "OBS-002", "REL-004" ], "matched_packs": [ "AUT-001", "AUT-004", "CHG-001", "OBS-001", "OBS-003", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00204", "persona": "Operations professional", "archetype_id": "data_export", "risk": "risk_bearing", "query": "We need downloadable archives containing customer information as the main workflow. People should understand what happened.", "expected_characteristics": [ "DATA_EXPORT", "SENSITIVE_DATA" ], "detected_characteristics": [ "DATA_EXPORT" ], "missing_characteristics": [ "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-002", "DAT-005", "IDN-005", "REL-004" ], "matched_packs": [ "DAT-001", "DAT-002" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00211", "persona": "Media producer", "archetype_id": "external_sync", "risk": "risk_bearing", "query": "My project needs customer-data synchronization through an external API for regular day-to-day use. Keep the experience simple.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "SENSITIVE_DATA" ], "detected_characteristics": [ "EXTERNAL_DEPENDENCY" ], "missing_characteristics": [ "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "OBS-003", "REL-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00214", "persona": "Media producer", "archetype_id": "payroll", "risk": "risk_bearing", "query": "My project needs employee compensation records and payroll payments as the main workflow. The first version can stay focused.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "detected_characteristics": [ "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00215", "persona": "Enterprise product manager", "archetype_id": "external_sync", "risk": "risk_bearing", "query": "Our next feature is customer-data synchronization through an external API inside an existing application. Keep the experience simple.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "SENSITIVE_DATA" ], "detected_characteristics": [ "EXTERNAL_DEPENDENCY" ], "missing_characteristics": [ "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "OBS-003", "REL-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00223", "persona": "Independent developer", "archetype_id": "payroll", "risk": "risk_bearing", "query": "Build payroll processing that sends employee pay to a provider for regular day-to-day use. Keep the experience simple.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "detected_characteristics": [ "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00224", "persona": "Enterprise product manager", "archetype_id": "database_migration", "risk": "risk_bearing", "query": "Our next feature is a live data migration that rewrites existing rows inside an existing application. This is the main feature.", "expected_characteristics": [ "IRREVERSIBLE_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "IRREVERSIBLE_ACTION" ], "missing_characteristics": [ "PRODUCTION_MODIFICATION" ], "additional_characteristics": [ "CONSEQUENTIAL_ACTION" ], "expected_packs": [ "AUT-002", "CHG-001", "REL-004" ], "matched_packs": [ "AUT-001", "AUT-002", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00233", "persona": "Media producer", "archetype_id": "external_sync", "risk": "risk_bearing", "query": "My project needs daily data sync with an outside business platform as the main workflow. This is the main feature.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "SENSITIVE_DATA" ], "detected_characteristics": [ "EXTERNAL_DEPENDENCY" ], "missing_characteristics": [ "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "OBS-003", "REL-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00234", "persona": "Media producer", "archetype_id": "automated_refund", "risk": "risk_bearing", "query": "My project needs a bot that sends eligible customers their money back inside an existing application. People should understand what happened.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "TOOL_ACCESS" ], "missing_characteristics": [ "FINANCIAL_ACTION" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00243", "persona": "Site reliability engineer", "archetype_id": "payroll", "risk": "risk_bearing", "query": "We are planning employee compensation records and payroll payments as the main workflow. Keep the experience simple.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "detected_characteristics": [ "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00244", "persona": "Security lead", "archetype_id": "external_sync", "risk": "risk_bearing", "query": "Review a design for daily data sync with an outside business platform as the main workflow. Keep the experience simple.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "SENSITIVE_DATA" ], "detected_characteristics": [ "EXTERNAL_DEPENDENCY" ], "missing_characteristics": [ "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "OBS-003", "REL-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00246", "persona": "Site reliability engineer", "archetype_id": "external_sync", "risk": "risk_bearing", "query": "We are planning daily data sync with an outside business platform as the main workflow. The first version can stay focused.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "SENSITIVE_DATA" ], "detected_characteristics": [ "EXTERNAL_DEPENDENCY" ], "missing_characteristics": [ "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "OBS-003", "REL-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00248", "persona": "Accounting professional", "archetype_id": "document_upload", "risk": "risk_bearing", "query": "I need a tool with a portal where people upload confidential PDF forms for a new service. This is the main feature.", "expected_characteristics": [ "SENSITIVE_DATA", "UNTRUSTED_INPUT" ], "detected_characteristics": [ "UNTRUSTED_INPUT" ], "missing_characteristics": [ "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "INP-001", "INP-002", "REL-004" ], "matched_packs": [ "INP-001", "INP-002" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00251", "persona": "Small-business owner", "archetype_id": "data_export", "risk": "risk_bearing", "query": "Could you make bulk export of private account data for regular day-to-day use. The first version can stay focused.", "expected_characteristics": [ "DATA_EXPORT", "SENSITIVE_DATA" ], "detected_characteristics": [ "DATA_EXPORT" ], "missing_characteristics": [ "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-002", "DAT-005", "IDN-005", "REL-004" ], "matched_packs": [ "DAT-001", "DAT-002" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00256", "persona": "Student or learner", "archetype_id": "scheduled_automation", "risk": "risk_bearing", "query": "For a learning project, make an unattended batch job that processes pending cases for a new service. It should work consistently.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "HIGH_COST_CONSUMPTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [ "HIGH_COST_CONSUMPTION" ], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-003", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00261", "persona": "Enterprise product manager", "archetype_id": "deployment_agent", "risk": "risk_bearing", "query": "Our next feature is a deployment tool that modifies live services for a small team. The first version can stay focused.", "expected_characteristics": [ "PRIVILEGED_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "PRIVILEGED_ACTION" ], "missing_characteristics": [ "PRODUCTION_MODIFICATION" ], "additional_characteristics": [ "CONSEQUENTIAL_ACTION" ], "expected_packs": [ "CHG-001", "IDN-004", "IDN-005", "OBS-002", "REL-004" ], "matched_packs": [ "AUT-001", "AUT-004", "IDN-004", "IDN-005", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00264", "persona": "Student or learner", "archetype_id": "card_payment", "risk": "risk_bearing", "query": "For a learning project, make checkout with card payments through a payment provider for a small team. It should work consistently.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION" ], "detected_characteristics": [ "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION" ], "missing_characteristics": [ "CONSEQUENTIAL_ACTION" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003", "REL-001" ], "matched_packs": [ "AUT-001", "OBS-003", "REL-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00267", "persona": "Non-technical founder", "archetype_id": "admin_audit", "risk": "risk_bearing", "query": "I want staff tools for suspending and restoring user accounts for a small team. People should understand what happened.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "PRIVILEGED_ACTION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "IRREVERSIBLE_ACTION" ], "missing_characteristics": [ "PRIVILEGED_ACTION" ], "additional_characteristics": [ "IRREVERSIBLE_ACTION" ], "expected_packs": [ "AUT-001", "AUT-004", "IDN-004", "IDN-005", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-002", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00268", "persona": "Nonprofit program lead", "archetype_id": "automated_refund", "risk": "risk_bearing", "query": "Our organization needs a workflow that calls a billing tool to approve refunds for regular day-to-day use. It should work consistently.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00276", "persona": "Non-technical founder", "archetype_id": "automated_refund", "risk": "risk_bearing", "query": "I want automatic refund processing without manual entry inside an existing application. Keep the experience simple.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00281", "persona": "Machine-learning engineer", "archetype_id": "automated_refund", "risk": "risk_bearing", "query": "Create a workflow that calls a billing tool to approve refunds for a new service. Keep the experience simple.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00283", "persona": "Financial-technology engineer", "archetype_id": "external_sync", "risk": "risk_bearing", "query": "The system should provide a connection that sends private records to a third-party service inside an existing application. This is the main feature.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "SENSITIVE_DATA" ], "detected_characteristics": [ "EXTERNAL_DEPENDENCY" ], "missing_characteristics": [ "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "OBS-003", "REL-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00286", "persona": "Non-technical founder", "archetype_id": "automated_refund", "risk": "risk_bearing", "query": "I want a workflow that calls a billing tool to approve refunds for regular day-to-day use. It should work consistently.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00288", "persona": "Human-resources manager", "archetype_id": "card_payment", "risk": "risk_bearing", "query": "We would use checkout with card payments through a payment provider for a new service. The first version can stay focused.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION" ], "detected_characteristics": [ "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION" ], "missing_characteristics": [ "CONSEQUENTIAL_ACTION" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003", "REL-001" ], "matched_packs": [ "AUT-001", "OBS-003", "REL-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00289", "persona": "Financial-technology engineer", "archetype_id": "scheduled_automation", "risk": "risk_bearing", "query": "The system should provide a background workflow that updates records every morning for a new service. This is the main feature.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00291", "persona": "Student or learner", "archetype_id": "database_migration", "risk": "risk_bearing", "query": "For a learning project, make production database changes with a difficult rollback for regular day-to-day use. This is the main feature.", "expected_characteristics": [ "IRREVERSIBLE_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "PRIVILEGED_ACTION", "PRODUCTION_MODIFICATION" ], "missing_characteristics": [ "IRREVERSIBLE_ACTION" ], "additional_characteristics": [ "CONSEQUENTIAL_ACTION", "PRIVILEGED_ACTION" ], "expected_packs": [ "AUT-002", "CHG-001", "REL-004" ], "matched_packs": [ "AUT-001", "AUT-004", "CHG-001", "IDN-004", "IDN-005", "OBS-001", "OBS-002", "OBS-003", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00292", "persona": "Enterprise product manager", "archetype_id": "deployment_agent", "risk": "risk_bearing", "query": "Our next feature is automatic production releases with infrastructure updates for a small team. Keep the experience simple.", "expected_characteristics": [ "PRIVILEGED_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "PRODUCTION_MODIFICATION" ], "missing_characteristics": [ "PRIVILEGED_ACTION" ], "additional_characteristics": [], "expected_packs": [ "CHG-001", "IDN-004", "IDN-005", "OBS-002", "REL-004" ], "matched_packs": [ "CHG-001", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00293", "persona": "Site reliability engineer", "archetype_id": "automated_refund", "risk": "risk_bearing", "query": "We are planning a service that triggers refunds through an external function inside an existing application. This is the main feature.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00309", "persona": "Financial-technology engineer", "archetype_id": "deployment_agent", "risk": "risk_bearing", "query": "The system should provide an agent that deploys code and changes production configuration for a new service. This is the main feature.", "expected_characteristics": [ "PRIVILEGED_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "PRODUCTION_MODIFICATION" ], "missing_characteristics": [ "PRIVILEGED_ACTION" ], "additional_characteristics": [ "CONSEQUENTIAL_ACTION" ], "expected_packs": [ "CHG-001", "IDN-004", "IDN-005", "OBS-002", "REL-004" ], "matched_packs": [ "AUT-001", "AUT-004", "CHG-001", "OBS-001", "OBS-003", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00310", "persona": "Independent developer", "archetype_id": "automated_refund", "risk": "risk_bearing", "query": "Build automatic refund processing without manual entry inside an existing application. Keep the experience simple.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00313", "persona": "Media producer", "archetype_id": "automated_refund", "risk": "risk_bearing", "query": "My project needs a service that triggers refunds through an external function as the main workflow. It should work consistently.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00316", "persona": "Mobile developer", "archetype_id": "payroll", "risk": "risk_bearing", "query": "The app needs a tool that calculates pay and submits it to a payroll vendor for regular day-to-day use. It should work consistently.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "detected_characteristics": [ "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00318", "persona": "Enterprise product manager", "archetype_id": "payroll", "risk": "risk_bearing", "query": "Our next feature is monthly payroll with employee banking and tax details for regular day-to-day use. This is the main feature.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "detected_characteristics": [ "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00323", "persona": "Human-resources manager", "archetype_id": "external_notifications", "risk": "risk_bearing", "query": "We would use a workflow that sends customer alerts through a messaging API inside an existing application. People should understand what happened.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003", "REL-001" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00327", "persona": "Financial-technology engineer", "archetype_id": "payroll", "risk": "risk_bearing", "query": "The system should provide payroll processing that sends employee pay to a provider inside an existing application. Keep the experience simple.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "detected_characteristics": [ "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00329", "persona": "Government analyst", "archetype_id": "card_payment", "risk": "risk_bearing", "query": "The proposed service includes checkout with card payments through a payment provider inside an existing application. People should understand what happened.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION" ], "detected_characteristics": [ "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION" ], "missing_characteristics": [ "CONSEQUENTIAL_ACTION" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003", "REL-001" ], "matched_packs": [ "AUT-001", "OBS-003", "REL-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00336", "persona": "Independent developer", "archetype_id": "deployment_agent", "risk": "risk_bearing", "query": "Build automatic production releases with infrastructure updates as the main workflow. It should work consistently.", "expected_characteristics": [ "PRIVILEGED_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "PRODUCTION_MODIFICATION" ], "missing_characteristics": [ "PRIVILEGED_ACTION" ], "additional_characteristics": [], "expected_packs": [ "CHG-001", "IDN-004", "IDN-005", "OBS-002", "REL-004" ], "matched_packs": [ "CHG-001", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00338", "persona": "Enterprise product manager", "archetype_id": "database_migration", "risk": "risk_bearing", "query": "Our next feature is schema migrations applied directly to the live database for a new service. People should understand what happened.", "expected_characteristics": [ "IRREVERSIBLE_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "IRREVERSIBLE_ACTION" ], "missing_characteristics": [ "PRODUCTION_MODIFICATION" ], "additional_characteristics": [ "CONSEQUENTIAL_ACTION" ], "expected_packs": [ "AUT-002", "CHG-001", "REL-004" ], "matched_packs": [ "AUT-001", "AUT-002", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00339", "persona": "Data engineer", "archetype_id": "card_payment", "risk": "risk_bearing", "query": "Implement checkout with card payments through a payment provider for a small team. People should understand what happened.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION" ], "detected_characteristics": [ "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION" ], "missing_characteristics": [ "CONSEQUENTIAL_ACTION" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003", "REL-001" ], "matched_packs": [ "AUT-001", "OBS-003", "REL-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00343", "persona": "Security lead", "archetype_id": "admin_audit", "risk": "risk_bearing", "query": "Review a design for staff tools for suspending and restoring user accounts inside an existing application. People should understand what happened.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "PRIVILEGED_ACTION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "IRREVERSIBLE_ACTION" ], "missing_characteristics": [ "PRIVILEGED_ACTION" ], "additional_characteristics": [ "IRREVERSIBLE_ACTION" ], "expected_packs": [ "AUT-001", "AUT-004", "IDN-004", "IDN-005", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-002", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00345", "persona": "Non-technical founder", "archetype_id": "data_export", "risk": "risk_bearing", "query": "I want bulk export of private account data inside an existing application. It should work consistently.", "expected_characteristics": [ "DATA_EXPORT", "SENSITIVE_DATA" ], "detected_characteristics": [ "DATA_EXPORT" ], "missing_characteristics": [ "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-002", "DAT-005", "IDN-005", "REL-004" ], "matched_packs": [ "DAT-001", "DAT-002" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00347", "persona": "Enterprise product manager", "archetype_id": "automated_refund", "risk": "risk_bearing", "query": "Our next feature is a service that triggers refunds through an external function for regular day-to-day use. The first version can stay focused.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00353", "persona": "Media producer", "archetype_id": "scheduled_automation", "risk": "risk_bearing", "query": "My project needs scheduled automation that changes order statuses for regular day-to-day use. Keep the experience simple.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00359", "persona": "Machine-learning engineer", "archetype_id": "card_payment", "risk": "risk_bearing", "query": "Create online orders that charge a customer's card for a new service. The first version can stay focused.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION" ], "detected_characteristics": [ "FINANCIAL_ACTION" ], "missing_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003", "REL-001" ], "matched_packs": [ "AUT-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00362", "persona": "Marketplace operator", "archetype_id": "document_upload", "risk": "risk_bearing", "query": "We are adding a portal where people upload confidential PDF forms for regular day-to-day use. This is the main feature.", "expected_characteristics": [ "SENSITIVE_DATA", "UNTRUSTED_INPUT" ], "detected_characteristics": [ "UNTRUSTED_INPUT" ], "missing_characteristics": [ "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "INP-001", "INP-002", "REL-004" ], "matched_packs": [ "INP-001", "INP-002" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00371", "persona": "Small-business owner", "archetype_id": "external_notifications", "risk": "risk_bearing", "query": "Could you make a workflow that sends customer alerts through a messaging API for a small team. Keep the experience simple.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003", "REL-001" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00377", "persona": "Human-resources manager", "archetype_id": "external_notifications", "risk": "risk_bearing", "query": "We would use a workflow that sends customer alerts through a messaging API as the main workflow. People should understand what happened.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003", "REL-001" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00378", "persona": "Media producer", "archetype_id": "scheduled_automation", "risk": "risk_bearing", "query": "My project needs a nightly job that reconciles and updates invoices inside an existing application. People should understand what happened.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [ "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "DAT-001", "DAT-005", "IDN-005", "OBS-001", "OBS-003", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00386", "persona": "Data engineer", "archetype_id": "payroll", "risk": "risk_bearing", "query": "Implement employee compensation records and payroll payments as the main workflow. Keep the experience simple.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "detected_characteristics": [ "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00396", "persona": "Healthcare administrator", "archetype_id": "automated_refund", "risk": "risk_bearing", "query": "Please help us set up automatic refund processing without manual entry for a small team. It should work consistently.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00398", "persona": "Financial-technology engineer", "archetype_id": "database_migration", "risk": "risk_bearing", "query": "The system should provide production database changes with a difficult rollback inside an existing application. Keep the experience simple.", "expected_characteristics": [ "IRREVERSIBLE_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "PRIVILEGED_ACTION", "PRODUCTION_MODIFICATION" ], "missing_characteristics": [ "IRREVERSIBLE_ACTION" ], "additional_characteristics": [ "CONSEQUENTIAL_ACTION", "PRIVILEGED_ACTION" ], "expected_packs": [ "AUT-002", "CHG-001", "REL-004" ], "matched_packs": [ "AUT-001", "AUT-004", "CHG-001", "IDN-004", "IDN-005", "OBS-001", "OBS-002", "OBS-003", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00399", "persona": "Site reliability engineer", "archetype_id": "database_migration", "risk": "risk_bearing", "query": "We are planning a live data migration that rewrites existing rows as the main workflow. This is the main feature.", "expected_characteristics": [ "IRREVERSIBLE_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "IRREVERSIBLE_ACTION" ], "missing_characteristics": [ "PRODUCTION_MODIFICATION" ], "additional_characteristics": [ "CONSEQUENTIAL_ACTION" ], "expected_packs": [ "AUT-002", "CHG-001", "REL-004" ], "matched_packs": [ "AUT-001", "AUT-002", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00405", "persona": "Independent developer", "archetype_id": "automated_refund", "risk": "risk_bearing", "query": "Build a service that triggers refunds through an external function for a new service. This is the main feature.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00408", "persona": "Financial-technology engineer", "archetype_id": "automated_refund", "risk": "risk_bearing", "query": "The system should provide a service that triggers refunds through an external function inside an existing application. This is the main feature.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00413", "persona": "Data engineer", "archetype_id": "external_sync", "risk": "risk_bearing", "query": "Implement daily data sync with an outside business platform inside an existing application. Keep the experience simple.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "SENSITIVE_DATA" ], "detected_characteristics": [ "EXTERNAL_DEPENDENCY" ], "missing_characteristics": [ "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "OBS-003", "REL-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00414", "persona": "Enterprise product manager", "archetype_id": "deployment_agent", "risk": "risk_bearing", "query": "Our next feature is automatic production releases with infrastructure updates for regular day-to-day use. It should work consistently.", "expected_characteristics": [ "PRIVILEGED_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "PRODUCTION_MODIFICATION" ], "missing_characteristics": [ "PRIVILEGED_ACTION" ], "additional_characteristics": [], "expected_packs": [ "CHG-001", "IDN-004", "IDN-005", "OBS-002", "REL-004" ], "matched_packs": [ "CHG-001", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00418", "persona": "Non-technical founder", "archetype_id": "data_export", "risk": "risk_bearing", "query": "I want a feature that downloads employee information for regular day-to-day use. This is the main feature.", "expected_characteristics": [ "DATA_EXPORT", "SENSITIVE_DATA" ], "detected_characteristics": [ "DATA_EXPORT" ], "missing_characteristics": [ "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-002", "DAT-005", "IDN-005", "REL-004" ], "matched_packs": [ "DAT-001", "DAT-002" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00420", "persona": "Nonprofit program lead", "archetype_id": "database_migration", "risk": "risk_bearing", "query": "Our organization needs schema migrations applied directly to the live database for regular day-to-day use. People should understand what happened.", "expected_characteristics": [ "IRREVERSIBLE_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "IRREVERSIBLE_ACTION" ], "missing_characteristics": [ "PRODUCTION_MODIFICATION" ], "additional_characteristics": [ "CONSEQUENTIAL_ACTION" ], "expected_packs": [ "AUT-002", "CHG-001", "REL-004" ], "matched_packs": [ "AUT-001", "AUT-002", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00421", "persona": "Nonprofit program lead", "archetype_id": "deployment_agent", "risk": "risk_bearing", "query": "Our organization needs a deployment tool that modifies live services for a small team. This is the main feature.", "expected_characteristics": [ "PRIVILEGED_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "PRIVILEGED_ACTION" ], "missing_characteristics": [ "PRODUCTION_MODIFICATION" ], "additional_characteristics": [ "CONSEQUENTIAL_ACTION" ], "expected_packs": [ "CHG-001", "IDN-004", "IDN-005", "OBS-002", "REL-004" ], "matched_packs": [ "AUT-001", "AUT-004", "IDN-004", "IDN-005", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00422", "persona": "Non-technical founder", "archetype_id": "database_migration", "risk": "risk_bearing", "query": "I want an operation that transforms records in the production database inside an existing application. The first version can stay focused.", "expected_characteristics": [ "IRREVERSIBLE_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "PRIVILEGED_ACTION", "PRODUCTION_MODIFICATION" ], "missing_characteristics": [ "IRREVERSIBLE_ACTION" ], "additional_characteristics": [ "CONSEQUENTIAL_ACTION", "PRIVILEGED_ACTION" ], "expected_packs": [ "AUT-002", "CHG-001", "REL-004" ], "matched_packs": [ "AUT-001", "AUT-004", "CHG-001", "IDN-004", "IDN-005", "OBS-001", "OBS-002", "OBS-003", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00423", "persona": "Independent developer", "archetype_id": "database_migration", "risk": "risk_bearing", "query": "Build schema migrations applied directly to the live database for regular day-to-day use. It should work consistently.", "expected_characteristics": [ "IRREVERSIBLE_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "IRREVERSIBLE_ACTION" ], "missing_characteristics": [ "PRODUCTION_MODIFICATION" ], "additional_characteristics": [ "CONSEQUENTIAL_ACTION" ], "expected_packs": [ "AUT-002", "CHG-001", "REL-004" ], "matched_packs": [ "AUT-001", "AUT-002", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00424", "persona": "Non-technical founder", "archetype_id": "automated_refund", "risk": "risk_bearing", "query": "I want a bot that sends eligible customers their money back for a new service. This is the main feature.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "TOOL_ACCESS" ], "missing_characteristics": [ "FINANCIAL_ACTION" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00431", "persona": "Independent developer", "archetype_id": "webhook", "risk": "risk_bearing", "query": "Build incoming webhooks from an external billing provider for a small team. It should work consistently.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "UNTRUSTED_INPUT" ], "detected_characteristics": [ "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION" ], "missing_characteristics": [ "UNTRUSTED_INPUT" ], "additional_characteristics": [ "FINANCIAL_ACTION" ], "expected_packs": [ "INP-001", "INP-002", "OBS-003", "REL-001" ], "matched_packs": [ "AUT-001", "OBS-003", "REL-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00440", "persona": "Nonprofit program lead", "archetype_id": "automated_refund", "risk": "risk_bearing", "query": "Our organization needs automatic refund processing without manual entry for a new service. People should understand what happened.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00444", "persona": "Student or learner", "archetype_id": "automated_refund", "risk": "risk_bearing", "query": "For a learning project, make a workflow that calls a billing tool to approve refunds for a new service. Keep the experience simple.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00446", "persona": "Enterprise product manager", "archetype_id": "card_payment", "risk": "risk_bearing", "query": "Our next feature is subscription billing that collects payments as the main workflow. It should work consistently.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION" ], "detected_characteristics": [ "FINANCIAL_ACTION" ], "missing_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003", "REL-001" ], "matched_packs": [ "AUT-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00450", "persona": "Data engineer", "archetype_id": "database_migration", "risk": "risk_bearing", "query": "Implement a live data migration that rewrites existing rows for a new service. People should understand what happened.", "expected_characteristics": [ "IRREVERSIBLE_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "IRREVERSIBLE_ACTION" ], "missing_characteristics": [ "PRODUCTION_MODIFICATION" ], "additional_characteristics": [ "CONSEQUENTIAL_ACTION" ], "expected_packs": [ "AUT-002", "CHG-001", "REL-004" ], "matched_packs": [ "AUT-001", "AUT-002", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00451", "persona": "Operations professional", "archetype_id": "scheduled_automation", "risk": "risk_bearing", "query": "We need an unattended batch job that processes pending cases for regular day-to-day use. This is the main feature.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "HIGH_COST_CONSUMPTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [ "HIGH_COST_CONSUMPTION" ], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-003", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00460", "persona": "Enterprise product manager", "archetype_id": "deployment_agent", "risk": "risk_bearing", "query": "Our next feature is automatic production releases with infrastructure updates for a new service. This is the main feature.", "expected_characteristics": [ "PRIVILEGED_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "PRODUCTION_MODIFICATION" ], "missing_characteristics": [ "PRIVILEGED_ACTION" ], "additional_characteristics": [], "expected_packs": [ "CHG-001", "IDN-004", "IDN-005", "OBS-002", "REL-004" ], "matched_packs": [ "CHG-001", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00462", "persona": "Small-business owner", "archetype_id": "deployment_agent", "risk": "risk_bearing", "query": "Could you make automation that applies configuration changes in production for a new service. People should understand what happened.", "expected_characteristics": [ "PRIVILEGED_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "PRIVILEGED_ACTION" ], "missing_characteristics": [ "PRODUCTION_MODIFICATION" ], "additional_characteristics": [ "CONSEQUENTIAL_ACTION" ], "expected_packs": [ "CHG-001", "IDN-004", "IDN-005", "OBS-002", "REL-004" ], "matched_packs": [ "AUT-001", "AUT-004", "IDN-004", "IDN-005", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00463", "persona": "Nonprofit program lead", "archetype_id": "scheduled_automation", "risk": "risk_bearing", "query": "Our organization needs a background workflow that updates records every morning for regular day-to-day use. The first version can stay focused.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00469", "persona": "Government analyst", "archetype_id": "database_migration", "risk": "risk_bearing", "query": "The proposed service includes a live data migration that rewrites existing rows inside an existing application. People should understand what happened.", "expected_characteristics": [ "IRREVERSIBLE_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "IRREVERSIBLE_ACTION" ], "missing_characteristics": [ "PRODUCTION_MODIFICATION" ], "additional_characteristics": [ "CONSEQUENTIAL_ACTION" ], "expected_packs": [ "AUT-002", "CHG-001", "REL-004" ], "matched_packs": [ "AUT-001", "AUT-002", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00471", "persona": "Media producer", "archetype_id": "api_crawler", "risk": "risk_bearing", "query": "My project needs an integration that may generate substantial API usage and cost for regular day-to-day use. This is the main feature.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "HIGH_COST_CONSUMPTION" ], "detected_characteristics": [ "HIGH_COST_CONSUMPTION" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-003", "OBS-003", "REL-001", "REL-003" ], "matched_packs": [ "AUT-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00497", "persona": "Site reliability engineer", "archetype_id": "external_sync", "risk": "risk_bearing", "query": "We are planning customer-data synchronization through an external API for a new service. The first version can stay focused.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "SENSITIVE_DATA" ], "detected_characteristics": [ "EXTERNAL_DEPENDENCY" ], "missing_characteristics": [ "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "OBS-003", "REL-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00501", "persona": "Media producer", "archetype_id": "deployment_agent", "risk": "risk_bearing", "query": "My project needs automatic production releases with infrastructure updates inside an existing application. People should understand what happened.", "expected_characteristics": [ "PRIVILEGED_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "PRODUCTION_MODIFICATION" ], "missing_characteristics": [ "PRIVILEGED_ACTION" ], "additional_characteristics": [], "expected_packs": [ "CHG-001", "IDN-004", "IDN-005", "OBS-002", "REL-004" ], "matched_packs": [ "CHG-001", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00515", "persona": "Nonprofit program lead", "archetype_id": "database_migration", "risk": "risk_bearing", "query": "Our organization needs schema migrations applied directly to the live database as the main workflow. Keep the experience simple.", "expected_characteristics": [ "IRREVERSIBLE_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "IRREVERSIBLE_ACTION" ], "missing_characteristics": [ "PRODUCTION_MODIFICATION" ], "additional_characteristics": [ "CONSEQUENTIAL_ACTION" ], "expected_packs": [ "AUT-002", "CHG-001", "REL-004" ], "matched_packs": [ "AUT-001", "AUT-002", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00516", "persona": "Site reliability engineer", "archetype_id": "payroll", "risk": "risk_bearing", "query": "We are planning payroll processing that sends employee pay to a provider for a new service. This is the main feature.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "detected_characteristics": [ "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00518", "persona": "Security lead", "archetype_id": "scheduled_automation", "risk": "risk_bearing", "query": "Review a design for a nightly job that reconciles and updates invoices for a small team. Keep the experience simple.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [ "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "DAT-001", "DAT-005", "IDN-005", "OBS-001", "OBS-003", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00525", "persona": "Machine-learning engineer", "archetype_id": "deployment_agent", "risk": "risk_bearing", "query": "Create automatic production releases with infrastructure updates for a small team. This is the main feature.", "expected_characteristics": [ "PRIVILEGED_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "PRODUCTION_MODIFICATION" ], "missing_characteristics": [ "PRIVILEGED_ACTION" ], "additional_characteristics": [], "expected_packs": [ "CHG-001", "IDN-004", "IDN-005", "OBS-002", "REL-004" ], "matched_packs": [ "CHG-001", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00532", "persona": "Enterprise product manager", "archetype_id": "database_migration", "risk": "risk_bearing", "query": "Our next feature is schema migrations applied directly to the live database as the main workflow. The first version can stay focused.", "expected_characteristics": [ "IRREVERSIBLE_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "IRREVERSIBLE_ACTION" ], "missing_characteristics": [ "PRODUCTION_MODIFICATION" ], "additional_characteristics": [ "CONSEQUENTIAL_ACTION" ], "expected_packs": [ "AUT-002", "CHG-001", "REL-004" ], "matched_packs": [ "AUT-001", "AUT-002", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00534", "persona": "Educator", "archetype_id": "scheduled_automation", "risk": "risk_bearing", "query": "I would like scheduled automation that changes order statuses for a small team. Keep the experience simple.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00537", "persona": "Site reliability engineer", "archetype_id": "automated_refund", "risk": "risk_bearing", "query": "We are planning a workflow that calls a billing tool to approve refunds for a new service. People should understand what happened.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00538", "persona": "Media producer", "archetype_id": "card_payment", "risk": "risk_bearing", "query": "My project needs online orders that charge a customer's card as the main workflow. People should understand what happened.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION" ], "detected_characteristics": [ "FINANCIAL_ACTION" ], "missing_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003", "REL-001" ], "matched_packs": [ "AUT-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00542", "persona": "Marketplace operator", "archetype_id": "card_payment", "risk": "risk_bearing", "query": "We are adding a checkout page that sends charges to a card gateway for a small team. This is the main feature.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION" ], "detected_characteristics": [ "FINANCIAL_ACTION" ], "missing_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003", "REL-001" ], "matched_packs": [ "AUT-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00545", "persona": "Independent developer", "archetype_id": "payroll", "risk": "risk_bearing", "query": "Build payroll processing that sends employee pay to a provider inside an existing application. People should understand what happened.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "detected_characteristics": [ "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00548", "persona": "Nonprofit program lead", "archetype_id": "external_sync", "risk": "risk_bearing", "query": "Our organization needs customer-data synchronization through an external API for a small team. Keep the experience simple.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "SENSITIVE_DATA" ], "detected_characteristics": [ "EXTERNAL_DEPENDENCY" ], "missing_characteristics": [ "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "OBS-003", "REL-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00554", "persona": "Operations professional", "archetype_id": "external_sync", "risk": "risk_bearing", "query": "We need a connection that sends private records to a third-party service inside an existing application. Keep the experience simple.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "SENSITIVE_DATA" ], "detected_characteristics": [ "EXTERNAL_DEPENDENCY" ], "missing_characteristics": [ "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "OBS-003", "REL-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00559", "persona": "Government analyst", "archetype_id": "account_login", "risk": "risk_bearing", "query": "The proposed service includes registration and login using customer credentials inside an existing application. People should understand what happened.", "expected_characteristics": [ "SENSITIVE_DATA", "UNTRUSTED_INPUT" ], "detected_characteristics": [ "UNTRUSTED_INPUT" ], "missing_characteristics": [ "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "INP-001", "INP-002", "REL-004" ], "matched_packs": [ "INP-001", "INP-002" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00567", "persona": "Student or learner", "archetype_id": "deployment_agent", "risk": "risk_bearing", "query": "For a learning project, make automation that applies configuration changes in production as the main workflow. It should work consistently.", "expected_characteristics": [ "PRIVILEGED_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "PRIVILEGED_ACTION" ], "missing_characteristics": [ "PRODUCTION_MODIFICATION" ], "additional_characteristics": [ "CONSEQUENTIAL_ACTION" ], "expected_packs": [ "CHG-001", "IDN-004", "IDN-005", "OBS-002", "REL-004" ], "matched_packs": [ "AUT-001", "AUT-004", "IDN-004", "IDN-005", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00571", "persona": "Machine-learning engineer", "archetype_id": "external_sync", "risk": "risk_bearing", "query": "Create customer-data synchronization through an external API for a small team. This is the main feature.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "SENSITIVE_DATA" ], "detected_characteristics": [ "EXTERNAL_DEPENDENCY" ], "missing_characteristics": [ "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "OBS-003", "REL-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00575", "persona": "Marketplace operator", "archetype_id": "external_sync", "risk": "risk_bearing", "query": "We are adding daily data sync with an outside business platform for regular day-to-day use. It should work consistently.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "SENSITIVE_DATA" ], "detected_characteristics": [ "EXTERNAL_DEPENDENCY" ], "missing_characteristics": [ "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "OBS-003", "REL-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00578", "persona": "Mobile developer", "archetype_id": "scheduled_automation", "risk": "risk_bearing", "query": "The app needs automatic overnight reconciliation through internal tools for a small team. This is the main feature.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [ "FINANCIAL_ACTION" ], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00580", "persona": "Accounting professional", "archetype_id": "automated_refund", "risk": "risk_bearing", "query": "I need a tool with a service that triggers refunds through an external function for a new service. People should understand what happened.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00581", "persona": "Government analyst", "archetype_id": "database_migration", "risk": "risk_bearing", "query": "The proposed service includes a live data migration that rewrites existing rows for a small team. Keep the experience simple.", "expected_characteristics": [ "IRREVERSIBLE_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "IRREVERSIBLE_ACTION" ], "missing_characteristics": [ "PRODUCTION_MODIFICATION" ], "additional_characteristics": [ "CONSEQUENTIAL_ACTION" ], "expected_packs": [ "AUT-002", "CHG-001", "REL-004" ], "matched_packs": [ "AUT-001", "AUT-002", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00583", "persona": "Financial-technology engineer", "archetype_id": "scheduled_automation", "risk": "risk_bearing", "query": "The system should provide a background workflow that updates records every morning for a new service. People should understand what happened.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00588", "persona": "Human-resources manager", "archetype_id": "external_notifications", "risk": "risk_bearing", "query": "We would use a workflow that sends customer alerts through a messaging API for regular day-to-day use. Keep the experience simple.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003", "REL-001" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00589", "persona": "Government analyst", "archetype_id": "payroll", "risk": "risk_bearing", "query": "The proposed service includes employee compensation records and payroll payments for regular day-to-day use. People should understand what happened.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "detected_characteristics": [ "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00610", "persona": "Marketplace operator", "archetype_id": "admin_audit", "risk": "risk_bearing", "query": "We are adding staff tools for suspending and restoring user accounts for regular day-to-day use. The first version can stay focused.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "PRIVILEGED_ACTION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "IRREVERSIBLE_ACTION" ], "missing_characteristics": [ "PRIVILEGED_ACTION" ], "additional_characteristics": [ "IRREVERSIBLE_ACTION" ], "expected_packs": [ "AUT-001", "AUT-004", "IDN-004", "IDN-005", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-002", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00611", "persona": "Media producer", "archetype_id": "document_upload", "risk": "risk_bearing", "query": "My project needs a portal where people upload confidential PDF forms for a small team. This is the main feature.", "expected_characteristics": [ "SENSITIVE_DATA", "UNTRUSTED_INPUT" ], "detected_characteristics": [ "UNTRUSTED_INPUT" ], "missing_characteristics": [ "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "INP-001", "INP-002", "REL-004" ], "matched_packs": [ "INP-001", "INP-002" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00612", "persona": "Human-resources manager", "archetype_id": "document_upload", "risk": "risk_bearing", "query": "We would use a portal where people upload confidential PDF forms as the main workflow. The first version can stay focused.", "expected_characteristics": [ "SENSITIVE_DATA", "UNTRUSTED_INPUT" ], "detected_characteristics": [ "UNTRUSTED_INPUT" ], "missing_characteristics": [ "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "INP-001", "INP-002", "REL-004" ], "matched_packs": [ "INP-001", "INP-002" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00615", "persona": "Human-resources manager", "archetype_id": "deployment_agent", "risk": "risk_bearing", "query": "We would use a deployment tool that modifies live services for a new service. This is the main feature.", "expected_characteristics": [ "PRIVILEGED_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "PRIVILEGED_ACTION" ], "missing_characteristics": [ "PRODUCTION_MODIFICATION" ], "additional_characteristics": [ "CONSEQUENTIAL_ACTION" ], "expected_packs": [ "CHG-001", "IDN-004", "IDN-005", "OBS-002", "REL-004" ], "matched_packs": [ "AUT-001", "AUT-004", "IDN-004", "IDN-005", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00616", "persona": "Marketplace operator", "archetype_id": "database_migration", "risk": "risk_bearing", "query": "We are adding a live data migration that rewrites existing rows as the main workflow. This is the main feature.", "expected_characteristics": [ "IRREVERSIBLE_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "IRREVERSIBLE_ACTION" ], "missing_characteristics": [ "PRODUCTION_MODIFICATION" ], "additional_characteristics": [ "CONSEQUENTIAL_ACTION" ], "expected_packs": [ "AUT-002", "CHG-001", "REL-004" ], "matched_packs": [ "AUT-001", "AUT-002", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00617", "persona": "Operations professional", "archetype_id": "card_payment", "risk": "risk_bearing", "query": "We need subscription billing that collects payments for regular day-to-day use. This is the main feature.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION" ], "detected_characteristics": [ "FINANCIAL_ACTION" ], "missing_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003", "REL-001" ], "matched_packs": [ "AUT-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00618", "persona": "Data engineer", "archetype_id": "automated_refund", "risk": "risk_bearing", "query": "Implement automatic refund processing without manual entry for regular day-to-day use. People should understand what happened.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00621", "persona": "Non-technical founder", "archetype_id": "card_payment", "risk": "risk_bearing", "query": "I want checkout with card payments through a payment provider for regular day-to-day use. This is the main feature.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION" ], "detected_characteristics": [ "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION" ], "missing_characteristics": [ "CONSEQUENTIAL_ACTION" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003", "REL-001" ], "matched_packs": [ "AUT-001", "OBS-003", "REL-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00624", "persona": "Machine-learning engineer", "archetype_id": "card_payment", "risk": "risk_bearing", "query": "Create a checkout page that sends charges to a card gateway for a new service. It should work consistently.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION" ], "detected_characteristics": [ "FINANCIAL_ACTION" ], "missing_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003", "REL-001" ], "matched_packs": [ "AUT-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00628", "persona": "Operations professional", "archetype_id": "automated_refund", "risk": "risk_bearing", "query": "We need a service that triggers refunds through an external function for regular day-to-day use. It should work consistently.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00632", "persona": "Site reliability engineer", "archetype_id": "webhook", "risk": "risk_bearing", "query": "We are planning outside services posting event payloads to our API as the main workflow. This is the main feature.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "UNTRUSTED_INPUT" ], "detected_characteristics": [ "EXTERNAL_DEPENDENCY" ], "missing_characteristics": [ "UNTRUSTED_INPUT" ], "additional_characteristics": [], "expected_packs": [ "INP-001", "INP-002", "OBS-003", "REL-001" ], "matched_packs": [ "OBS-003", "REL-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00637", "persona": "Educator", "archetype_id": "card_payment", "risk": "risk_bearing", "query": "I would like checkout with card payments through a payment provider inside an existing application. People should understand what happened.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION" ], "detected_characteristics": [ "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION" ], "missing_characteristics": [ "CONSEQUENTIAL_ACTION" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003", "REL-001" ], "matched_packs": [ "AUT-001", "OBS-003", "REL-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00639", "persona": "Media producer", "archetype_id": "payroll", "risk": "risk_bearing", "query": "My project needs a tool that calculates pay and submits it to a payroll vendor for regular day-to-day use. The first version can stay focused.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "detected_characteristics": [ "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00642", "persona": "Government analyst", "archetype_id": "account_login", "risk": "risk_bearing", "query": "The proposed service includes registration and login using customer credentials inside an existing application. It should work consistently.", "expected_characteristics": [ "SENSITIVE_DATA", "UNTRUSTED_INPUT" ], "detected_characteristics": [ "UNTRUSTED_INPUT" ], "missing_characteristics": [ "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "INP-001", "INP-002", "REL-004" ], "matched_packs": [ "INP-001", "INP-002" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00644", "persona": "Educator", "archetype_id": "payroll", "risk": "risk_bearing", "query": "I would like payroll processing that sends employee pay to a provider for regular day-to-day use. Keep the experience simple.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "detected_characteristics": [ "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00652", "persona": "Operations professional", "archetype_id": "data_export", "risk": "risk_bearing", "query": "We need bulk export of private account data inside an existing application. People should understand what happened.", "expected_characteristics": [ "DATA_EXPORT", "SENSITIVE_DATA" ], "detected_characteristics": [ "DATA_EXPORT" ], "missing_characteristics": [ "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-002", "DAT-005", "IDN-005", "REL-004" ], "matched_packs": [ "DAT-001", "DAT-002" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00654", "persona": "Financial-technology engineer", "archetype_id": "deployment_agent", "risk": "risk_bearing", "query": "The system should provide a deployment tool that modifies live services for a small team. It should work consistently.", "expected_characteristics": [ "PRIVILEGED_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "PRIVILEGED_ACTION" ], "missing_characteristics": [ "PRODUCTION_MODIFICATION" ], "additional_characteristics": [ "CONSEQUENTIAL_ACTION" ], "expected_packs": [ "CHG-001", "IDN-004", "IDN-005", "OBS-002", "REL-004" ], "matched_packs": [ "AUT-001", "AUT-004", "IDN-004", "IDN-005", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00657", "persona": "Healthcare administrator", "archetype_id": "payroll", "risk": "risk_bearing", "query": "Please help us set up employee compensation records and payroll payments as the main workflow. This is the main feature.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "detected_characteristics": [ "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00660", "persona": "Non-technical founder", "archetype_id": "scheduled_automation", "risk": "risk_bearing", "query": "I want scheduled automation that changes order statuses as the main workflow. This is the main feature.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00664", "persona": "Human-resources manager", "archetype_id": "payroll", "risk": "risk_bearing", "query": "We would use a salary system connected to an outside payroll service as the main workflow. The first version can stay focused.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "detected_characteristics": [ "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00666", "persona": "Operations professional", "archetype_id": "webhook", "risk": "risk_bearing", "query": "We need outside services posting event payloads to our API for a new service. People should understand what happened.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "UNTRUSTED_INPUT" ], "detected_characteristics": [ "EXTERNAL_DEPENDENCY" ], "missing_characteristics": [ "UNTRUSTED_INPUT" ], "additional_characteristics": [], "expected_packs": [ "INP-001", "INP-002", "OBS-003", "REL-001" ], "matched_packs": [ "OBS-003", "REL-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00668", "persona": "Human-resources manager", "archetype_id": "automated_refund", "risk": "risk_bearing", "query": "We would use a bot that sends eligible customers their money back for regular day-to-day use. The first version can stay focused.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "TOOL_ACCESS" ], "missing_characteristics": [ "FINANCIAL_ACTION" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00669", "persona": "Marketplace operator", "archetype_id": "account_login", "risk": "risk_bearing", "query": "We are adding registration and login using customer credentials as the main workflow. This is the main feature.", "expected_characteristics": [ "SENSITIVE_DATA", "UNTRUSTED_INPUT" ], "detected_characteristics": [ "UNTRUSTED_INPUT" ], "missing_characteristics": [ "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "INP-001", "INP-002", "REL-004" ], "matched_packs": [ "INP-001", "INP-002" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00677", "persona": "Independent developer", "archetype_id": "deployment_agent", "risk": "risk_bearing", "query": "Build a deployment tool that modifies live services for a small team. The first version can stay focused.", "expected_characteristics": [ "PRIVILEGED_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "PRIVILEGED_ACTION" ], "missing_characteristics": [ "PRODUCTION_MODIFICATION" ], "additional_characteristics": [ "CONSEQUENTIAL_ACTION" ], "expected_packs": [ "CHG-001", "IDN-004", "IDN-005", "OBS-002", "REL-004" ], "matched_packs": [ "AUT-001", "AUT-004", "IDN-004", "IDN-005", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00686", "persona": "Independent developer", "archetype_id": "ai_sensitive_summary", "risk": "risk_bearing", "query": "Build automatic model-generated summaries of internal investigations for regular day-to-day use. The first version can stay focused.", "expected_characteristics": [ "AI_CONTROLLED", "SENSITIVE_DATA" ], "detected_characteristics": [ "AI_CONTROLLED" ], "missing_characteristics": [ "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "AI-004", "DAT-001", "DAT-005", "IDN-005", "REL-004" ], "matched_packs": [ "AI-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00687", "persona": "Accounting professional", "archetype_id": "scheduled_automation", "risk": "risk_bearing", "query": "I need a tool with an unattended batch job that processes pending cases as the main workflow. It should work consistently.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "HIGH_COST_CONSUMPTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [ "HIGH_COST_CONSUMPTION" ], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-003", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00688", "persona": "Mobile developer", "archetype_id": "payroll", "risk": "risk_bearing", "query": "The app needs a salary system connected to an outside payroll service as the main workflow. Keep the experience simple.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "detected_characteristics": [ "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00690", "persona": "Independent developer", "archetype_id": "webhook", "risk": "risk_bearing", "query": "Build outside services posting event payloads to our API for regular day-to-day use. People should understand what happened.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "UNTRUSTED_INPUT" ], "detected_characteristics": [ "EXTERNAL_DEPENDENCY" ], "missing_characteristics": [ "UNTRUSTED_INPUT" ], "additional_characteristics": [], "expected_packs": [ "INP-001", "INP-002", "OBS-003", "REL-001" ], "matched_packs": [ "OBS-003", "REL-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00694", "persona": "Government analyst", "archetype_id": "external_sync", "risk": "risk_bearing", "query": "The proposed service includes a connection that sends private records to a third-party service as the main workflow. Keep the experience simple.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "SENSITIVE_DATA" ], "detected_characteristics": [ "EXTERNAL_DEPENDENCY" ], "missing_characteristics": [ "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "OBS-003", "REL-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00696", "persona": "Media producer", "archetype_id": "payroll", "risk": "risk_bearing", "query": "My project needs employee compensation records and payroll payments as the main workflow. Keep the experience simple.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "detected_characteristics": [ "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00697", "persona": "Operations professional", "archetype_id": "admin_audit", "risk": "risk_bearing", "query": "We need staff tools for suspending and restoring user accounts for a new service. The first version can stay focused.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "PRIVILEGED_ACTION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "IRREVERSIBLE_ACTION" ], "missing_characteristics": [ "PRIVILEGED_ACTION" ], "additional_characteristics": [ "IRREVERSIBLE_ACTION" ], "expected_packs": [ "AUT-001", "AUT-004", "IDN-004", "IDN-005", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-002", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00698", "persona": "Human-resources manager", "archetype_id": "card_payment", "risk": "risk_bearing", "query": "We would use a purchase flow connected to an external payment processor inside an existing application. It should work consistently.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION" ], "detected_characteristics": [ "EXTERNAL_DEPENDENCY" ], "missing_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003", "REL-001" ], "matched_packs": [ "OBS-003", "REL-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00710", "persona": "Marketplace operator", "archetype_id": "database_migration", "risk": "risk_bearing", "query": "We are adding a live data migration that rewrites existing rows for a small team. Keep the experience simple.", "expected_characteristics": [ "IRREVERSIBLE_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "IRREVERSIBLE_ACTION" ], "missing_characteristics": [ "PRODUCTION_MODIFICATION" ], "additional_characteristics": [ "CONSEQUENTIAL_ACTION" ], "expected_packs": [ "AUT-002", "CHG-001", "REL-004" ], "matched_packs": [ "AUT-001", "AUT-002", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00714", "persona": "Independent developer", "archetype_id": "scheduled_automation", "risk": "risk_bearing", "query": "Build an unattended batch job that processes pending cases for regular day-to-day use. The first version can stay focused.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "HIGH_COST_CONSUMPTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [ "HIGH_COST_CONSUMPTION" ], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-003", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00715", "persona": "Operations professional", "archetype_id": "external_sync", "risk": "risk_bearing", "query": "We need a connection that sends private records to a third-party service as the main workflow. This is the main feature.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "SENSITIVE_DATA" ], "detected_characteristics": [ "EXTERNAL_DEPENDENCY" ], "missing_characteristics": [ "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "OBS-003", "REL-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00719", "persona": "Non-technical founder", "archetype_id": "data_export", "risk": "risk_bearing", "query": "I want a feature that downloads employee information as the main workflow. People should understand what happened.", "expected_characteristics": [ "DATA_EXPORT", "SENSITIVE_DATA" ], "detected_characteristics": [ "DATA_EXPORT" ], "missing_characteristics": [ "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-002", "DAT-005", "IDN-005", "REL-004" ], "matched_packs": [ "DAT-001", "DAT-002" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00720", "persona": "Educator", "archetype_id": "automated_refund", "risk": "risk_bearing", "query": "I would like a service that triggers refunds through an external function for regular day-to-day use. It should work consistently.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00722", "persona": "Human-resources manager", "archetype_id": "admin_audit", "risk": "risk_bearing", "query": "We would use staff tools for suspending and restoring user accounts for a small team. People should understand what happened.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "PRIVILEGED_ACTION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "IRREVERSIBLE_ACTION" ], "missing_characteristics": [ "PRIVILEGED_ACTION" ], "additional_characteristics": [ "IRREVERSIBLE_ACTION" ], "expected_packs": [ "AUT-001", "AUT-004", "IDN-004", "IDN-005", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-002", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00726", "persona": "Enterprise product manager", "archetype_id": "api_crawler", "risk": "risk_bearing", "query": "Our next feature is bulk calls to a paid third-party data service inside an existing application. Keep the experience simple.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "HIGH_COST_CONSUMPTION" ], "detected_characteristics": [ "EXTERNAL_DEPENDENCY" ], "missing_characteristics": [ "HIGH_COST_CONSUMPTION" ], "additional_characteristics": [], "expected_packs": [ "AUT-003", "OBS-003", "REL-001", "REL-003" ], "matched_packs": [ "OBS-003", "REL-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00731", "persona": "Machine-learning engineer", "archetype_id": "webhook", "risk": "risk_bearing", "query": "Create incoming webhooks from an external billing provider inside an existing application. The first version can stay focused.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "UNTRUSTED_INPUT" ], "detected_characteristics": [ "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION" ], "missing_characteristics": [ "UNTRUSTED_INPUT" ], "additional_characteristics": [ "FINANCIAL_ACTION" ], "expected_packs": [ "INP-001", "INP-002", "OBS-003", "REL-001" ], "matched_packs": [ "AUT-001", "OBS-003", "REL-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00740", "persona": "Small-business owner", "archetype_id": "account_login", "risk": "risk_bearing", "query": "Could you make registration and login using customer credentials for regular day-to-day use. Keep the experience simple.", "expected_characteristics": [ "SENSITIVE_DATA", "UNTRUSTED_INPUT" ], "detected_characteristics": [ "UNTRUSTED_INPUT" ], "missing_characteristics": [ "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "INP-001", "INP-002", "REL-004" ], "matched_packs": [ "INP-001", "INP-002" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00742", "persona": "Accounting professional", "archetype_id": "deployment_agent", "risk": "risk_bearing", "query": "I need a tool with automation that applies configuration changes in production for a new service. The first version can stay focused.", "expected_characteristics": [ "PRIVILEGED_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "PRIVILEGED_ACTION" ], "missing_characteristics": [ "PRODUCTION_MODIFICATION" ], "additional_characteristics": [ "CONSEQUENTIAL_ACTION" ], "expected_packs": [ "CHG-001", "IDN-004", "IDN-005", "OBS-002", "REL-004" ], "matched_packs": [ "AUT-001", "AUT-004", "IDN-004", "IDN-005", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00744", "persona": "Small-business owner", "archetype_id": "deployment_agent", "risk": "risk_bearing", "query": "Could you make a deployment tool that modifies live services for a new service. This is the main feature.", "expected_characteristics": [ "PRIVILEGED_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "PRIVILEGED_ACTION" ], "missing_characteristics": [ "PRODUCTION_MODIFICATION" ], "additional_characteristics": [ "CONSEQUENTIAL_ACTION" ], "expected_packs": [ "CHG-001", "IDN-004", "IDN-005", "OBS-002", "REL-004" ], "matched_packs": [ "AUT-001", "AUT-004", "IDN-004", "IDN-005", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] } ], "failure_example_limit": 200, "limitations": [ "All requests are synthetic and are not evidence of users, adoption, market demand, or real-world accuracy.", "Expected characteristics were authored at the scenario-archetype level by the project and have not been independently adjudicated.", "Expected Control Packs are derived by applying the published pack rules to the authored expected characteristics; this tests detection-to-pack continuity, not whether the pack library is normatively complete.", "Generated wording variants can expose brittleness, but repeated variants are not statistically independent observations.", "Results should be interpreted alongside the smaller hand-authored reference benchmarks, not as a replacement for them." ], "interpretation": "A large synthetic stress test of deterministic detection and pack continuity. It is not an adoption count, representative user sample, independent safety evaluation, or real-world accuracy estimate." }