{ "evaluation_id": "large-query-evaluation-v1", "evaluation_version": "1.0.0", "methodology_version": "0.1.0", "generated_at": "2026-08-14T15:58:49Z", "seed": 20260814, "config_digest": "sha256:680bc91a7685de3838842e2213522956ecba04f939b2a19d4d3b205bc8146309", "generated_corpus_digest": "sha256:01c78779f39e064ff7f6beaf4a229ec8a39f110e41685ab059eb82fbe8f008c7", "query_count": 25000, "unique_query_count": 25000, "risk_query_count": 19914, "low_risk_query_count": 5086, "persona_count": 20, "archetype_count": 30, "runtime": { "seconds": 40.748, "queries_per_second": 613.5 }, "headline_metrics": { "risk_query_pack_hit_rate": 0.707241, "expected_characteristic_micro_recall": 0.438028, "complete_expected_characteristic_rate": 0.184343, "expected_pack_continuity_rate": 0.640052, "low_risk_false_positive_rate": 0.20409 }, "failure_counts": { "expected_characteristic_missing": 16243, "expected_pack_family_not_reached": 7168, "low_risk_false_positive": 1038, "risk_query_returned_no_pack": 5830 }, "characteristics": [ { "id": "AI_CONTROLLED", "expected_count": 2446, "detected_when_expected": 1637, "recall": 0.669256, "total_detected_count": 1637 }, { "id": "CONSEQUENTIAL_ACTION", "expected_count": 6621, "detected_when_expected": 3822, "recall": 0.577254, "total_detected_count": 5621 }, { "id": "DATA_EXPORT", "expected_count": 809, "detected_when_expected": 0, "recall": 0.0, "total_detected_count": 0 }, { "id": "EXTERNAL_DEPENDENCY", "expected_count": 4992, "detected_when_expected": 1206, "recall": 0.241587, "total_detected_count": 1206 }, { "id": "FINANCIAL_ACTION", "expected_count": 2515, "detected_when_expected": 2197, "recall": 0.873559, "total_detected_count": 2680 }, { "id": "HIGH_COST_CONSUMPTION", "expected_count": 1625, "detected_when_expected": 662, "recall": 0.407385, "total_detected_count": 830 }, { "id": "IRREVERSIBLE_ACTION", "expected_count": 1686, "detected_when_expected": 852, "recall": 0.505338, "total_detected_count": 1028 }, { "id": "MULTI_TENANT", "expected_count": 853, "detected_when_expected": 324, "recall": 0.379836, "total_detected_count": 5122 }, { "id": "PRIVILEGED_ACTION", "expected_count": 2463, "detected_when_expected": 809, "recall": 0.328461, "total_detected_count": 1342 }, { "id": "PRODUCTION_MODIFICATION", "expected_count": 1649, "detected_when_expected": 1022, "recall": 0.61977, "total_detected_count": 1022 }, { "id": "SENSITIVE_DATA", "expected_count": 5881, "detected_when_expected": 3057, "recall": 0.51981, "total_detected_count": 7108 }, { "id": "TOOL_ACCESS", "expected_count": 1636, "detected_when_expected": 150, "recall": 0.091687, "total_detected_count": 479 }, { "id": "UNTRUSTED_INPUT", "expected_count": 5075, "detected_when_expected": 1017, "recall": 0.200394, "total_detected_count": 1350 } ], "most_frequently_matched_packs": [ { "pack_id": "IDN-005", "count": 8171, "rate": 0.32684 }, { "pack_id": "REL-004", "count": 7913, "rate": 0.31652 }, { "pack_id": "AUT-001", "count": 7295, "rate": 0.2918 }, { "pack_id": "DAT-005", "count": 7108, "rate": 0.28432 }, { "pack_id": "DAT-001", "count": 7108, "rate": 0.28432 }, { "pack_id": "OBS-003", "count": 6655, "rate": 0.2662 }, { "pack_id": "AUT-004", "count": 5950, "rate": 0.238 }, { "pack_id": "OBS-001", "count": 5621, "rate": 0.22484 }, { "pack_id": "DAT-003", "count": 5122, "rate": 0.20488 }, { "pack_id": "AI-004", "count": 1637, "rate": 0.06548 }, { "pack_id": "INP-001", "count": 1350, "rate": 0.054 }, { "pack_id": "INP-002", "count": 1350, "rate": 0.054 }, { "pack_id": "IDN-004", "count": 1342, "rate": 0.05368 }, { "pack_id": "REL-001", "count": 1206, "rate": 0.04824 }, { "pack_id": "AUT-002", "count": 1028, "rate": 0.04112 }, { "pack_id": "CHG-001", "count": 1022, "rate": 0.04088 }, { "pack_id": "AUT-003", "count": 830, "rate": 0.0332 }, { "pack_id": "AI-001", "count": 653, "rate": 0.02612 }, { "pack_id": "OBS-002", "count": 533, "rate": 0.02132 }, { "pack_id": "DAT-004", "count": 198, "rate": 0.00792 }, { "pack_id": "REL-003", "count": 164, "rate": 0.00656 }, { "pack_id": "IDN-001", "count": 150, "rate": 0.006 }, { "pack_id": "IDN-003", "count": 90, "rate": 0.0036 } ], "personas": [ { "id": "accountant", "label": "Accounting professional", "query_count": 1256, "risk_query_count": 1007, "low_risk_query_count": 249, "risk_pack_hit_rate": 0.720953, "complete_expected_characteristic_rate": 0.180735, "expected_pack_continuity_rate": 0.653426, "low_risk_false_positive_rate": 0.184739 }, { "id": "data_engineer", "label": "Data engineer", "query_count": 1264, "risk_query_count": 995, "low_risk_query_count": 269, "risk_pack_hit_rate": 0.725628, "complete_expected_characteristic_rate": 0.188945, "expected_pack_continuity_rate": 0.653266, "low_risk_false_positive_rate": 0.211896 }, { "id": "educator", "label": "Educator", "query_count": 1228, "risk_query_count": 972, "low_risk_query_count": 256, "risk_pack_hit_rate": 0.711934, "complete_expected_characteristic_rate": 0.174897, "expected_pack_continuity_rate": 0.638889, "low_risk_false_positive_rate": 0.222656 }, { "id": "enterprise_product", "label": "Enterprise product manager", "query_count": 1265, "risk_query_count": 1002, "low_risk_query_count": 263, "risk_pack_hit_rate": 0.697605, "complete_expected_characteristic_rate": 0.181637, "expected_pack_continuity_rate": 0.631737, "low_risk_false_positive_rate": 0.174905 }, { "id": "fintech_engineer", "label": "Financial-technology engineer", "query_count": 1270, "risk_query_count": 1001, "low_risk_query_count": 269, "risk_pack_hit_rate": 0.699301, "complete_expected_characteristic_rate": 0.190809, "expected_pack_continuity_rate": 0.632368, "low_risk_false_positive_rate": 0.223048 }, { "id": "government_analyst", "label": "Government analyst", "query_count": 1247, "risk_query_count": 987, "low_risk_query_count": 260, "risk_pack_hit_rate": 0.701114, "complete_expected_characteristic_rate": 0.181358, "expected_pack_continuity_rate": 0.640324, "low_risk_false_positive_rate": 0.207692 }, { "id": "healthcare_admin", "label": "Healthcare administrator", "query_count": 1277, "risk_query_count": 1031, "low_risk_query_count": 246, "risk_pack_hit_rate": 0.700291, "complete_expected_characteristic_rate": 0.185257, "expected_pack_continuity_rate": 0.640155, "low_risk_false_positive_rate": 0.195122 }, { "id": "hr_manager", "label": "Human-resources manager", "query_count": 1310, "risk_query_count": 1066, "low_risk_query_count": 244, "risk_pack_hit_rate": 0.704503, "complete_expected_characteristic_rate": 0.172608, "expected_pack_continuity_rate": 0.633208, "low_risk_false_positive_rate": 0.180328 }, { "id": "indie_developer", "label": "Independent developer", "query_count": 1266, "risk_query_count": 1000, "low_risk_query_count": 266, "risk_pack_hit_rate": 0.719, "complete_expected_characteristic_rate": 0.201, "expected_pack_continuity_rate": 0.647, "low_risk_false_positive_rate": 0.221805 }, { "id": "marketplace_operator", "label": "Marketplace operator", "query_count": 1229, "risk_query_count": 984, "low_risk_query_count": 245, "risk_pack_hit_rate": 0.703252, "complete_expected_characteristic_rate": 0.191057, "expected_pack_continuity_rate": 0.637195, "low_risk_false_positive_rate": 0.220408 }, { "id": "media_producer", "label": "Media producer", "query_count": 1285, "risk_query_count": 1022, "low_risk_query_count": 263, "risk_pack_hit_rate": 0.700587, "complete_expected_characteristic_rate": 0.180039, "expected_pack_continuity_rate": 0.627202, "low_risk_false_positive_rate": 0.209125 }, { "id": "ml_engineer", "label": "Machine-learning engineer", "query_count": 1267, "risk_query_count": 996, "low_risk_query_count": 271, "risk_pack_hit_rate": 0.708835, "complete_expected_characteristic_rate": 0.197791, "expected_pack_continuity_rate": 0.648594, "low_risk_false_positive_rate": 0.210332 }, { "id": "mobile_developer", "label": "Mobile developer", "query_count": 1229, "risk_query_count": 976, "low_risk_query_count": 253, "risk_pack_hit_rate": 0.714139, "complete_expected_characteristic_rate": 0.181352, "expected_pack_continuity_rate": 0.655738, "low_risk_false_positive_rate": 0.162055 }, { "id": "nonprofit_lead", "label": "Nonprofit program lead", "query_count": 1234, "risk_query_count": 996, "low_risk_query_count": 238, "risk_pack_hit_rate": 0.700803, "complete_expected_characteristic_rate": 0.182731, "expected_pack_continuity_rate": 0.640562, "low_risk_false_positive_rate": 0.218487 }, { "id": "nontechnical_founder", "label": "Non-technical founder", "query_count": 1241, "risk_query_count": 995, "low_risk_query_count": 246, "risk_pack_hit_rate": 0.703518, "complete_expected_characteristic_rate": 0.192965, "expected_pack_continuity_rate": 0.640201, "low_risk_false_positive_rate": 0.235772 }, { "id": "operations", "label": "Operations professional", "query_count": 1218, "risk_query_count": 973, "low_risk_query_count": 245, "risk_pack_hit_rate": 0.706064, "complete_expected_characteristic_rate": 0.1963, "expected_pack_continuity_rate": 0.636177, "low_risk_false_positive_rate": 0.232653 }, { "id": "security_lead", "label": "Security lead", "query_count": 1249, "risk_query_count": 998, "low_risk_query_count": 251, "risk_pack_hit_rate": 0.703407, "complete_expected_characteristic_rate": 0.182365, "expected_pack_continuity_rate": 0.634269, "low_risk_false_positive_rate": 0.219124 }, { "id": "small_business_owner", "label": "Small-business owner", "query_count": 1188, "risk_query_count": 931, "low_risk_query_count": 257, "risk_pack_hit_rate": 0.701396, "complete_expected_characteristic_rate": 0.167562, "expected_pack_continuity_rate": 0.630505, "low_risk_false_positive_rate": 0.171206 }, { "id": "sre", "label": "Site reliability engineer", "query_count": 1226, "risk_query_count": 979, "low_risk_query_count": 247, "risk_pack_hit_rate": 0.712972, "complete_expected_characteristic_rate": 0.181818, "expected_pack_continuity_rate": 0.637385, "low_risk_false_positive_rate": 0.17004 }, { "id": "student", "label": "Student or learner", "query_count": 1251, "risk_query_count": 1003, "low_risk_query_count": 248, "risk_pack_hit_rate": 0.70987, "complete_expected_characteristic_rate": 0.175474, "expected_pack_continuity_rate": 0.643071, "low_risk_false_positive_rate": 0.209677 } ], "archetypes": [ { "id": "account_login", "label": "account_login", "query_count": 889, "risk_query_count": 889, "low_risk_query_count": 0, "risk_pack_hit_rate": 0.526434, "complete_expected_characteristic_rate": 0.0, "expected_pack_continuity_rate": 0.526434, "low_risk_false_positive_rate": 0.0, "distinct_detection_signatures": 3, "dominant_signature_rate": 0.473566 }, { "id": "admin_audit", "label": "admin_audit", "query_count": 847, "risk_query_count": 847, "low_risk_query_count": 0, "risk_pack_hit_rate": 0.68595, "complete_expected_characteristic_rate": 0.391972, "expected_pack_continuity_rate": 0.68595, "low_risk_false_positive_rate": 0.0, "distinct_detection_signatures": 6, "dominant_signature_rate": 0.31405 }, { "id": "ai_decision", "label": "ai_decision", "query_count": 800, "risk_query_count": 800, "low_risk_query_count": 0, "risk_pack_hit_rate": 0.86, "complete_expected_characteristic_rate": 0.0, "expected_pack_continuity_rate": 0.825, "low_risk_false_positive_rate": 0.0, "distinct_detection_signatures": 6, "dominant_signature_rate": 0.47625 }, { "id": "ai_sensitive_summary", "label": "ai_sensitive_summary", "query_count": 805, "risk_query_count": 805, "low_risk_query_count": 0, "risk_pack_hit_rate": 0.843478, "complete_expected_characteristic_rate": 0.285714, "expected_pack_continuity_rate": 0.843478, "low_risk_false_positive_rate": 0.0, "distinct_detection_signatures": 8, "dominant_signature_rate": 0.172671 }, { "id": "ai_support", "label": "ai_support", "query_count": 841, "risk_query_count": 841, "low_risk_query_count": 0, "risk_pack_hit_rate": 0.83591, "complete_expected_characteristic_rate": 0.59453, "expected_pack_continuity_rate": 0.791914, "low_risk_false_positive_rate": 0.0, "distinct_detection_signatures": 6, "dominant_signature_rate": 0.487515 }, { "id": "api_crawler", "label": "api_crawler", "query_count": 802, "risk_query_count": 802, "low_risk_query_count": 0, "risk_pack_hit_rate": 0.684539, "complete_expected_characteristic_rate": 0.204489, "expected_pack_continuity_rate": 0.613466, "low_risk_false_positive_rate": 0.0, "distinct_detection_signatures": 6, "dominant_signature_rate": 0.331671 }, { "id": "automated_refund", "label": "automated_refund", "query_count": 845, "risk_query_count": 845, "low_risk_query_count": 0, "risk_pack_hit_rate": 1.0, "complete_expected_characteristic_rate": 0.0, "expected_pack_continuity_rate": 1.0, "low_risk_false_positive_rate": 0.0, "distinct_detection_signatures": 4, "dominant_signature_rate": 0.653254 }, { "id": "bulk_update", "label": "bulk_update", "query_count": 813, "risk_query_count": 813, "low_risk_query_count": 0, "risk_pack_hit_rate": 0.664207, "complete_expected_characteristic_rate": 0.382534, "expected_pack_continuity_rate": 0.382534, "low_risk_false_positive_rate": 0.0, "distinct_detection_signatures": 5, "dominant_signature_rate": 0.335793 }, { "id": "calculator", "label": "calculator", "query_count": 857, "risk_query_count": 0, "low_risk_query_count": 857, "risk_pack_hit_rate": 0.0, "complete_expected_characteristic_rate": 0.0, "expected_pack_continuity_rate": 0.0, "low_risk_false_positive_rate": 0.210035, "distinct_detection_signatures": 2, "dominant_signature_rate": 0.789965 }, { "id": "card_payment", "label": "card_payment", "query_count": 840, "risk_query_count": 840, "low_risk_query_count": 0, "risk_pack_hit_rate": 1.0, "complete_expected_characteristic_rate": 0.0, "expected_pack_continuity_rate": 1.0, "low_risk_false_positive_rate": 0.0, "distinct_detection_signatures": 6, "dominant_signature_rate": 0.480952 }, { "id": "color_palette", "label": "color_palette", "query_count": 853, "risk_query_count": 0, "low_risk_query_count": 853, "risk_pack_hit_rate": 0.0, "complete_expected_characteristic_rate": 0.0, "expected_pack_continuity_rate": 0.0, "low_risk_false_positive_rate": 0.181712, "distinct_detection_signatures": 2, "dominant_signature_rate": 0.818288 }, { "id": "countdown", "label": "countdown", "query_count": 839, "risk_query_count": 0, "low_risk_query_count": 839, "risk_pack_hit_rate": 0.0, "complete_expected_characteristic_rate": 0.0, "expected_pack_continuity_rate": 0.0, "low_risk_false_positive_rate": 0.219309, "distinct_detection_signatures": 2, "dominant_signature_rate": 0.780691 }, { "id": "data_export", "label": "data_export", "query_count": 809, "risk_query_count": 809, "low_risk_query_count": 0, "risk_pack_hit_rate": 0.337454, "complete_expected_characteristic_rate": 0.0, "expected_pack_continuity_rate": 0.337454, "low_risk_false_positive_rate": 0.0, "distinct_detection_signatures": 3, "dominant_signature_rate": 0.662546 }, { "id": "database_migration", "label": "database_migration", "query_count": 842, "risk_query_count": 842, "low_risk_query_count": 0, "risk_pack_hit_rate": 1.0, "complete_expected_characteristic_rate": 0.22209, "expected_pack_continuity_rate": 1.0, "low_risk_false_positive_rate": 0.0, "distinct_detection_signatures": 6, "dominant_signature_rate": 0.319477 }, { "id": "deployment_agent", "label": "deployment_agent", "query_count": 807, "risk_query_count": 807, "low_risk_query_count": 0, "risk_pack_hit_rate": 1.0, "complete_expected_characteristic_rate": 0.0, "expected_pack_continuity_rate": 0.687732, "low_risk_false_positive_rate": 0.0, "distinct_detection_signatures": 8, "dominant_signature_rate": 0.312268 }, { "id": "document_upload", "label": "document_upload", "query_count": 857, "risk_query_count": 857, "low_risk_query_count": 0, "risk_pack_hit_rate": 0.523921, "complete_expected_characteristic_rate": 0.035006, "expected_pack_continuity_rate": 0.523921, "low_risk_false_positive_rate": 0.0, "distinct_detection_signatures": 5, "dominant_signature_rate": 0.476079 }, { "id": "expensive_processing", "label": "expensive_processing", "query_count": 823, "risk_query_count": 823, "low_risk_query_count": 0, "risk_pack_hit_rate": 0.358445, "complete_expected_characteristic_rate": 0.206561, "expected_pack_continuity_rate": 0.206561, "low_risk_false_positive_rate": 0.0, "distinct_detection_signatures": 4, "dominant_signature_rate": 0.641555 }, { "id": "external_notifications", "label": "external_notifications", "query_count": 841, "risk_query_count": 841, "low_risk_query_count": 0, "risk_pack_hit_rate": 1.0, "complete_expected_characteristic_rate": 0.204518, "expected_pack_continuity_rate": 1.0, "low_risk_false_positive_rate": 0.0, "distinct_detection_signatures": 4, "dominant_signature_rate": 0.638526 }, { "id": "external_sync", "label": "external_sync", "query_count": 838, "risk_query_count": 838, "low_risk_query_count": 0, "risk_pack_hit_rate": 0.844869, "complete_expected_characteristic_rate": 0.089499, "expected_pack_continuity_rate": 0.844869, "low_risk_false_positive_rate": 0.0, "distinct_detection_signatures": 5, "dominant_signature_rate": 0.332936 }, { "id": "file_converter", "label": "file_converter", "query_count": 778, "risk_query_count": 778, "low_risk_query_count": 0, "risk_pack_hit_rate": 0.186375, "complete_expected_characteristic_rate": 0.0, "expected_pack_continuity_rate": 0.0, "low_risk_false_positive_rate": 0.0, "distinct_detection_signatures": 2, "dominant_signature_rate": 0.813625 }, { "id": "multi_tenant", "label": "multi_tenant", "query_count": 853, "risk_query_count": 853, "low_risk_query_count": 0, "risk_pack_hit_rate": 0.379836, "complete_expected_characteristic_rate": 0.379836, "expected_pack_continuity_rate": 0.379836, "low_risk_false_positive_rate": 0.0, "distinct_detection_signatures": 2, "dominant_signature_rate": 0.620164 }, { "id": "payroll", "label": "payroll", "query_count": 830, "risk_query_count": 830, "low_risk_query_count": 0, "risk_pack_hit_rate": 1.0, "complete_expected_characteristic_rate": 0.0, "expected_pack_continuity_rate": 1.0, "low_risk_false_positive_rate": 0.0, "distinct_detection_signatures": 2, "dominant_signature_rate": 0.804819 }, { "id": "permanent_delete", "label": "permanent_delete", "query_count": 844, "risk_query_count": 844, "low_risk_query_count": 0, "risk_pack_hit_rate": 0.53673, "complete_expected_characteristic_rate": 0.421801, "expected_pack_continuity_rate": 0.421801, "low_risk_false_positive_rate": 0.0, "distinct_detection_signatures": 6, "dominant_signature_rate": 0.46327 }, { "id": "permissions", "label": "permissions", "query_count": 809, "risk_query_count": 809, "low_risk_query_count": 0, "risk_pack_hit_rate": 0.679852, "complete_expected_characteristic_rate": 0.589617, "expected_pack_continuity_rate": 0.679852, "low_risk_false_positive_rate": 0.0, "distinct_detection_signatures": 4, "dominant_signature_rate": 0.469716 }, { "id": "personal_checklist", "label": "personal_checklist", "query_count": 862, "risk_query_count": 0, "low_risk_query_count": 862, "risk_pack_hit_rate": 0.0, "complete_expected_characteristic_rate": 0.0, "expected_pack_continuity_rate": 0.0, "low_risk_false_positive_rate": 0.213457, "distinct_detection_signatures": 2, "dominant_signature_rate": 0.786543 }, { "id": "public_submission", "label": "public_submission", "query_count": 869, "risk_query_count": 869, "low_risk_query_count": 0, "risk_pack_hit_rate": 0.516686, "complete_expected_characteristic_rate": 0.394707, "expected_pack_continuity_rate": 0.394707, "low_risk_false_positive_rate": 0.0, "distinct_detection_signatures": 4, "dominant_signature_rate": 0.483314 }, { "id": "scheduled_automation", "label": "scheduled_automation", "query_count": 791, "risk_query_count": 791, "low_risk_query_count": 0, "risk_pack_hit_rate": 1.0, "complete_expected_characteristic_rate": 0.0, "expected_pack_continuity_rate": 1.0, "low_risk_false_positive_rate": 0.0, "distinct_detection_signatures": 7, "dominant_signature_rate": 0.303413 }, { "id": "static_page", "label": "static_page", "query_count": 833, "risk_query_count": 0, "low_risk_query_count": 833, "risk_pack_hit_rate": 0.0, "complete_expected_characteristic_rate": 0.0, "expected_pack_continuity_rate": 0.0, "low_risk_false_positive_rate": 0.195678, "distinct_detection_signatures": 2, "dominant_signature_rate": 0.804322 }, { "id": "unit_converter", "label": "unit_converter", "query_count": 842, "risk_query_count": 0, "low_risk_query_count": 842, "risk_pack_hit_rate": 0.0, "complete_expected_characteristic_rate": 0.0, "expected_pack_continuity_rate": 0.0, "low_risk_false_positive_rate": 0.204276, "distinct_detection_signatures": 2, "dominant_signature_rate": 0.795724 }, { "id": "webhook", "label": "webhook", "query_count": 841, "risk_query_count": 841, "low_risk_query_count": 0, "risk_pack_hit_rate": 0.516052, "complete_expected_characteristic_rate": 0.0, "expected_pack_continuity_rate": 0.204518, "low_risk_false_positive_rate": 0.0, "distinct_detection_signatures": 6, "dominant_signature_rate": 0.483948 } ], "failure_examples": [ { "query_id": "LQE-00001", "persona": "Media producer", "archetype_id": "admin_audit", "risk": "risk_bearing", "query": "My project needs an administrative console that changes account status inside an existing application. Keep the experience simple.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "PRIVILEGED_ACTION" ], "detected_characteristics": [], "missing_characteristics": [ "CONSEQUENTIAL_ACTION", "PRIVILEGED_ACTION" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "IDN-004", "IDN-005", "OBS-001", "OBS-003" ], "matched_packs": [], "failure_types": [ "expected_characteristic_missing", "risk_query_returned_no_pack", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00003", "persona": "Small-business owner", "archetype_id": "expensive_processing", "risk": "risk_bearing", "query": "Could you make large video-processing jobs that consume substantial compute for regular day-to-day use. People should understand what happened.", "expected_characteristics": [ "HIGH_COST_CONSUMPTION" ], "detected_characteristics": [], "missing_characteristics": [ "HIGH_COST_CONSUMPTION" ], "additional_characteristics": [], "expected_packs": [ "AUT-003" ], "matched_packs": [], "failure_types": [ "expected_characteristic_missing", "risk_query_returned_no_pack", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00005", "persona": "Student or learner", "archetype_id": "scheduled_automation", "risk": "risk_bearing", "query": "For a learning project, make scheduled automation that changes order statuses inside an existing application. It should work consistently.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00006", "persona": "Non-technical founder", "archetype_id": "account_login", "risk": "risk_bearing", "query": "I want a member login that stores account information as the main workflow. The first version can stay focused.", "expected_characteristics": [ "SENSITIVE_DATA", "UNTRUSTED_INPUT" ], "detected_characteristics": [], "missing_characteristics": [ "SENSITIVE_DATA", "UNTRUSTED_INPUT" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "INP-001", "INP-002", "REL-004" ], "matched_packs": [], "failure_types": [ "expected_characteristic_missing", "risk_query_returned_no_pack", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00007", "persona": "Non-technical founder", "archetype_id": "automated_refund", "risk": "risk_bearing", "query": "I want automatic refunds issued through the payment API for a small team. Keep the experience simple.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION", "MULTI_TENANT", "SENSITIVE_DATA" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "DAT-001", "DAT-003", "DAT-005", "IDN-005", "OBS-001", "OBS-003", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00009", "persona": "Accounting professional", "archetype_id": "ai_decision", "risk": "risk_bearing", "query": "I need a tool with a model-assisted workflow that approves or rejects submitted requests for regular day-to-day use. It should work consistently.", "expected_characteristics": [ "AI_CONTROLLED", "CONSEQUENTIAL_ACTION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION" ], "missing_characteristics": [ "AI_CONTROLLED" ], "additional_characteristics": [], "expected_packs": [ "AI-004", "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00010", "persona": "Financial-technology engineer", "archetype_id": "calculator", "risk": "low_risk", "query": "The system should provide a small arithmetic practice tool for a small team. The first version can stay focused.", "expected_characteristics": [], "detected_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "missing_characteristics": [], "additional_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "expected_packs": [], "matched_packs": [ "DAT-001", "DAT-003", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "low_risk_false_positive" ] }, { "query_id": "LQE-00014", "persona": "Independent developer", "archetype_id": "unit_converter", "risk": "low_risk", "query": "Build a unit converter for length and temperature for a small team. Keep the experience simple.", "expected_characteristics": [], "detected_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "missing_characteristics": [], "additional_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "expected_packs": [], "matched_packs": [ "DAT-001", "DAT-003", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "low_risk_false_positive" ] }, { "query_id": "LQE-00017", "persona": "Media producer", "archetype_id": "ai_decision", "risk": "risk_bearing", "query": "My project needs an AI feature that determines whether an account is restricted inside an existing application. People should understand what happened.", "expected_characteristics": [ "AI_CONTROLLED", "CONSEQUENTIAL_ACTION" ], "detected_characteristics": [ "AI_CONTROLLED" ], "missing_characteristics": [ "CONSEQUENTIAL_ACTION" ], "additional_characteristics": [], "expected_packs": [ "AI-004", "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "matched_packs": [ "AI-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00018", "persona": "Healthcare administrator", "archetype_id": "public_submission", "risk": "risk_bearing", "query": "Please help us set up a contact form that receives free-text messages for regular day-to-day use. The first version can stay focused.", "expected_characteristics": [ "UNTRUSTED_INPUT" ], "detected_characteristics": [], "missing_characteristics": [ "UNTRUSTED_INPUT" ], "additional_characteristics": [], "expected_packs": [ "INP-001", "INP-002" ], "matched_packs": [], "failure_types": [ "expected_characteristic_missing", "risk_query_returned_no_pack", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00019", "persona": "Accounting professional", "archetype_id": "document_upload", "risk": "risk_bearing", "query": "I need a tool with a portal where people upload confidential PDF forms for regular day-to-day use. This is the main feature.", "expected_characteristics": [ "SENSITIVE_DATA", "UNTRUSTED_INPUT" ], "detected_characteristics": [], "missing_characteristics": [ "SENSITIVE_DATA", "UNTRUSTED_INPUT" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "INP-001", "INP-002", "REL-004" ], "matched_packs": [], "failure_types": [ "expected_characteristic_missing", "risk_query_returned_no_pack", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00021", "persona": "Independent developer", "archetype_id": "automated_refund", "risk": "risk_bearing", "query": "Build automatic refunds issued through the payment API as the main workflow. The first version can stay focused.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00022", "persona": "Operations professional", "archetype_id": "payroll", "risk": "risk_bearing", "query": "We need a tool that calculates pay and submits it to a payroll vendor for regular day-to-day use. The first version can stay focused.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "detected_characteristics": [ "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00023", "persona": "Security lead", "archetype_id": "deployment_agent", "risk": "risk_bearing", "query": "Review a design for automatic production releases with infrastructure updates for regular day-to-day use. People should understand what happened.", "expected_characteristics": [ "PRIVILEGED_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "PRODUCTION_MODIFICATION" ], "missing_characteristics": [ "PRIVILEGED_ACTION" ], "additional_characteristics": [], "expected_packs": [ "CHG-001", "IDN-004", "IDN-005", "OBS-002", "REL-004" ], "matched_packs": [ "CHG-001", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00024", "persona": "Mobile developer", "archetype_id": "external_sync", "risk": "risk_bearing", "query": "The app needs an integration that exchanges client information with a vendor inside an existing application. This is the main feature.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "SENSITIVE_DATA" ], "detected_characteristics": [ "SENSITIVE_DATA" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "DAT-001", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00025", "persona": "Healthcare administrator", "archetype_id": "deployment_agent", "risk": "risk_bearing", "query": "Please help us set up an agent that deploys code and changes production configuration for regular day-to-day use. Keep the experience simple.", "expected_characteristics": [ "PRIVILEGED_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "PRODUCTION_MODIFICATION" ], "missing_characteristics": [ "PRIVILEGED_ACTION" ], "additional_characteristics": [ "CONSEQUENTIAL_ACTION" ], "expected_packs": [ "CHG-001", "IDN-004", "IDN-005", "OBS-002", "REL-004" ], "matched_packs": [ "AUT-001", "AUT-004", "CHG-001", "OBS-001", "OBS-003", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00027", "persona": "Government analyst", "archetype_id": "ai_support", "risk": "risk_bearing", "query": "The proposed service includes an assistant that uses AI to reply to support requests for a new service. It should work consistently.", "expected_characteristics": [ "AI_CONTROLLED", "UNTRUSTED_INPUT" ], "detected_characteristics": [ "AI_CONTROLLED", "TOOL_ACCESS" ], "missing_characteristics": [ "UNTRUSTED_INPUT" ], "additional_characteristics": [ "TOOL_ACCESS" ], "expected_packs": [ "AI-001", "AI-004", "INP-001", "INP-002" ], "matched_packs": [ "AI-004", "AUT-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00028", "persona": "Educator", "archetype_id": "automated_refund", "risk": "risk_bearing", "query": "I would like a workflow that calls a billing tool to approve refunds for a new service. Keep the experience simple.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00030", "persona": "Operations professional", "archetype_id": "database_migration", "risk": "risk_bearing", "query": "We need schema migrations applied directly to the live database for regular day-to-day use. This is the main feature.", "expected_characteristics": [ "IRREVERSIBLE_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "IRREVERSIBLE_ACTION" ], "missing_characteristics": [ "PRODUCTION_MODIFICATION" ], "additional_characteristics": [ "CONSEQUENTIAL_ACTION" ], "expected_packs": [ "AUT-002", "CHG-001", "REL-004" ], "matched_packs": [ "AUT-001", "AUT-002", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00033", "persona": "Data engineer", "archetype_id": "account_login", "risk": "risk_bearing", "query": "Implement a member login that stores account information for regular day-to-day use. Keep the experience simple.", "expected_characteristics": [ "SENSITIVE_DATA", "UNTRUSTED_INPUT" ], "detected_characteristics": [], "missing_characteristics": [ "SENSITIVE_DATA", "UNTRUSTED_INPUT" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "INP-001", "INP-002", "REL-004" ], "matched_packs": [], "failure_types": [ "expected_characteristic_missing", "risk_query_returned_no_pack", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00034", "persona": "Marketplace operator", "archetype_id": "file_converter", "risk": "risk_bearing", "query": "We are adding an image transformer for user-provided files for a small team. Keep the experience simple.", "expected_characteristics": [ "UNTRUSTED_INPUT" ], "detected_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "missing_characteristics": [ "UNTRUSTED_INPUT" ], "additional_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "expected_packs": [ "INP-001", "INP-002" ], "matched_packs": [ "DAT-001", "DAT-003", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00036", "persona": "Marketplace operator", "archetype_id": "webhook", "risk": "risk_bearing", "query": "We are adding incoming webhooks from an external billing provider for regular day-to-day use. Keep the experience simple.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "UNTRUSTED_INPUT" ], "detected_characteristics": [ "FINANCIAL_ACTION" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY", "UNTRUSTED_INPUT" ], "additional_characteristics": [ "FINANCIAL_ACTION" ], "expected_packs": [ "INP-001", "INP-002", "OBS-003", "REL-001" ], "matched_packs": [ "AUT-001" ], "failure_types": [ "expected_characteristic_missing", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00038", "persona": "Educator", "archetype_id": "account_login", "risk": "risk_bearing", "query": "I would like sign-in for users with account recovery for a new service. It should work consistently.", "expected_characteristics": [ "SENSITIVE_DATA", "UNTRUSTED_INPUT" ], "detected_characteristics": [], "missing_characteristics": [ "SENSITIVE_DATA", "UNTRUSTED_INPUT" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "INP-001", "INP-002", "REL-004" ], "matched_packs": [], "failure_types": [ "expected_characteristic_missing", "risk_query_returned_no_pack", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00041", "persona": "Independent developer", "archetype_id": "payroll", "risk": "risk_bearing", "query": "Build a salary system connected to an outside payroll service inside an existing application. People should understand what happened.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "detected_characteristics": [ "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00042", "persona": "Security lead", "archetype_id": "scheduled_automation", "risk": "risk_bearing", "query": "Review a design for automatic overnight reconciliation through internal tools inside an existing application. The first version can stay focused.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [ "FINANCIAL_ACTION" ], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00043", "persona": "Non-technical founder", "archetype_id": "payroll", "risk": "risk_bearing", "query": "I want monthly payroll with employee banking and tax details for a small team. This is the main feature.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "detected_characteristics": [ "FINANCIAL_ACTION", "MULTI_TENANT", "SENSITIVE_DATA" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [ "MULTI_TENANT" ], "expected_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "AUT-001", "DAT-001", "DAT-003", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00044", "persona": "Mobile developer", "archetype_id": "external_sync", "risk": "risk_bearing", "query": "The app needs an integration that exchanges client information with a vendor as the main workflow. People should understand what happened.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "SENSITIVE_DATA" ], "detected_characteristics": [ "SENSITIVE_DATA" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "DAT-001", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00045", "persona": "Enterprise product manager", "archetype_id": "permanent_delete", "risk": "risk_bearing", "query": "Our next feature is a button that irreversibly removes stored projects for a new service. This is the main feature.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "IRREVERSIBLE_ACTION" ], "detected_characteristics": [], "missing_characteristics": [ "CONSEQUENTIAL_ACTION", "IRREVERSIBLE_ACTION" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-002", "AUT-004", "OBS-001", "OBS-003" ], "matched_packs": [], "failure_types": [ "expected_characteristic_missing", "risk_query_returned_no_pack", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00047", "persona": "Human-resources manager", "archetype_id": "automated_refund", "risk": "risk_bearing", "query": "We would use a service that triggers refunds through an external function for a small team. Keep the experience simple.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION", "MULTI_TENANT", "SENSITIVE_DATA" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "DAT-001", "DAT-003", "DAT-005", "IDN-005", "OBS-001", "OBS-003", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00049", "persona": "Non-technical founder", "archetype_id": "data_export", "risk": "risk_bearing", "query": "I want a feature that downloads employee information inside an existing application. Keep the experience simple.", "expected_characteristics": [ "DATA_EXPORT", "SENSITIVE_DATA" ], "detected_characteristics": [], "missing_characteristics": [ "DATA_EXPORT", "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-002", "DAT-005", "IDN-005", "REL-004" ], "matched_packs": [], "failure_types": [ "expected_characteristic_missing", "risk_query_returned_no_pack", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00050", "persona": "Financial-technology engineer", "archetype_id": "deployment_agent", "risk": "risk_bearing", "query": "The system should provide an agent that deploys code and changes production configuration for a small team. People should understand what happened.", "expected_characteristics": [ "PRIVILEGED_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "MULTI_TENANT", "PRODUCTION_MODIFICATION", "SENSITIVE_DATA" ], "missing_characteristics": [ "PRIVILEGED_ACTION" ], "additional_characteristics": [ "CONSEQUENTIAL_ACTION", "MULTI_TENANT", "SENSITIVE_DATA" ], "expected_packs": [ "CHG-001", "IDN-004", "IDN-005", "OBS-002", "REL-004" ], "matched_packs": [ "AUT-001", "AUT-004", "CHG-001", "DAT-001", "DAT-003", "DAT-005", "IDN-005", "OBS-001", "OBS-003", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00058", "persona": "Site reliability engineer", "archetype_id": "payroll", "risk": "risk_bearing", "query": "We are planning employee compensation records and payroll payments for regular day-to-day use. This is the main feature.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "detected_characteristics": [ "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00060", "persona": "Enterprise product manager", "archetype_id": "expensive_processing", "risk": "risk_bearing", "query": "Our next feature is compute-heavy report generation for large datasets for a small team. The first version can stay focused.", "expected_characteristics": [ "HIGH_COST_CONSUMPTION" ], "detected_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "missing_characteristics": [ "HIGH_COST_CONSUMPTION" ], "additional_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "expected_packs": [ "AUT-003" ], "matched_packs": [ "DAT-001", "DAT-003", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00061", "persona": "Site reliability engineer", "archetype_id": "external_notifications", "risk": "risk_bearing", "query": "We are planning email notifications delivered by an outside service inside an existing application. People should understand what happened.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003", "REL-001" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00062", "persona": "Government analyst", "archetype_id": "payroll", "risk": "risk_bearing", "query": "The proposed service includes a tool that calculates pay and submits it to a payroll vendor for a new service. It should work consistently.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "detected_characteristics": [ "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00063", "persona": "Marketplace operator", "archetype_id": "account_login", "risk": "risk_bearing", "query": "We are adding a member login that stores account information for a small team. It should work consistently.", "expected_characteristics": [ "SENSITIVE_DATA", "UNTRUSTED_INPUT" ], "detected_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "missing_characteristics": [ "UNTRUSTED_INPUT" ], "additional_characteristics": [ "MULTI_TENANT" ], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "INP-001", "INP-002", "REL-004" ], "matched_packs": [ "DAT-001", "DAT-003", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00064", "persona": "Independent developer", "archetype_id": "scheduled_automation", "risk": "risk_bearing", "query": "Build an unattended batch job that processes pending cases as the main workflow. Keep the experience simple.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "HIGH_COST_CONSUMPTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [ "HIGH_COST_CONSUMPTION" ], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-003", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00066", "persona": "Independent developer", "archetype_id": "expensive_processing", "risk": "risk_bearing", "query": "Build resource-intensive media conversion jobs for a small team. The first version can stay focused.", "expected_characteristics": [ "HIGH_COST_CONSUMPTION" ], "detected_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "missing_characteristics": [ "HIGH_COST_CONSUMPTION" ], "additional_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "expected_packs": [ "AUT-003" ], "matched_packs": [ "DAT-001", "DAT-003", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00069", "persona": "Accounting professional", "archetype_id": "expensive_processing", "risk": "risk_bearing", "query": "I need a tool with compute-heavy report generation for large datasets for a new service. The first version can stay focused.", "expected_characteristics": [ "HIGH_COST_CONSUMPTION" ], "detected_characteristics": [], "missing_characteristics": [ "HIGH_COST_CONSUMPTION" ], "additional_characteristics": [], "expected_packs": [ "AUT-003" ], "matched_packs": [], "failure_types": [ "expected_characteristic_missing", "risk_query_returned_no_pack", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00070", "persona": "Site reliability engineer", "archetype_id": "external_notifications", "risk": "risk_bearing", "query": "We are planning automatic reminders using a third-party communications provider for a small team. The first version can stay focused.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "MULTI_TENANT", "SENSITIVE_DATA" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "expected_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003", "REL-001" ], "matched_packs": [ "AUT-001", "AUT-004", "DAT-001", "DAT-003", "DAT-005", "IDN-005", "OBS-001", "OBS-003", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00071", "persona": "Small-business owner", "archetype_id": "static_page", "risk": "low_risk", "query": "Could you make a simple portfolio with fixed text and pictures for a small team. People should understand what happened.", "expected_characteristics": [], "detected_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "missing_characteristics": [], "additional_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "expected_packs": [], "matched_packs": [ "DAT-001", "DAT-003", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "low_risk_false_positive" ] }, { "query_id": "LQE-00074", "persona": "Non-technical founder", "archetype_id": "permissions", "risk": "risk_bearing", "query": "I want permission management for staff and contractors for regular day-to-day use. It should work consistently.", "expected_characteristics": [ "PRIVILEGED_ACTION" ], "detected_characteristics": [], "missing_characteristics": [ "PRIVILEGED_ACTION" ], "additional_characteristics": [], "expected_packs": [ "IDN-004", "IDN-005" ], "matched_packs": [], "failure_types": [ "expected_characteristic_missing", "risk_query_returned_no_pack", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00075", "persona": "Enterprise product manager", "archetype_id": "file_converter", "risk": "risk_bearing", "query": "Our next feature is an image transformer for user-provided files for regular day-to-day use. This is the main feature.", "expected_characteristics": [ "UNTRUSTED_INPUT" ], "detected_characteristics": [], "missing_characteristics": [ "UNTRUSTED_INPUT" ], "additional_characteristics": [], "expected_packs": [ "INP-001", "INP-002" ], "matched_packs": [], "failure_types": [ "expected_characteristic_missing", "risk_query_returned_no_pack", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00076", "persona": "Machine-learning engineer", "archetype_id": "admin_audit", "risk": "risk_bearing", "query": "Create staff tools for suspending and restoring user accounts inside an existing application. It should work consistently.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "PRIVILEGED_ACTION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "IRREVERSIBLE_ACTION" ], "missing_characteristics": [ "PRIVILEGED_ACTION" ], "additional_characteristics": [ "IRREVERSIBLE_ACTION" ], "expected_packs": [ "AUT-001", "AUT-004", "IDN-004", "IDN-005", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-002", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00078", "persona": "Healthcare administrator", "archetype_id": "document_upload", "risk": "risk_bearing", "query": "Please help us set up uploads for customer contracts and identity documents for a small team. Keep the experience simple.", "expected_characteristics": [ "SENSITIVE_DATA", "UNTRUSTED_INPUT" ], "detected_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "missing_characteristics": [ "UNTRUSTED_INPUT" ], "additional_characteristics": [ "MULTI_TENANT" ], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "INP-001", "INP-002", "REL-004" ], "matched_packs": [ "DAT-001", "DAT-003", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00079", "persona": "Operations professional", "archetype_id": "database_migration", "risk": "risk_bearing", "query": "We need an operation that transforms records in the production database for a small team. Keep the experience simple.", "expected_characteristics": [ "IRREVERSIBLE_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "MULTI_TENANT", "PRIVILEGED_ACTION", "PRODUCTION_MODIFICATION", "SENSITIVE_DATA" ], "missing_characteristics": [ "IRREVERSIBLE_ACTION" ], "additional_characteristics": [ "CONSEQUENTIAL_ACTION", "MULTI_TENANT", "PRIVILEGED_ACTION", "SENSITIVE_DATA" ], "expected_packs": [ "AUT-002", "CHG-001", "REL-004" ], "matched_packs": [ "AUT-001", "AUT-004", "CHG-001", "DAT-001", "DAT-003", "DAT-005", "IDN-004", "IDN-005", "OBS-001", "OBS-002", "OBS-003", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00080", "persona": "Enterprise product manager", "archetype_id": "multi_tenant", "risk": "risk_bearing", "query": "Our next feature is a multi-customer portal with organization-specific records for a new service. The first version can stay focused.", "expected_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "detected_characteristics": [], "missing_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-003", "DAT-005", "IDN-005", "REL-004" ], "matched_packs": [], "failure_types": [ "expected_characteristic_missing", "risk_query_returned_no_pack", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00081", "persona": "Nonprofit program lead", "archetype_id": "countdown", "risk": "low_risk", "query": "Our organization needs a basic interval timer with no saved data for a small team. The first version can stay focused.", "expected_characteristics": [], "detected_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "missing_characteristics": [], "additional_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "expected_packs": [], "matched_packs": [ "DAT-001", "DAT-003", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "low_risk_false_positive" ] }, { "query_id": "LQE-00084", "persona": "Marketplace operator", "archetype_id": "card_payment", "risk": "risk_bearing", "query": "We are adding a purchase flow connected to an external payment processor inside an existing application. This is the main feature.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION" ], "detected_characteristics": [ "EXTERNAL_DEPENDENCY" ], "missing_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003", "REL-001" ], "matched_packs": [ "OBS-003", "REL-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00085", "persona": "Operations professional", "archetype_id": "multi_tenant", "risk": "risk_bearing", "query": "We need a multi-customer portal with organization-specific records for a new service. Keep the experience simple.", "expected_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "detected_characteristics": [], "missing_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-003", "DAT-005", "IDN-005", "REL-004" ], "matched_packs": [], "failure_types": [ "expected_characteristic_missing", "risk_query_returned_no_pack", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00087", "persona": "Media producer", "archetype_id": "document_upload", "risk": "risk_bearing", "query": "My project needs a feature that accepts sensitive files from outside users for regular day-to-day use. This is the main feature.", "expected_characteristics": [ "SENSITIVE_DATA", "UNTRUSTED_INPUT" ], "detected_characteristics": [], "missing_characteristics": [ "SENSITIVE_DATA", "UNTRUSTED_INPUT" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "INP-001", "INP-002", "REL-004" ], "matched_packs": [], "failure_types": [ "expected_characteristic_missing", "risk_query_returned_no_pack", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00088", "persona": "Student or learner", "archetype_id": "ai_support", "risk": "risk_bearing", "query": "For a learning project, make an assistant that uses AI to reply to support requests as the main workflow. This is the main feature.", "expected_characteristics": [ "AI_CONTROLLED", "UNTRUSTED_INPUT" ], "detected_characteristics": [ "AI_CONTROLLED", "TOOL_ACCESS" ], "missing_characteristics": [ "UNTRUSTED_INPUT" ], "additional_characteristics": [ "TOOL_ACCESS" ], "expected_packs": [ "AI-001", "AI-004", "INP-001", "INP-002" ], "matched_packs": [ "AI-004", "AUT-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00089", "persona": "Operations professional", "archetype_id": "data_export", "risk": "risk_bearing", "query": "We need exports of customer records to downloadable CSV files for a small team. This is the main feature.", "expected_characteristics": [ "DATA_EXPORT", "SENSITIVE_DATA" ], "detected_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "missing_characteristics": [ "DATA_EXPORT" ], "additional_characteristics": [ "MULTI_TENANT" ], "expected_packs": [ "DAT-001", "DAT-002", "DAT-005", "IDN-005", "REL-004" ], "matched_packs": [ "DAT-001", "DAT-003", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00090", "persona": "Healthcare administrator", "archetype_id": "automated_refund", "risk": "risk_bearing", "query": "Please help us set up a service that triggers refunds through an external function as the main workflow. People should understand what happened.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00091", "persona": "Nonprofit program lead", "archetype_id": "ai_decision", "risk": "risk_bearing", "query": "Our organization needs an AI recommendation that decides which applications receive priority inside an existing application. The first version can stay focused.", "expected_characteristics": [ "AI_CONTROLLED", "CONSEQUENTIAL_ACTION" ], "detected_characteristics": [ "AI_CONTROLLED" ], "missing_characteristics": [ "CONSEQUENTIAL_ACTION" ], "additional_characteristics": [], "expected_packs": [ "AI-004", "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "matched_packs": [ "AI-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00093", "persona": "Non-technical founder", "archetype_id": "external_sync", "risk": "risk_bearing", "query": "I want daily data sync with an outside business platform for regular day-to-day use. Keep the experience simple.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "SENSITIVE_DATA" ], "detected_characteristics": [], "missing_characteristics": [ "EXTERNAL_DEPENDENCY", "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [], "failure_types": [ "expected_characteristic_missing", "risk_query_returned_no_pack", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00097", "persona": "Enterprise product manager", "archetype_id": "ai_support", "risk": "risk_bearing", "query": "Our next feature is a model-powered help desk for messages from customers for regular day-to-day use. This is the main feature.", "expected_characteristics": [ "AI_CONTROLLED", "UNTRUSTED_INPUT" ], "detected_characteristics": [], "missing_characteristics": [ "AI_CONTROLLED", "UNTRUSTED_INPUT" ], "additional_characteristics": [], "expected_packs": [ "AI-001", "AI-004", "INP-001", "INP-002" ], "matched_packs": [], "failure_types": [ "expected_characteristic_missing", "risk_query_returned_no_pack", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00098", "persona": "Government analyst", "archetype_id": "payroll", "risk": "risk_bearing", "query": "The proposed service includes monthly payroll with employee banking and tax details inside an existing application. Keep the experience simple.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "detected_characteristics": [ "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00099", "persona": "Student or learner", "archetype_id": "payroll", "risk": "risk_bearing", "query": "For a learning project, make a salary system connected to an outside payroll service inside an existing application. Keep the experience simple.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "detected_characteristics": [ "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00100", "persona": "Nonprofit program lead", "archetype_id": "admin_audit", "risk": "risk_bearing", "query": "Our organization needs an administrative console that changes account status for a small team. It should work consistently.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "PRIVILEGED_ACTION" ], "detected_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "missing_characteristics": [ "CONSEQUENTIAL_ACTION", "PRIVILEGED_ACTION" ], "additional_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "expected_packs": [ "AUT-001", "AUT-004", "IDN-004", "IDN-005", "OBS-001", "OBS-003" ], "matched_packs": [ "DAT-001", "DAT-003", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00103", "persona": "Machine-learning engineer", "archetype_id": "external_sync", "risk": "risk_bearing", "query": "Create customer-data synchronization through an external API for regular day-to-day use. This is the main feature.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "SENSITIVE_DATA" ], "detected_characteristics": [ "EXTERNAL_DEPENDENCY" ], "missing_characteristics": [ "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "OBS-003", "REL-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00106", "persona": "Non-technical founder", "archetype_id": "admin_audit", "risk": "risk_bearing", "query": "I want staff tools for suspending and restoring user accounts for regular day-to-day use. The first version can stay focused.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "PRIVILEGED_ACTION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "IRREVERSIBLE_ACTION" ], "missing_characteristics": [ "PRIVILEGED_ACTION" ], "additional_characteristics": [ "IRREVERSIBLE_ACTION" ], "expected_packs": [ "AUT-001", "AUT-004", "IDN-004", "IDN-005", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-002", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00110", "persona": "Independent developer", "archetype_id": "file_converter", "risk": "risk_bearing", "query": "Build a service that opens and converts outside documents inside an existing application. It should work consistently.", "expected_characteristics": [ "UNTRUSTED_INPUT" ], "detected_characteristics": [], "missing_characteristics": [ "UNTRUSTED_INPUT" ], "additional_characteristics": [], "expected_packs": [ "INP-001", "INP-002" ], "matched_packs": [], "failure_types": [ "expected_characteristic_missing", "risk_query_returned_no_pack", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00112", "persona": "Government analyst", "archetype_id": "deployment_agent", "risk": "risk_bearing", "query": "The proposed service includes automation that applies configuration changes in production for a small team. People should understand what happened.", "expected_characteristics": [ "PRIVILEGED_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "MULTI_TENANT", "SENSITIVE_DATA" ], "missing_characteristics": [ "PRIVILEGED_ACTION", "PRODUCTION_MODIFICATION" ], "additional_characteristics": [ "CONSEQUENTIAL_ACTION", "MULTI_TENANT", "SENSITIVE_DATA" ], "expected_packs": [ "CHG-001", "IDN-004", "IDN-005", "OBS-002", "REL-004" ], "matched_packs": [ "AUT-001", "AUT-004", "DAT-001", "DAT-003", "DAT-005", "IDN-005", "OBS-001", "OBS-003", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00113", "persona": "Non-technical founder", "archetype_id": "webhook", "risk": "risk_bearing", "query": "I want incoming webhooks from an external billing provider for a new service. Keep the experience simple.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "UNTRUSTED_INPUT" ], "detected_characteristics": [ "FINANCIAL_ACTION" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY", "UNTRUSTED_INPUT" ], "additional_characteristics": [ "FINANCIAL_ACTION" ], "expected_packs": [ "INP-001", "INP-002", "OBS-003", "REL-001" ], "matched_packs": [ "AUT-001" ], "failure_types": [ "expected_characteristic_missing", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00114", "persona": "Marketplace operator", "archetype_id": "scheduled_automation", "risk": "risk_bearing", "query": "We are adding an unattended batch job that processes pending cases for a new service. People should understand what happened.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "HIGH_COST_CONSUMPTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [ "HIGH_COST_CONSUMPTION" ], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-003", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00115", "persona": "Security lead", "archetype_id": "account_login", "risk": "risk_bearing", "query": "Review a design for authenticated user accounts with password resets for a small team. Keep the experience simple.", "expected_characteristics": [ "SENSITIVE_DATA", "UNTRUSTED_INPUT" ], "detected_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "missing_characteristics": [ "UNTRUSTED_INPUT" ], "additional_characteristics": [ "MULTI_TENANT" ], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "INP-001", "INP-002", "REL-004" ], "matched_packs": [ "DAT-001", "DAT-003", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00116", "persona": "Financial-technology engineer", "archetype_id": "database_migration", "risk": "risk_bearing", "query": "The system should provide schema migrations applied directly to the live database as the main workflow. This is the main feature.", "expected_characteristics": [ "IRREVERSIBLE_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "IRREVERSIBLE_ACTION" ], "missing_characteristics": [ "PRODUCTION_MODIFICATION" ], "additional_characteristics": [ "CONSEQUENTIAL_ACTION" ], "expected_packs": [ "AUT-002", "CHG-001", "REL-004" ], "matched_packs": [ "AUT-001", "AUT-002", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00117", "persona": "Independent developer", "archetype_id": "data_export", "risk": "risk_bearing", "query": "Build a report that sends confidential records outside the application as the main workflow. It should work consistently.", "expected_characteristics": [ "DATA_EXPORT", "SENSITIVE_DATA" ], "detected_characteristics": [], "missing_characteristics": [ "DATA_EXPORT", "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-002", "DAT-005", "IDN-005", "REL-004" ], "matched_packs": [], "failure_types": [ "expected_characteristic_missing", "risk_query_returned_no_pack", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00118", "persona": "Accounting professional", "archetype_id": "ai_decision", "risk": "risk_bearing", "query": "I need a tool with an AI recommendation that decides which applications receive priority as the main workflow. People should understand what happened.", "expected_characteristics": [ "AI_CONTROLLED", "CONSEQUENTIAL_ACTION" ], "detected_characteristics": [ "AI_CONTROLLED" ], "missing_characteristics": [ "CONSEQUENTIAL_ACTION" ], "additional_characteristics": [], "expected_packs": [ "AI-004", "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "matched_packs": [ "AI-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00119", "persona": "Operations professional", "archetype_id": "multi_tenant", "risk": "risk_bearing", "query": "We need a SaaS application where every company has private data for a new service. It should work consistently.", "expected_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "detected_characteristics": [], "missing_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-003", "DAT-005", "IDN-005", "REL-004" ], "matched_packs": [], "failure_types": [ "expected_characteristic_missing", "risk_query_returned_no_pack", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00120", "persona": "Nonprofit program lead", "archetype_id": "account_login", "risk": "risk_bearing", "query": "Our organization needs a member login that stores account information inside an existing application. Keep the experience simple.", "expected_characteristics": [ "SENSITIVE_DATA", "UNTRUSTED_INPUT" ], "detected_characteristics": [], "missing_characteristics": [ "SENSITIVE_DATA", "UNTRUSTED_INPUT" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "INP-001", "INP-002", "REL-004" ], "matched_packs": [], "failure_types": [ "expected_characteristic_missing", "risk_query_returned_no_pack", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00121", "persona": "Healthcare administrator", "archetype_id": "scheduled_automation", "risk": "risk_bearing", "query": "Please help us set up scheduled automation that changes order statuses as the main workflow. The first version can stay focused.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00122", "persona": "Small-business owner", "archetype_id": "data_export", "risk": "risk_bearing", "query": "Could you make a report that sends confidential records outside the application for regular day-to-day use. People should understand what happened.", "expected_characteristics": [ "DATA_EXPORT", "SENSITIVE_DATA" ], "detected_characteristics": [], "missing_characteristics": [ "DATA_EXPORT", "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-002", "DAT-005", "IDN-005", "REL-004" ], "matched_packs": [], "failure_types": [ "expected_characteristic_missing", "risk_query_returned_no_pack", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00124", "persona": "Accounting professional", "archetype_id": "deployment_agent", "risk": "risk_bearing", "query": "I need a tool with an agent that deploys code and changes production configuration for regular day-to-day use. Keep the experience simple.", "expected_characteristics": [ "PRIVILEGED_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "PRODUCTION_MODIFICATION" ], "missing_characteristics": [ "PRIVILEGED_ACTION" ], "additional_characteristics": [ "CONSEQUENTIAL_ACTION" ], "expected_packs": [ "CHG-001", "IDN-004", "IDN-005", "OBS-002", "REL-004" ], "matched_packs": [ "AUT-001", "AUT-004", "CHG-001", "OBS-001", "OBS-003", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00125", "persona": "Independent developer", "archetype_id": "ai_decision", "risk": "risk_bearing", "query": "Build AI scoring that changes which cases move forward for a new service. People should understand what happened.", "expected_characteristics": [ "AI_CONTROLLED", "CONSEQUENTIAL_ACTION" ], "detected_characteristics": [ "AI_CONTROLLED" ], "missing_characteristics": [ "CONSEQUENTIAL_ACTION" ], "additional_characteristics": [], "expected_packs": [ "AI-004", "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "matched_packs": [ "AI-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00126", "persona": "Site reliability engineer", "archetype_id": "scheduled_automation", "risk": "risk_bearing", "query": "We are planning a background workflow that updates records every morning as the main workflow. This is the main feature.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00130", "persona": "Data engineer", "archetype_id": "account_login", "risk": "risk_bearing", "query": "Implement authenticated user accounts with password resets for regular day-to-day use. Keep the experience simple.", "expected_characteristics": [ "SENSITIVE_DATA", "UNTRUSTED_INPUT" ], "detected_characteristics": [ "SENSITIVE_DATA" ], "missing_characteristics": [ "UNTRUSTED_INPUT" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "INP-001", "INP-002", "REL-004" ], "matched_packs": [ "DAT-001", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00131", "persona": "Security lead", "archetype_id": "calculator", "risk": "low_risk", "query": "Review a design for a simple percentage calculator for a small team. This is the main feature.", "expected_characteristics": [], "detected_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "missing_characteristics": [], "additional_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "expected_packs": [], "matched_packs": [ "DAT-001", "DAT-003", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "low_risk_false_positive" ] }, { "query_id": "LQE-00132", "persona": "Independent developer", "archetype_id": "payroll", "risk": "risk_bearing", "query": "Build payroll processing that sends employee pay to a provider for a new service. It should work consistently.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "detected_characteristics": [ "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00133", "persona": "Financial-technology engineer", "archetype_id": "external_sync", "risk": "risk_bearing", "query": "The system should provide synchronization of customer records with an external CRM API for regular day-to-day use. This is the main feature.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "SENSITIVE_DATA" ], "detected_characteristics": [ "SENSITIVE_DATA" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "DAT-001", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00135", "persona": "Marketplace operator", "archetype_id": "external_notifications", "risk": "risk_bearing", "query": "We are adding outbound notifications sent to users through an external platform for regular day-to-day use. This is the main feature.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003", "REL-001" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00136", "persona": "Student or learner", "archetype_id": "external_notifications", "risk": "risk_bearing", "query": "For a learning project, make automatic reminders using a third-party communications provider for a new service. The first version can stay focused.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003", "REL-001" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00138", "persona": "Human-resources manager", "archetype_id": "bulk_update", "risk": "risk_bearing", "query": "We would use one command that modifies a large set of records inside an existing application. Keep the experience simple.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION" ], "detected_characteristics": [], "missing_characteristics": [ "CONSEQUENTIAL_ACTION" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "matched_packs": [], "failure_types": [ "expected_characteristic_missing", "risk_query_returned_no_pack", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00141", "persona": "Non-technical founder", "archetype_id": "database_migration", "risk": "risk_bearing", "query": "I want schema migrations applied directly to the live database inside an existing application. It should work consistently.", "expected_characteristics": [ "IRREVERSIBLE_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "IRREVERSIBLE_ACTION" ], "missing_characteristics": [ "PRODUCTION_MODIFICATION" ], "additional_characteristics": [ "CONSEQUENTIAL_ACTION" ], "expected_packs": [ "AUT-002", "CHG-001", "REL-004" ], "matched_packs": [ "AUT-001", "AUT-002", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00143", "persona": "Operations professional", "archetype_id": "automated_refund", "risk": "risk_bearing", "query": "We need automatic refunds issued through the payment API for regular day-to-day use. It should work consistently.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00145", "persona": "Nonprofit program lead", "archetype_id": "webhook", "risk": "risk_bearing", "query": "Our organization needs outside services posting event payloads to our API for a new service. People should understand what happened.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "UNTRUSTED_INPUT" ], "detected_characteristics": [], "missing_characteristics": [ "EXTERNAL_DEPENDENCY", "UNTRUSTED_INPUT" ], "additional_characteristics": [], "expected_packs": [ "INP-001", "INP-002", "OBS-003", "REL-001" ], "matched_packs": [], "failure_types": [ "expected_characteristic_missing", "risk_query_returned_no_pack", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00146", "persona": "Student or learner", "archetype_id": "ai_support", "risk": "risk_bearing", "query": "For a learning project, make a model-powered help desk for messages from customers for a small team. This is the main feature.", "expected_characteristics": [ "AI_CONTROLLED", "UNTRUSTED_INPUT" ], "detected_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "missing_characteristics": [ "AI_CONTROLLED", "UNTRUSTED_INPUT" ], "additional_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "expected_packs": [ "AI-001", "AI-004", "INP-001", "INP-002" ], "matched_packs": [ "DAT-001", "DAT-003", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00147", "persona": "Accounting professional", "archetype_id": "deployment_agent", "risk": "risk_bearing", "query": "I need a tool with automatic production releases with infrastructure updates inside an existing application. Keep the experience simple.", "expected_characteristics": [ "PRIVILEGED_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "PRODUCTION_MODIFICATION" ], "missing_characteristics": [ "PRIVILEGED_ACTION" ], "additional_characteristics": [], "expected_packs": [ "CHG-001", "IDN-004", "IDN-005", "OBS-002", "REL-004" ], "matched_packs": [ "CHG-001", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00148", "persona": "Operations professional", "archetype_id": "external_notifications", "risk": "risk_bearing", "query": "We need automatic reminders using a third-party communications provider inside an existing application. The first version can stay focused.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003", "REL-001" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00149", "persona": "Machine-learning engineer", "archetype_id": "account_login", "risk": "risk_bearing", "query": "Create a member login that stores account information inside an existing application. The first version can stay focused.", "expected_characteristics": [ "SENSITIVE_DATA", "UNTRUSTED_INPUT" ], "detected_characteristics": [], "missing_characteristics": [ "SENSITIVE_DATA", "UNTRUSTED_INPUT" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "INP-001", "INP-002", "REL-004" ], "matched_packs": [], "failure_types": [ "expected_characteristic_missing", "risk_query_returned_no_pack", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00150", "persona": "Healthcare administrator", "archetype_id": "account_login", "risk": "risk_bearing", "query": "Please help us set up authenticated user accounts with password resets inside an existing application. This is the main feature.", "expected_characteristics": [ "SENSITIVE_DATA", "UNTRUSTED_INPUT" ], "detected_characteristics": [ "SENSITIVE_DATA" ], "missing_characteristics": [ "UNTRUSTED_INPUT" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "INP-001", "INP-002", "REL-004" ], "matched_packs": [ "DAT-001", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00151", "persona": "Data engineer", "archetype_id": "deployment_agent", "risk": "risk_bearing", "query": "Implement a release agent with permission to update production systems for a small team. It should work consistently.", "expected_characteristics": [ "PRIVILEGED_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "MULTI_TENANT", "PRODUCTION_MODIFICATION", "SENSITIVE_DATA", "TOOL_ACCESS" ], "missing_characteristics": [ "PRIVILEGED_ACTION" ], "additional_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA", "TOOL_ACCESS" ], "expected_packs": [ "CHG-001", "IDN-004", "IDN-005", "OBS-002", "REL-004" ], "matched_packs": [ "AUT-004", "CHG-001", "DAT-001", "DAT-003", "DAT-005", "IDN-003", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00152", "persona": "Educator", "archetype_id": "external_notifications", "risk": "risk_bearing", "query": "I would like automatic reminders using a third-party communications provider for a new service. People should understand what happened.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003", "REL-001" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00153", "persona": "Human-resources manager", "archetype_id": "external_notifications", "risk": "risk_bearing", "query": "We would use a workflow that sends customer alerts through a messaging API as the main workflow. Keep the experience simple.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003", "REL-001" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00154", "persona": "Media producer", "archetype_id": "personal_checklist", "risk": "low_risk", "query": "My project needs a basic list where I tick completed items for a small team. The first version can stay focused.", "expected_characteristics": [], "detected_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "missing_characteristics": [], "additional_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "expected_packs": [], "matched_packs": [ "DAT-001", "DAT-003", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "low_risk_false_positive" ] }, { "query_id": "LQE-00155", "persona": "Non-technical founder", "archetype_id": "ai_sensitive_summary", "risk": "risk_bearing", "query": "I want a language model that summarizes sensitive application records for a new service. People should understand what happened.", "expected_characteristics": [ "AI_CONTROLLED", "SENSITIVE_DATA" ], "detected_characteristics": [ "AI_CONTROLLED", "UNTRUSTED_INPUT" ], "missing_characteristics": [ "SENSITIVE_DATA" ], "additional_characteristics": [ "UNTRUSTED_INPUT" ], "expected_packs": [ "AI-004", "DAT-001", "DAT-005", "IDN-005", "REL-004" ], "matched_packs": [ "AI-001", "AI-004", "INP-001", "INP-002" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00158", "persona": "Nonprofit program lead", "archetype_id": "external_notifications", "risk": "risk_bearing", "query": "Our organization needs outbound notifications sent to users through an external platform for regular day-to-day use. Keep the experience simple.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003", "REL-001" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00159", "persona": "Healthcare administrator", "archetype_id": "scheduled_automation", "risk": "risk_bearing", "query": "Please help us set up an unattended batch job that processes pending cases for regular day-to-day use. The first version can stay focused.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "HIGH_COST_CONSUMPTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [ "HIGH_COST_CONSUMPTION" ], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-003", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00160", "persona": "Human-resources manager", "archetype_id": "card_payment", "risk": "risk_bearing", "query": "We would use online orders that charge a customer's card as the main workflow. The first version can stay focused.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION" ], "detected_characteristics": [ "FINANCIAL_ACTION" ], "missing_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003", "REL-001" ], "matched_packs": [ "AUT-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00161", "persona": "Machine-learning engineer", "archetype_id": "external_notifications", "risk": "risk_bearing", "query": "Create a workflow that sends customer alerts through a messaging API for a new service. This is the main feature.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003", "REL-001" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00165", "persona": "Human-resources manager", "archetype_id": "unit_converter", "risk": "low_risk", "query": "We would use a small converter for metres and feet for a small team. People should understand what happened.", "expected_characteristics": [], "detected_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "missing_characteristics": [], "additional_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "expected_packs": [], "matched_packs": [ "DAT-001", "DAT-003", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "low_risk_false_positive" ] }, { "query_id": "LQE-00169", "persona": "Enterprise product manager", "archetype_id": "card_payment", "risk": "risk_bearing", "query": "Our next feature is online orders that charge a customer's card inside an existing application. This is the main feature.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION" ], "detected_characteristics": [ "FINANCIAL_ACTION" ], "missing_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003", "REL-001" ], "matched_packs": [ "AUT-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00171", "persona": "Nonprofit program lead", "archetype_id": "webhook", "risk": "risk_bearing", "query": "Our organization needs incoming webhooks from an external billing provider inside an existing application. This is the main feature.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "UNTRUSTED_INPUT" ], "detected_characteristics": [ "FINANCIAL_ACTION" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY", "UNTRUSTED_INPUT" ], "additional_characteristics": [ "FINANCIAL_ACTION" ], "expected_packs": [ "INP-001", "INP-002", "OBS-003", "REL-001" ], "matched_packs": [ "AUT-001" ], "failure_types": [ "expected_characteristic_missing", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00174", "persona": "Nonprofit program lead", "archetype_id": "file_converter", "risk": "risk_bearing", "query": "Our organization needs an image transformer for user-provided files for a small team. It should work consistently.", "expected_characteristics": [ "UNTRUSTED_INPUT" ], "detected_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "missing_characteristics": [ "UNTRUSTED_INPUT" ], "additional_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "expected_packs": [ "INP-001", "INP-002" ], "matched_packs": [ "DAT-001", "DAT-003", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00175", "persona": "Educator", "archetype_id": "external_notifications", "risk": "risk_bearing", "query": "I would like email notifications delivered by an outside service inside an existing application. People should understand what happened.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003", "REL-001" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00177", "persona": "Mobile developer", "archetype_id": "webhook", "risk": "risk_bearing", "query": "The app needs a public webhook receiver for external system events for a new service. People should understand what happened.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "UNTRUSTED_INPUT" ], "detected_characteristics": [ "EXTERNAL_DEPENDENCY" ], "missing_characteristics": [ "UNTRUSTED_INPUT" ], "additional_characteristics": [], "expected_packs": [ "INP-001", "INP-002", "OBS-003", "REL-001" ], "matched_packs": [ "OBS-003", "REL-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00178", "persona": "Security lead", "archetype_id": "card_payment", "risk": "risk_bearing", "query": "Review a design for checkout with card payments through a payment provider as the main workflow. This is the main feature.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION" ], "detected_characteristics": [ "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION" ], "missing_characteristics": [ "CONSEQUENTIAL_ACTION" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003", "REL-001" ], "matched_packs": [ "AUT-001", "OBS-003", "REL-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00179", "persona": "Non-technical founder", "archetype_id": "card_payment", "risk": "risk_bearing", "query": "I want a checkout page that sends charges to a card gateway inside an existing application. This is the main feature.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION" ], "detected_characteristics": [ "FINANCIAL_ACTION" ], "missing_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003", "REL-001" ], "matched_packs": [ "AUT-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00180", "persona": "Operations professional", "archetype_id": "external_sync", "risk": "risk_bearing", "query": "We need daily data sync with an outside business platform as the main workflow. This is the main feature.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "SENSITIVE_DATA" ], "detected_characteristics": [], "missing_characteristics": [ "EXTERNAL_DEPENDENCY", "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [], "failure_types": [ "expected_characteristic_missing", "risk_query_returned_no_pack", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00181", "persona": "Marketplace operator", "archetype_id": "data_export", "risk": "risk_bearing", "query": "We are adding downloadable archives containing customer information for regular day-to-day use. It should work consistently.", "expected_characteristics": [ "DATA_EXPORT", "SENSITIVE_DATA" ], "detected_characteristics": [], "missing_characteristics": [ "DATA_EXPORT", "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-002", "DAT-005", "IDN-005", "REL-004" ], "matched_packs": [], "failure_types": [ "expected_characteristic_missing", "risk_query_returned_no_pack", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00182", "persona": "Human-resources manager", "archetype_id": "card_payment", "risk": "risk_bearing", "query": "We would use a purchase flow connected to an external payment processor as the main workflow. The first version can stay focused.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION" ], "detected_characteristics": [ "EXTERNAL_DEPENDENCY" ], "missing_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003", "REL-001" ], "matched_packs": [ "OBS-003", "REL-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00186", "persona": "Media producer", "archetype_id": "ai_support", "risk": "risk_bearing", "query": "My project needs an assistant that uses AI to reply to support requests for a small team. This is the main feature.", "expected_characteristics": [ "AI_CONTROLLED", "UNTRUSTED_INPUT" ], "detected_characteristics": [ "AI_CONTROLLED", "MULTI_TENANT", "SENSITIVE_DATA", "TOOL_ACCESS" ], "missing_characteristics": [ "UNTRUSTED_INPUT" ], "additional_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA", "TOOL_ACCESS" ], "expected_packs": [ "AI-001", "AI-004", "INP-001", "INP-002" ], "matched_packs": [ "AI-004", "AUT-004", "DAT-001", "DAT-003", "DAT-005", "IDN-003", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00188", "persona": "Machine-learning engineer", "archetype_id": "deployment_agent", "risk": "risk_bearing", "query": "Create a deployment tool that modifies live services for regular day-to-day use. People should understand what happened.", "expected_characteristics": [ "PRIVILEGED_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION" ], "missing_characteristics": [ "PRIVILEGED_ACTION", "PRODUCTION_MODIFICATION" ], "additional_characteristics": [ "CONSEQUENTIAL_ACTION" ], "expected_packs": [ "CHG-001", "IDN-004", "IDN-005", "OBS-002", "REL-004" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00189", "persona": "Enterprise product manager", "archetype_id": "document_upload", "risk": "risk_bearing", "query": "Our next feature is an intake page for customer documents and scanned records as the main workflow. It should work consistently.", "expected_characteristics": [ "SENSITIVE_DATA", "UNTRUSTED_INPUT" ], "detected_characteristics": [], "missing_characteristics": [ "SENSITIVE_DATA", "UNTRUSTED_INPUT" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "INP-001", "INP-002", "REL-004" ], "matched_packs": [], "failure_types": [ "expected_characteristic_missing", "risk_query_returned_no_pack", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00190", "persona": "Data engineer", "archetype_id": "deployment_agent", "risk": "risk_bearing", "query": "Implement automation that applies configuration changes in production for a new service. People should understand what happened.", "expected_characteristics": [ "PRIVILEGED_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION" ], "missing_characteristics": [ "PRIVILEGED_ACTION", "PRODUCTION_MODIFICATION" ], "additional_characteristics": [ "CONSEQUENTIAL_ACTION" ], "expected_packs": [ "CHG-001", "IDN-004", "IDN-005", "OBS-002", "REL-004" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00192", "persona": "Accounting professional", "archetype_id": "external_notifications", "risk": "risk_bearing", "query": "I need a tool with automatic reminders using a third-party communications provider for regular day-to-day use. This is the main feature.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003", "REL-001" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00193", "persona": "Data engineer", "archetype_id": "ai_decision", "risk": "risk_bearing", "query": "Implement an AI recommendation that decides which applications receive priority for a new service. Keep the experience simple.", "expected_characteristics": [ "AI_CONTROLLED", "CONSEQUENTIAL_ACTION" ], "detected_characteristics": [ "AI_CONTROLLED" ], "missing_characteristics": [ "CONSEQUENTIAL_ACTION" ], "additional_characteristics": [], "expected_packs": [ "AI-004", "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "matched_packs": [ "AI-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00194", "persona": "Marketplace operator", "archetype_id": "automated_refund", "risk": "risk_bearing", "query": "We are adding a workflow that calls a billing tool to approve refunds for a new service. It should work consistently.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00195", "persona": "Financial-technology engineer", "archetype_id": "data_export", "risk": "risk_bearing", "query": "The system should provide exports of customer records to downloadable CSV files for a small team. Keep the experience simple.", "expected_characteristics": [ "DATA_EXPORT", "SENSITIVE_DATA" ], "detected_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "missing_characteristics": [ "DATA_EXPORT" ], "additional_characteristics": [ "MULTI_TENANT" ], "expected_packs": [ "DAT-001", "DAT-002", "DAT-005", "IDN-005", "REL-004" ], "matched_packs": [ "DAT-001", "DAT-003", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00196", "persona": "Independent developer", "archetype_id": "payroll", "risk": "risk_bearing", "query": "Build a salary system connected to an outside payroll service for regular day-to-day use. It should work consistently.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "detected_characteristics": [ "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00197", "persona": "Student or learner", "archetype_id": "database_migration", "risk": "risk_bearing", "query": "For a learning project, make schema migrations applied directly to the live database for regular day-to-day use. This is the main feature.", "expected_characteristics": [ "IRREVERSIBLE_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "IRREVERSIBLE_ACTION" ], "missing_characteristics": [ "PRODUCTION_MODIFICATION" ], "additional_characteristics": [ "CONSEQUENTIAL_ACTION" ], "expected_packs": [ "AUT-002", "CHG-001", "REL-004" ], "matched_packs": [ "AUT-001", "AUT-002", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00198", "persona": "Nonprofit program lead", "archetype_id": "permissions", "risk": "risk_bearing", "query": "Our organization needs a screen for granting and removing account access for a new service. It should work consistently.", "expected_characteristics": [ "PRIVILEGED_ACTION" ], "detected_characteristics": [], "missing_characteristics": [ "PRIVILEGED_ACTION" ], "additional_characteristics": [], "expected_packs": [ "IDN-004", "IDN-005" ], "matched_packs": [], "failure_types": [ "expected_characteristic_missing", "risk_query_returned_no_pack", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00202", "persona": "Site reliability engineer", "archetype_id": "webhook", "risk": "risk_bearing", "query": "We are planning incoming webhooks from an external billing provider for a small team. People should understand what happened.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "UNTRUSTED_INPUT" ], "detected_characteristics": [ "FINANCIAL_ACTION", "MULTI_TENANT", "SENSITIVE_DATA" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY", "UNTRUSTED_INPUT" ], "additional_characteristics": [ "FINANCIAL_ACTION", "MULTI_TENANT", "SENSITIVE_DATA" ], "expected_packs": [ "INP-001", "INP-002", "OBS-003", "REL-001" ], "matched_packs": [ "AUT-001", "DAT-001", "DAT-003", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00203", "persona": "Healthcare administrator", "archetype_id": "deployment_agent", "risk": "risk_bearing", "query": "Please help us set up an agent that deploys code and changes production configuration as the main workflow. Keep the experience simple.", "expected_characteristics": [ "PRIVILEGED_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "PRODUCTION_MODIFICATION" ], "missing_characteristics": [ "PRIVILEGED_ACTION" ], "additional_characteristics": [ "CONSEQUENTIAL_ACTION" ], "expected_packs": [ "CHG-001", "IDN-004", "IDN-005", "OBS-002", "REL-004" ], "matched_packs": [ "AUT-001", "AUT-004", "CHG-001", "OBS-001", "OBS-003", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00204", "persona": "Operations professional", "archetype_id": "data_export", "risk": "risk_bearing", "query": "We need downloadable archives containing customer information as the main workflow. People should understand what happened.", "expected_characteristics": [ "DATA_EXPORT", "SENSITIVE_DATA" ], "detected_characteristics": [], "missing_characteristics": [ "DATA_EXPORT", "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-002", "DAT-005", "IDN-005", "REL-004" ], "matched_packs": [], "failure_types": [ "expected_characteristic_missing", "risk_query_returned_no_pack", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00205", "persona": "Healthcare administrator", "archetype_id": "bulk_update", "risk": "risk_bearing", "query": "Please help us set up one command that modifies a large set of records inside an existing application. This is the main feature.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION" ], "detected_characteristics": [], "missing_characteristics": [ "CONSEQUENTIAL_ACTION" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "matched_packs": [], "failure_types": [ "expected_characteristic_missing", "risk_query_returned_no_pack", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00206", "persona": "Site reliability engineer", "archetype_id": "expensive_processing", "risk": "risk_bearing", "query": "We are planning compute-heavy report generation for large datasets for a small team. This is the main feature.", "expected_characteristics": [ "HIGH_COST_CONSUMPTION" ], "detected_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "missing_characteristics": [ "HIGH_COST_CONSUMPTION" ], "additional_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "expected_packs": [ "AUT-003" ], "matched_packs": [ "DAT-001", "DAT-003", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00207", "persona": "Security lead", "archetype_id": "document_upload", "risk": "risk_bearing", "query": "Review a design for uploads for customer contracts and identity documents as the main workflow. The first version can stay focused.", "expected_characteristics": [ "SENSITIVE_DATA", "UNTRUSTED_INPUT" ], "detected_characteristics": [ "SENSITIVE_DATA" ], "missing_characteristics": [ "UNTRUSTED_INPUT" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "INP-001", "INP-002", "REL-004" ], "matched_packs": [ "DAT-001", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00208", "persona": "Independent developer", "archetype_id": "bulk_update", "risk": "risk_bearing", "query": "Build a batch action that changes all active listings for regular day-to-day use. The first version can stay focused.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION" ], "detected_characteristics": [], "missing_characteristics": [ "CONSEQUENTIAL_ACTION" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "matched_packs": [], "failure_types": [ "expected_characteristic_missing", "risk_query_returned_no_pack", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00211", "persona": "Media producer", "archetype_id": "external_sync", "risk": "risk_bearing", "query": "My project needs customer-data synchronization through an external API for regular day-to-day use. Keep the experience simple.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "SENSITIVE_DATA" ], "detected_characteristics": [ "EXTERNAL_DEPENDENCY" ], "missing_characteristics": [ "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "OBS-003", "REL-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00213", "persona": "Accounting professional", "archetype_id": "external_notifications", "risk": "risk_bearing", "query": "I need a tool with email notifications delivered by an outside service for a small team. People should understand what happened.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "MULTI_TENANT", "SENSITIVE_DATA" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "expected_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003", "REL-001" ], "matched_packs": [ "AUT-001", "AUT-004", "DAT-001", "DAT-003", "DAT-005", "IDN-005", "OBS-001", "OBS-003", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00214", "persona": "Media producer", "archetype_id": "payroll", "risk": "risk_bearing", "query": "My project needs employee compensation records and payroll payments as the main workflow. The first version can stay focused.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "detected_characteristics": [ "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00215", "persona": "Enterprise product manager", "archetype_id": "external_sync", "risk": "risk_bearing", "query": "Our next feature is customer-data synchronization through an external API inside an existing application. Keep the experience simple.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "SENSITIVE_DATA" ], "detected_characteristics": [ "EXTERNAL_DEPENDENCY" ], "missing_characteristics": [ "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "OBS-003", "REL-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00217", "persona": "Marketplace operator", "archetype_id": "admin_audit", "risk": "risk_bearing", "query": "We are adding an administrative console that changes account status for a small team. Keep the experience simple.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "PRIVILEGED_ACTION" ], "detected_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "missing_characteristics": [ "CONSEQUENTIAL_ACTION", "PRIVILEGED_ACTION" ], "additional_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "expected_packs": [ "AUT-001", "AUT-004", "IDN-004", "IDN-005", "OBS-001", "OBS-003" ], "matched_packs": [ "DAT-001", "DAT-003", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00220", "persona": "Non-technical founder", "archetype_id": "file_converter", "risk": "risk_bearing", "query": "I want processing arbitrary files submitted through the website for a small team. It should work consistently.", "expected_characteristics": [ "UNTRUSTED_INPUT" ], "detected_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "missing_characteristics": [ "UNTRUSTED_INPUT" ], "additional_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "expected_packs": [ "INP-001", "INP-002" ], "matched_packs": [ "DAT-001", "DAT-003", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00222", "persona": "Operations professional", "archetype_id": "ai_support", "risk": "risk_bearing", "query": "We need an assistant that uses AI to reply to support requests as the main workflow. It should work consistently.", "expected_characteristics": [ "AI_CONTROLLED", "UNTRUSTED_INPUT" ], "detected_characteristics": [ "AI_CONTROLLED", "TOOL_ACCESS" ], "missing_characteristics": [ "UNTRUSTED_INPUT" ], "additional_characteristics": [ "TOOL_ACCESS" ], "expected_packs": [ "AI-001", "AI-004", "INP-001", "INP-002" ], "matched_packs": [ "AI-004", "AUT-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00223", "persona": "Independent developer", "archetype_id": "payroll", "risk": "risk_bearing", "query": "Build payroll processing that sends employee pay to a provider for regular day-to-day use. Keep the experience simple.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "detected_characteristics": [ "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00224", "persona": "Enterprise product manager", "archetype_id": "database_migration", "risk": "risk_bearing", "query": "Our next feature is a live data migration that rewrites existing rows inside an existing application. This is the main feature.", "expected_characteristics": [ "IRREVERSIBLE_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "IRREVERSIBLE_ACTION" ], "missing_characteristics": [ "PRODUCTION_MODIFICATION" ], "additional_characteristics": [ "CONSEQUENTIAL_ACTION" ], "expected_packs": [ "AUT-002", "CHG-001", "REL-004" ], "matched_packs": [ "AUT-001", "AUT-002", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00226", "persona": "Data engineer", "archetype_id": "permanent_delete", "risk": "risk_bearing", "query": "Implement hard deletion of all records selected by an operator inside an existing application. It should work consistently.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "IRREVERSIBLE_ACTION" ], "detected_characteristics": [], "missing_characteristics": [ "CONSEQUENTIAL_ACTION", "IRREVERSIBLE_ACTION" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-002", "AUT-004", "OBS-001", "OBS-003" ], "matched_packs": [], "failure_types": [ "expected_characteristic_missing", "risk_query_returned_no_pack", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00228", "persona": "Data engineer", "archetype_id": "account_login", "risk": "risk_bearing", "query": "Implement customer accounts with passwords and login sessions for a new service. It should work consistently.", "expected_characteristics": [ "SENSITIVE_DATA", "UNTRUSTED_INPUT" ], "detected_characteristics": [ "SENSITIVE_DATA" ], "missing_characteristics": [ "UNTRUSTED_INPUT" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "INP-001", "INP-002", "REL-004" ], "matched_packs": [ "DAT-001", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00229", "persona": "Nonprofit program lead", "archetype_id": "multi_tenant", "risk": "risk_bearing", "query": "Our organization needs a SaaS application where every company has private data inside an existing application. This is the main feature.", "expected_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "detected_characteristics": [], "missing_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-003", "DAT-005", "IDN-005", "REL-004" ], "matched_packs": [], "failure_types": [ "expected_characteristic_missing", "risk_query_returned_no_pack", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00230", "persona": "Accounting professional", "archetype_id": "bulk_update", "risk": "risk_bearing", "query": "I need a tool with an operation that updates thousands of customer records for a small team. People should understand what happened.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION" ], "detected_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "missing_characteristics": [ "CONSEQUENTIAL_ACTION" ], "additional_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "expected_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "matched_packs": [ "DAT-001", "DAT-003", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00231", "persona": "Healthcare administrator", "archetype_id": "document_upload", "risk": "risk_bearing", "query": "Please help us set up uploads for customer contracts and identity documents as the main workflow. It should work consistently.", "expected_characteristics": [ "SENSITIVE_DATA", "UNTRUSTED_INPUT" ], "detected_characteristics": [ "SENSITIVE_DATA" ], "missing_characteristics": [ "UNTRUSTED_INPUT" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "INP-001", "INP-002", "REL-004" ], "matched_packs": [ "DAT-001", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00232", "persona": "Nonprofit program lead", "archetype_id": "expensive_processing", "risk": "risk_bearing", "query": "Our organization needs high-volume document processing that can use significant resources for a small team. The first version can stay focused.", "expected_characteristics": [ "HIGH_COST_CONSUMPTION" ], "detected_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "missing_characteristics": [ "HIGH_COST_CONSUMPTION" ], "additional_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "expected_packs": [ "AUT-003" ], "matched_packs": [ "DAT-001", "DAT-003", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00233", "persona": "Media producer", "archetype_id": "external_sync", "risk": "risk_bearing", "query": "My project needs daily data sync with an outside business platform as the main workflow. This is the main feature.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "SENSITIVE_DATA" ], "detected_characteristics": [], "missing_characteristics": [ "EXTERNAL_DEPENDENCY", "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [], "failure_types": [ "expected_characteristic_missing", "risk_query_returned_no_pack", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00234", "persona": "Media producer", "archetype_id": "automated_refund", "risk": "risk_bearing", "query": "My project needs a bot that sends eligible customers their money back inside an existing application. People should understand what happened.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "TOOL_ACCESS" ], "missing_characteristics": [ "FINANCIAL_ACTION" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00235", "persona": "Small-business owner", "archetype_id": "admin_audit", "risk": "risk_bearing", "query": "Could you make an operator console for consequential account changes as the main workflow. This is the main feature.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "PRIVILEGED_ACTION" ], "detected_characteristics": [], "missing_characteristics": [ "CONSEQUENTIAL_ACTION", "PRIVILEGED_ACTION" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "IDN-004", "IDN-005", "OBS-001", "OBS-003" ], "matched_packs": [], "failure_types": [ "expected_characteristic_missing", "risk_query_returned_no_pack", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00236", "persona": "Machine-learning engineer", "archetype_id": "api_crawler", "risk": "risk_bearing", "query": "Create a scheduled importer that can consume an outside API quota for a new service. It should work consistently.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "HIGH_COST_CONSUMPTION" ], "detected_characteristics": [ "HIGH_COST_CONSUMPTION" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-003", "OBS-003", "REL-001", "REL-003" ], "matched_packs": [ "AUT-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00237", "persona": "Nonprofit program lead", "archetype_id": "calculator", "risk": "low_risk", "query": "Our organization needs a basic tip calculator with no saved information for a small team. The first version can stay focused.", "expected_characteristics": [], "detected_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "missing_characteristics": [], "additional_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "expected_packs": [], "matched_packs": [ "DAT-001", "DAT-003", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "low_risk_false_positive" ] }, { "query_id": "LQE-00238", "persona": "Mobile developer", "archetype_id": "deployment_agent", "risk": "risk_bearing", "query": "The app needs a release agent with permission to update production systems for a small team. People should understand what happened.", "expected_characteristics": [ "PRIVILEGED_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "MULTI_TENANT", "PRODUCTION_MODIFICATION", "SENSITIVE_DATA", "TOOL_ACCESS" ], "missing_characteristics": [ "PRIVILEGED_ACTION" ], "additional_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA", "TOOL_ACCESS" ], "expected_packs": [ "CHG-001", "IDN-004", "IDN-005", "OBS-002", "REL-004" ], "matched_packs": [ "AUT-004", "CHG-001", "DAT-001", "DAT-003", "DAT-005", "IDN-003", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00241", "persona": "Operations professional", "archetype_id": "public_submission", "risk": "risk_bearing", "query": "We need a contact form that receives free-text messages for a small team. It should work consistently.", "expected_characteristics": [ "UNTRUSTED_INPUT" ], "detected_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "missing_characteristics": [ "UNTRUSTED_INPUT" ], "additional_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "expected_packs": [ "INP-001", "INP-002" ], "matched_packs": [ "DAT-001", "DAT-003", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00243", "persona": "Site reliability engineer", "archetype_id": "payroll", "risk": "risk_bearing", "query": "We are planning employee compensation records and payroll payments as the main workflow. Keep the experience simple.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "detected_characteristics": [ "FINANCIAL_ACTION", "SENSITIVE_DATA" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "AUT-001", "DAT-001", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00244", "persona": "Security lead", "archetype_id": "external_sync", "risk": "risk_bearing", "query": "Review a design for daily data sync with an outside business platform as the main workflow. Keep the experience simple.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "SENSITIVE_DATA" ], "detected_characteristics": [], "missing_characteristics": [ "EXTERNAL_DEPENDENCY", "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [], "failure_types": [ "expected_characteristic_missing", "risk_query_returned_no_pack", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00246", "persona": "Site reliability engineer", "archetype_id": "external_sync", "risk": "risk_bearing", "query": "We are planning daily data sync with an outside business platform as the main workflow. The first version can stay focused.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "SENSITIVE_DATA" ], "detected_characteristics": [], "missing_characteristics": [ "EXTERNAL_DEPENDENCY", "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [], "failure_types": [ "expected_characteristic_missing", "risk_query_returned_no_pack", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00247", "persona": "Financial-technology engineer", "archetype_id": "countdown", "risk": "low_risk", "query": "The system should provide a timer that counts down from a chosen number for a small team. Keep the experience simple.", "expected_characteristics": [], "detected_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "missing_characteristics": [], "additional_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "expected_packs": [], "matched_packs": [ "DAT-001", "DAT-003", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "low_risk_false_positive" ] }, { "query_id": "LQE-00248", "persona": "Accounting professional", "archetype_id": "document_upload", "risk": "risk_bearing", "query": "I need a tool with a portal where people upload confidential PDF forms for a new service. This is the main feature.", "expected_characteristics": [ "SENSITIVE_DATA", "UNTRUSTED_INPUT" ], "detected_characteristics": [], "missing_characteristics": [ "SENSITIVE_DATA", "UNTRUSTED_INPUT" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "INP-001", "INP-002", "REL-004" ], "matched_packs": [], "failure_types": [ "expected_characteristic_missing", "risk_query_returned_no_pack", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00250", "persona": "Human-resources manager", "archetype_id": "permissions", "risk": "risk_bearing", "query": "We would use a screen for granting and removing account access for a small team. It should work consistently.", "expected_characteristics": [ "PRIVILEGED_ACTION" ], "detected_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "missing_characteristics": [ "PRIVILEGED_ACTION" ], "additional_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "expected_packs": [ "IDN-004", "IDN-005" ], "matched_packs": [ "DAT-001", "DAT-003", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00251", "persona": "Small-business owner", "archetype_id": "data_export", "risk": "risk_bearing", "query": "Could you make bulk export of private account data for regular day-to-day use. The first version can stay focused.", "expected_characteristics": [ "DATA_EXPORT", "SENSITIVE_DATA" ], "detected_characteristics": [], "missing_characteristics": [ "DATA_EXPORT", "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-002", "DAT-005", "IDN-005", "REL-004" ], "matched_packs": [], "failure_types": [ "expected_characteristic_missing", "risk_query_returned_no_pack", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00254", "persona": "Security lead", "archetype_id": "webhook", "risk": "risk_bearing", "query": "Review a design for an integration that receives callbacks from a vendor for a small team. It should work consistently.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "UNTRUSTED_INPUT" ], "detected_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY", "UNTRUSTED_INPUT" ], "additional_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "expected_packs": [ "INP-001", "INP-002", "OBS-003", "REL-001" ], "matched_packs": [ "DAT-001", "DAT-003", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00255", "persona": "Media producer", "archetype_id": "ai_sensitive_summary", "risk": "risk_bearing", "query": "My project needs AI summaries of confidential case notes for regular day-to-day use. Keep the experience simple.", "expected_characteristics": [ "AI_CONTROLLED", "SENSITIVE_DATA" ], "detected_characteristics": [ "AI_CONTROLLED" ], "missing_characteristics": [ "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "AI-004", "DAT-001", "DAT-005", "IDN-005", "REL-004" ], "matched_packs": [ "AI-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00256", "persona": "Student or learner", "archetype_id": "scheduled_automation", "risk": "risk_bearing", "query": "For a learning project, make an unattended batch job that processes pending cases for a new service. It should work consistently.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "HIGH_COST_CONSUMPTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [ "HIGH_COST_CONSUMPTION" ], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-003", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00257", "persona": "Non-technical founder", "archetype_id": "document_upload", "risk": "risk_bearing", "query": "I want a feature that accepts sensitive files from outside users for a small team. The first version can stay focused.", "expected_characteristics": [ "SENSITIVE_DATA", "UNTRUSTED_INPUT" ], "detected_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "missing_characteristics": [ "UNTRUSTED_INPUT" ], "additional_characteristics": [ "MULTI_TENANT" ], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "INP-001", "INP-002", "REL-004" ], "matched_packs": [ "DAT-001", "DAT-003", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00258", "persona": "Marketplace operator", "archetype_id": "file_converter", "risk": "risk_bearing", "query": "We are adding a service that opens and converts outside documents for regular day-to-day use. Keep the experience simple.", "expected_characteristics": [ "UNTRUSTED_INPUT" ], "detected_characteristics": [], "missing_characteristics": [ "UNTRUSTED_INPUT" ], "additional_characteristics": [], "expected_packs": [ "INP-001", "INP-002" ], "matched_packs": [], "failure_types": [ "expected_characteristic_missing", "risk_query_returned_no_pack", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00260", "persona": "Healthcare administrator", "archetype_id": "api_crawler", "risk": "risk_bearing", "query": "Please help us set up a scheduled importer that can consume an outside API quota as the main workflow. This is the main feature.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "HIGH_COST_CONSUMPTION" ], "detected_characteristics": [ "HIGH_COST_CONSUMPTION" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-003", "OBS-003", "REL-001", "REL-003" ], "matched_packs": [ "AUT-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00261", "persona": "Enterprise product manager", "archetype_id": "deployment_agent", "risk": "risk_bearing", "query": "Our next feature is a deployment tool that modifies live services for a small team. The first version can stay focused.", "expected_characteristics": [ "PRIVILEGED_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "MULTI_TENANT", "SENSITIVE_DATA" ], "missing_characteristics": [ "PRIVILEGED_ACTION", "PRODUCTION_MODIFICATION" ], "additional_characteristics": [ "CONSEQUENTIAL_ACTION", "MULTI_TENANT", "SENSITIVE_DATA" ], "expected_packs": [ "CHG-001", "IDN-004", "IDN-005", "OBS-002", "REL-004" ], "matched_packs": [ "AUT-001", "AUT-004", "DAT-001", "DAT-003", "DAT-005", "IDN-005", "OBS-001", "OBS-003", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00262", "persona": "Human-resources manager", "archetype_id": "external_notifications", "risk": "risk_bearing", "query": "We would use outbound notifications sent to users through an external platform as the main workflow. Keep the experience simple.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003", "REL-001" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00264", "persona": "Student or learner", "archetype_id": "card_payment", "risk": "risk_bearing", "query": "For a learning project, make checkout with card payments through a payment provider for a small team. It should work consistently.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION" ], "detected_characteristics": [ "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION", "MULTI_TENANT", "SENSITIVE_DATA" ], "missing_characteristics": [ "CONSEQUENTIAL_ACTION" ], "additional_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "expected_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003", "REL-001" ], "matched_packs": [ "AUT-001", "DAT-001", "DAT-003", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00267", "persona": "Non-technical founder", "archetype_id": "admin_audit", "risk": "risk_bearing", "query": "I want staff tools for suspending and restoring user accounts for a small team. People should understand what happened.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "PRIVILEGED_ACTION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "IRREVERSIBLE_ACTION", "MULTI_TENANT", "SENSITIVE_DATA" ], "missing_characteristics": [ "PRIVILEGED_ACTION" ], "additional_characteristics": [ "IRREVERSIBLE_ACTION", "MULTI_TENANT", "SENSITIVE_DATA" ], "expected_packs": [ "AUT-001", "AUT-004", "IDN-004", "IDN-005", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-002", "AUT-004", "DAT-001", "DAT-003", "DAT-004", "DAT-005", "IDN-005", "OBS-001", "OBS-003", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00268", "persona": "Nonprofit program lead", "archetype_id": "automated_refund", "risk": "risk_bearing", "query": "Our organization needs a workflow that calls a billing tool to approve refunds for regular day-to-day use. It should work consistently.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00269", "persona": "Machine-learning engineer", "archetype_id": "admin_audit", "risk": "risk_bearing", "query": "Create an operator console for consequential account changes as the main workflow. It should work consistently.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "PRIVILEGED_ACTION" ], "detected_characteristics": [], "missing_characteristics": [ "CONSEQUENTIAL_ACTION", "PRIVILEGED_ACTION" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "IDN-004", "IDN-005", "OBS-001", "OBS-003" ], "matched_packs": [], "failure_types": [ "expected_characteristic_missing", "risk_query_returned_no_pack", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00271", "persona": "Independent developer", "archetype_id": "ai_sensitive_summary", "risk": "risk_bearing", "query": "Build a model that summarizes employee records for a new service. People should understand what happened.", "expected_characteristics": [ "AI_CONTROLLED", "SENSITIVE_DATA" ], "detected_characteristics": [ "SENSITIVE_DATA" ], "missing_characteristics": [ "AI_CONTROLLED" ], "additional_characteristics": [], "expected_packs": [ "AI-004", "DAT-001", "DAT-005", "IDN-005", "REL-004" ], "matched_packs": [ "DAT-001", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00272", "persona": "Non-technical founder", "archetype_id": "document_upload", "risk": "risk_bearing", "query": "I want a feature that accepts sensitive files from outside users as the main workflow. People should understand what happened.", "expected_characteristics": [ "SENSITIVE_DATA", "UNTRUSTED_INPUT" ], "detected_characteristics": [], "missing_characteristics": [ "SENSITIVE_DATA", "UNTRUSTED_INPUT" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "INP-001", "INP-002", "REL-004" ], "matched_packs": [], "failure_types": [ "expected_characteristic_missing", "risk_query_returned_no_pack", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00273", "persona": "Enterprise product manager", "archetype_id": "deployment_agent", "risk": "risk_bearing", "query": "Our next feature is a release agent with permission to update production systems as the main workflow. Keep the experience simple.", "expected_characteristics": [ "PRIVILEGED_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "PRODUCTION_MODIFICATION", "TOOL_ACCESS" ], "missing_characteristics": [ "PRIVILEGED_ACTION" ], "additional_characteristics": [ "TOOL_ACCESS" ], "expected_packs": [ "CHG-001", "IDN-004", "IDN-005", "OBS-002", "REL-004" ], "matched_packs": [ "AUT-004", "CHG-001", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00274", "persona": "Security lead", "archetype_id": "ai_sensitive_summary", "risk": "risk_bearing", "query": "Review a design for a model that summarizes employee records for a new service. It should work consistently.", "expected_characteristics": [ "AI_CONTROLLED", "SENSITIVE_DATA" ], "detected_characteristics": [ "SENSITIVE_DATA" ], "missing_characteristics": [ "AI_CONTROLLED" ], "additional_characteristics": [], "expected_packs": [ "AI-004", "DAT-001", "DAT-005", "IDN-005", "REL-004" ], "matched_packs": [ "DAT-001", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00275", "persona": "Student or learner", "archetype_id": "data_export", "risk": "risk_bearing", "query": "For a learning project, make exports of customer records to downloadable CSV files for a small team. This is the main feature.", "expected_characteristics": [ "DATA_EXPORT", "SENSITIVE_DATA" ], "detected_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "missing_characteristics": [ "DATA_EXPORT" ], "additional_characteristics": [ "MULTI_TENANT" ], "expected_packs": [ "DAT-001", "DAT-002", "DAT-005", "IDN-005", "REL-004" ], "matched_packs": [ "DAT-001", "DAT-003", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00276", "persona": "Non-technical founder", "archetype_id": "automated_refund", "risk": "risk_bearing", "query": "I want automatic refund processing without manual entry inside an existing application. Keep the experience simple.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00281", "persona": "Machine-learning engineer", "archetype_id": "automated_refund", "risk": "risk_bearing", "query": "Create a workflow that calls a billing tool to approve refunds for a new service. Keep the experience simple.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00283", "persona": "Financial-technology engineer", "archetype_id": "external_sync", "risk": "risk_bearing", "query": "The system should provide a connection that sends private records to a third-party service inside an existing application. This is the main feature.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "SENSITIVE_DATA" ], "detected_characteristics": [ "EXTERNAL_DEPENDENCY" ], "missing_characteristics": [ "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "OBS-003", "REL-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00284", "persona": "Data engineer", "archetype_id": "admin_audit", "risk": "risk_bearing", "query": "Implement an administrative console that changes account status inside an existing application. The first version can stay focused.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "PRIVILEGED_ACTION" ], "detected_characteristics": [], "missing_characteristics": [ "CONSEQUENTIAL_ACTION", "PRIVILEGED_ACTION" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "IDN-004", "IDN-005", "OBS-001", "OBS-003" ], "matched_packs": [], "failure_types": [ "expected_characteristic_missing", "risk_query_returned_no_pack", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00286", "persona": "Non-technical founder", "archetype_id": "automated_refund", "risk": "risk_bearing", "query": "I want a workflow that calls a billing tool to approve refunds for regular day-to-day use. It should work consistently.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00287", "persona": "Government analyst", "archetype_id": "expensive_processing", "risk": "risk_bearing", "query": "The proposed service includes high-volume document processing that can use significant resources as the main workflow. People should understand what happened.", "expected_characteristics": [ "HIGH_COST_CONSUMPTION" ], "detected_characteristics": [], "missing_characteristics": [ "HIGH_COST_CONSUMPTION" ], "additional_characteristics": [], "expected_packs": [ "AUT-003" ], "matched_packs": [], "failure_types": [ "expected_characteristic_missing", "risk_query_returned_no_pack", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00288", "persona": "Human-resources manager", "archetype_id": "card_payment", "risk": "risk_bearing", "query": "We would use checkout with card payments through a payment provider for a new service. The first version can stay focused.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION" ], "detected_characteristics": [ "EXTERNAL_DEPENDENCY", "FINANCIAL_ACTION" ], "missing_characteristics": [ "CONSEQUENTIAL_ACTION" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003", "REL-001" ], "matched_packs": [ "AUT-001", "OBS-003", "REL-001" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00289", "persona": "Financial-technology engineer", "archetype_id": "scheduled_automation", "risk": "risk_bearing", "query": "The system should provide a background workflow that updates records every morning for a new service. This is the main feature.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00291", "persona": "Student or learner", "archetype_id": "database_migration", "risk": "risk_bearing", "query": "For a learning project, make production database changes with a difficult rollback for regular day-to-day use. This is the main feature.", "expected_characteristics": [ "IRREVERSIBLE_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "PRIVILEGED_ACTION", "PRODUCTION_MODIFICATION" ], "missing_characteristics": [ "IRREVERSIBLE_ACTION" ], "additional_characteristics": [ "CONSEQUENTIAL_ACTION", "PRIVILEGED_ACTION" ], "expected_packs": [ "AUT-002", "CHG-001", "REL-004" ], "matched_packs": [ "AUT-001", "AUT-004", "CHG-001", "IDN-004", "IDN-005", "OBS-001", "OBS-002", "OBS-003", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00292", "persona": "Enterprise product manager", "archetype_id": "deployment_agent", "risk": "risk_bearing", "query": "Our next feature is automatic production releases with infrastructure updates for a small team. Keep the experience simple.", "expected_characteristics": [ "PRIVILEGED_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "MULTI_TENANT", "PRODUCTION_MODIFICATION", "SENSITIVE_DATA" ], "missing_characteristics": [ "PRIVILEGED_ACTION" ], "additional_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "expected_packs": [ "CHG-001", "IDN-004", "IDN-005", "OBS-002", "REL-004" ], "matched_packs": [ "CHG-001", "DAT-001", "DAT-003", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00293", "persona": "Site reliability engineer", "archetype_id": "automated_refund", "risk": "risk_bearing", "query": "We are planning a service that triggers refunds through an external function inside an existing application. This is the main feature.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00294", "persona": "Security lead", "archetype_id": "ai_decision", "risk": "risk_bearing", "query": "Review a design for an AI feature that determines whether an account is restricted for a small team. This is the main feature.", "expected_characteristics": [ "AI_CONTROLLED", "CONSEQUENTIAL_ACTION" ], "detected_characteristics": [ "AI_CONTROLLED", "MULTI_TENANT", "SENSITIVE_DATA" ], "missing_characteristics": [ "CONSEQUENTIAL_ACTION" ], "additional_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "expected_packs": [ "AI-004", "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "matched_packs": [ "AI-004", "DAT-001", "DAT-003", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00295", "persona": "Data engineer", "archetype_id": "personal_checklist", "risk": "low_risk", "query": "Implement a local checklist that stays in the browser for a small team. Keep the experience simple.", "expected_characteristics": [], "detected_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "missing_characteristics": [], "additional_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "expected_packs": [], "matched_packs": [ "DAT-001", "DAT-003", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "low_risk_false_positive" ] }, { "query_id": "LQE-00297", "persona": "Educator", "archetype_id": "external_sync", "risk": "risk_bearing", "query": "I would like synchronization of customer records with an external CRM API for a small team. This is the main feature.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "SENSITIVE_DATA" ], "detected_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY" ], "additional_characteristics": [ "MULTI_TENANT" ], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "OBS-003", "REL-001", "REL-004" ], "matched_packs": [ "DAT-001", "DAT-003", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00298", "persona": "Small-business owner", "archetype_id": "document_upload", "risk": "risk_bearing", "query": "Could you make an intake page for customer documents and scanned records inside an existing application. People should understand what happened.", "expected_characteristics": [ "SENSITIVE_DATA", "UNTRUSTED_INPUT" ], "detected_characteristics": [], "missing_characteristics": [ "SENSITIVE_DATA", "UNTRUSTED_INPUT" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "INP-001", "INP-002", "REL-004" ], "matched_packs": [], "failure_types": [ "expected_characteristic_missing", "risk_query_returned_no_pack", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00300", "persona": "Healthcare administrator", "archetype_id": "multi_tenant", "risk": "risk_bearing", "query": "Please help us set up separate customer accounts sharing the same application and database for regular day-to-day use. People should understand what happened.", "expected_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "detected_characteristics": [], "missing_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-003", "DAT-005", "IDN-005", "REL-004" ], "matched_packs": [], "failure_types": [ "expected_characteristic_missing", "risk_query_returned_no_pack", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00302", "persona": "Independent developer", "archetype_id": "expensive_processing", "risk": "risk_bearing", "query": "Build high-volume document processing that can use significant resources for a small team. Keep the experience simple.", "expected_characteristics": [ "HIGH_COST_CONSUMPTION" ], "detected_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "missing_characteristics": [ "HIGH_COST_CONSUMPTION" ], "additional_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "expected_packs": [ "AUT-003" ], "matched_packs": [ "DAT-001", "DAT-003", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00303", "persona": "Financial-technology engineer", "archetype_id": "account_login", "risk": "risk_bearing", "query": "The system should provide customer accounts with passwords and login sessions as the main workflow. It should work consistently.", "expected_characteristics": [ "SENSITIVE_DATA", "UNTRUSTED_INPUT" ], "detected_characteristics": [ "SENSITIVE_DATA" ], "missing_characteristics": [ "UNTRUSTED_INPUT" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "INP-001", "INP-002", "REL-004" ], "matched_packs": [ "DAT-001", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00304", "persona": "Student or learner", "archetype_id": "document_upload", "risk": "risk_bearing", "query": "For a learning project, make uploads for customer contracts and identity documents inside an existing application. Keep the experience simple.", "expected_characteristics": [ "SENSITIVE_DATA", "UNTRUSTED_INPUT" ], "detected_characteristics": [ "SENSITIVE_DATA" ], "missing_characteristics": [ "UNTRUSTED_INPUT" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-005", "IDN-005", "INP-001", "INP-002", "REL-004" ], "matched_packs": [ "DAT-001", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00305", "persona": "Marketplace operator", "archetype_id": "multi_tenant", "risk": "risk_bearing", "query": "We are adding software for many organizations with isolated workspaces for a new service. This is the main feature.", "expected_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "detected_characteristics": [], "missing_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-003", "DAT-005", "IDN-005", "REL-004" ], "matched_packs": [], "failure_types": [ "expected_characteristic_missing", "risk_query_returned_no_pack", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00306", "persona": "Enterprise product manager", "archetype_id": "multi_tenant", "risk": "risk_bearing", "query": "Our next feature is separate customer accounts sharing the same application and database for a new service. The first version can stay focused.", "expected_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "detected_characteristics": [], "missing_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "additional_characteristics": [], "expected_packs": [ "DAT-001", "DAT-003", "DAT-005", "IDN-005", "REL-004" ], "matched_packs": [], "failure_types": [ "expected_characteristic_missing", "risk_query_returned_no_pack", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00308", "persona": "Government analyst", "archetype_id": "webhook", "risk": "risk_bearing", "query": "The proposed service includes an integration that receives callbacks from a vendor for a small team. The first version can stay focused.", "expected_characteristics": [ "EXTERNAL_DEPENDENCY", "UNTRUSTED_INPUT" ], "detected_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "missing_characteristics": [ "EXTERNAL_DEPENDENCY", "UNTRUSTED_INPUT" ], "additional_characteristics": [ "MULTI_TENANT", "SENSITIVE_DATA" ], "expected_packs": [ "INP-001", "INP-002", "OBS-003", "REL-001" ], "matched_packs": [ "DAT-001", "DAT-003", "DAT-005", "IDN-005", "REL-004" ], "failure_types": [ "expected_characteristic_missing", "expected_pack_family_not_reached" ] }, { "query_id": "LQE-00309", "persona": "Financial-technology engineer", "archetype_id": "deployment_agent", "risk": "risk_bearing", "query": "The system should provide an agent that deploys code and changes production configuration for a new service. This is the main feature.", "expected_characteristics": [ "PRIVILEGED_ACTION", "PRODUCTION_MODIFICATION" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "PRODUCTION_MODIFICATION" ], "missing_characteristics": [ "PRIVILEGED_ACTION" ], "additional_characteristics": [ "CONSEQUENTIAL_ACTION" ], "expected_packs": [ "CHG-001", "IDN-004", "IDN-005", "OBS-002", "REL-004" ], "matched_packs": [ "AUT-001", "AUT-004", "CHG-001", "OBS-001", "OBS-003", "REL-004" ], "failure_types": [ "expected_characteristic_missing" ] }, { "query_id": "LQE-00310", "persona": "Independent developer", "archetype_id": "automated_refund", "risk": "risk_bearing", "query": "Build automatic refund processing without manual entry inside an existing application. Keep the experience simple.", "expected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION", "TOOL_ACCESS" ], "detected_characteristics": [ "CONSEQUENTIAL_ACTION", "FINANCIAL_ACTION" ], "missing_characteristics": [ "TOOL_ACCESS" ], "additional_characteristics": [], "expected_packs": [ "AUT-001", "AUT-004", "IDN-001", "OBS-001", "OBS-003" ], "matched_packs": [ "AUT-001", "AUT-004", "OBS-001", "OBS-003" ], "failure_types": [ "expected_characteristic_missing" ] } ], "failure_example_limit": 200, "limitations": [ "All requests are synthetic and are not evidence of users, adoption, market demand, or real-world accuracy.", "Expected characteristics were authored at the scenario-archetype level by the project and have not been independently adjudicated.", "Expected Control Packs are derived by applying the published pack rules to the authored expected characteristics; this tests detection-to-pack continuity, not whether the pack library is normatively complete.", "Generated wording variants can expose brittleness, but repeated variants are not statistically independent observations.", "Results should be interpreted alongside the smaller hand-authored reference benchmarks, not as a replacement for them." ], "interpretation": "A large synthetic stress test of deterministic detection and pack continuity. It is not an adoption count, representative user sample, independent safety evaluation, or real-world accuracy estimate." }