{
  "evaluation_id": "large-query-evaluation-v1",
  "version": "1.0.0",
  "methodology_version": "0.1.0",
  "seed": 20260814,
  "target_query_count": 25000,
  "description": "Deterministically generated, synthetic first-line project descriptions used to stress-test characteristic detection and Control Pack applicability across varied wording and user backgrounds.",
  "limitations": [
    "All requests are synthetic and are not evidence of users, adoption, market demand, or real-world accuracy.",
    "Expected characteristics were authored at the scenario-archetype level by the project and have not been independently adjudicated.",
    "Expected Control Packs are derived by applying the published pack rules to the authored expected characteristics; this tests detection-to-pack continuity, not whether the pack library is normatively complete.",
    "Generated wording variants can expose brittleness, but repeated variants are not statistically independent observations.",
    "Results should be interpreted alongside the smaller hand-authored reference benchmarks, not as a replacement for them."
  ],
  "personas": [
    {"id": "nontechnical_founder", "label": "Non-technical founder", "pattern": "I want {request}"},
    {"id": "small_business_owner", "label": "Small-business owner", "pattern": "Could you make {request}"},
    {"id": "indie_developer", "label": "Independent developer", "pattern": "Build {request}"},
    {"id": "enterprise_product", "label": "Enterprise product manager", "pattern": "Our next feature is {request}"},
    {"id": "operations", "label": "Operations professional", "pattern": "We need {request}"},
    {"id": "data_engineer", "label": "Data engineer", "pattern": "Implement {request}"},
    {"id": "ml_engineer", "label": "Machine-learning engineer", "pattern": "Create {request}"},
    {"id": "healthcare_admin", "label": "Healthcare administrator", "pattern": "Please help us set up {request}"},
    {"id": "sre", "label": "Site reliability engineer", "pattern": "We are planning {request}"},
    {"id": "mobile_developer", "label": "Mobile developer", "pattern": "The app needs {request}"},
    {"id": "government_analyst", "label": "Government analyst", "pattern": "The proposed service includes {request}"},
    {"id": "fintech_engineer", "label": "Financial-technology engineer", "pattern": "The system should provide {request}"},
    {"id": "security_lead", "label": "Security lead", "pattern": "Review a design for {request}"},
    {"id": "educator", "label": "Educator", "pattern": "I would like {request}"},
    {"id": "marketplace_operator", "label": "Marketplace operator", "pattern": "We are adding {request}"},
    {"id": "media_producer", "label": "Media producer", "pattern": "My project needs {request}"},
    {"id": "student", "label": "Student or learner", "pattern": "For a learning project, make {request}"},
    {"id": "nonprofit_lead", "label": "Nonprofit program lead", "pattern": "Our organization needs {request}"},
    {"id": "accountant", "label": "Accounting professional", "pattern": "I need a tool with {request}"},
    {"id": "hr_manager", "label": "Human-resources manager", "pattern": "We would use {request}"}
  ],
  "scope_variants": [
    "for a new service",
    "for a small team",
    "as the main workflow",
    "inside an existing application",
    "for regular day-to-day use"
  ],
  "detail_variants": [
    "Keep the experience simple",
    "This is the main feature",
    "It should work consistently",
    "People should understand what happened",
    "The first version can stay focused"
  ],
  "archetypes": [
    {"id":"ai_support","risk":"risk_bearing","expected_characteristics":["AI_CONTROLLED","UNTRUSTED_INPUT"],"requests":["an AI support assistant that answers customer messages","a customer-service chatbot that responds to incoming questions","a model-powered help desk for messages from customers","an assistant that uses AI to reply to support requests","a support inbox where a language model drafts answers"]},
    {"id":"ai_sensitive_summary","risk":"risk_bearing","expected_characteristics":["AI_CONTROLLED","SENSITIVE_DATA"],"requests":["AI summaries of confidential case notes","a model that summarizes employee records","an AI feature that condenses private client files","automatic model-generated summaries of internal investigations","a language model that summarizes sensitive application records"]},
    {"id":"ai_decision","risk":"risk_bearing","expected_characteristics":["AI_CONTROLLED","CONSEQUENTIAL_ACTION"],"requests":["an AI recommendation that decides which applications receive priority","a model-assisted workflow that approves or rejects submitted requests","AI scoring that changes which cases move forward","a model that selects who receives a limited benefit","an AI feature that determines whether an account is restricted"]},
    {"id":"document_upload","risk":"risk_bearing","expected_characteristics":["UNTRUSTED_INPUT","SENSITIVE_DATA"],"requests":["uploads for customer contracts and identity documents","a portal where people upload confidential PDF forms","file uploads containing private client paperwork","an intake page for customer documents and scanned records","a feature that accepts sensitive files from outside users"]},
    {"id":"public_submission","risk":"risk_bearing","expected_characteristics":["UNTRUSTED_INPUT"],"requests":["a public form where visitors submit comments","a community page that accepts user posts","a feedback box open to anyone on the internet","a contact form that receives free-text messages","a public listing where users provide descriptions"]},
    {"id":"card_payment","risk":"risk_bearing","expected_characteristics":["FINANCIAL_ACTION","CONSEQUENTIAL_ACTION","EXTERNAL_DEPENDENCY"],"requests":["checkout with card payments through a payment provider","online orders that charge a customer's card","a purchase flow connected to an external payment processor","subscription billing that collects payments","a checkout page that sends charges to a card gateway"]},
    {"id":"automated_refund","risk":"risk_bearing","expected_characteristics":["FINANCIAL_ACTION","CONSEQUENTIAL_ACTION","TOOL_ACCESS"],"requests":["automatic refunds issued through the payment API","a bot that sends eligible customers their money back","a workflow that calls a billing tool to approve refunds","automatic refund processing without manual entry","a service that triggers refunds through an external function"]},
    {"id":"payroll","risk":"risk_bearing","expected_characteristics":["FINANCIAL_ACTION","SENSITIVE_DATA","EXTERNAL_DEPENDENCY"],"requests":["payroll processing that sends employee pay to a provider","a salary system connected to an outside payroll service","employee compensation records and payroll payments","a tool that calculates pay and submits it to a payroll vendor","monthly payroll with employee banking and tax details"]},
    {"id":"permissions","risk":"risk_bearing","expected_characteristics":["PRIVILEGED_ACTION"],"requests":["administrators changing staff roles and permissions","a screen for granting and removing account access","role-based access managed by privileged operators","an admin tool that changes what users are allowed to do","permission management for staff and contractors"]},
    {"id":"bulk_update","risk":"risk_bearing","expected_characteristics":["CONSEQUENTIAL_ACTION"],"requests":["a bulk update that changes every product price","an operation that updates thousands of customer records","a batch action that changes all active listings","one command that modifies a large set of records","a mass update across the entire catalogue"]},
    {"id":"permanent_delete","risk":"risk_bearing","expected_characteristics":["IRREVERSIBLE_ACTION","CONSEQUENTIAL_ACTION"],"requests":["permanent deletion of customer accounts and records","a button that irreversibly removes stored projects","hard deletion of all records selected by an operator","an action that permanently erases uploaded documents","a destructive cleanup that cannot be undone"]},
    {"id":"scheduled_automation","risk":"risk_bearing","expected_characteristics":["CONSEQUENTIAL_ACTION","TOOL_ACCESS"],"requests":["a nightly job that reconciles and updates invoices","scheduled automation that changes order statuses","a background workflow that updates records every morning","an unattended batch job that processes pending cases","automatic overnight reconciliation through internal tools"]},
    {"id":"external_notifications","risk":"risk_bearing","expected_characteristics":["CONSEQUENTIAL_ACTION","EXTERNAL_DEPENDENCY"],"requests":["automatic text messages sent through an SMS provider","email notifications delivered by an outside service","a workflow that sends customer alerts through a messaging API","automatic reminders using a third-party communications provider","outbound notifications sent to users through an external platform"]},
    {"id":"multi_tenant","risk":"risk_bearing","expected_characteristics":["MULTI_TENANT","SENSITIVE_DATA"],"requests":["a multi-tenant CRM that keeps each customer's records separate","software for many organizations with isolated workspaces","a SaaS application where every company has private data","separate customer accounts sharing the same application and database","a multi-customer portal with organization-specific records"]},
    {"id":"data_export","risk":"risk_bearing","expected_characteristics":["DATA_EXPORT","SENSITIVE_DATA"],"requests":["exports of customer records to downloadable CSV files","a feature that downloads employee information","bulk export of private account data","a report that sends confidential records outside the application","downloadable archives containing customer information"]},
    {"id":"external_sync","risk":"risk_bearing","expected_characteristics":["EXTERNAL_DEPENDENCY","SENSITIVE_DATA"],"requests":["synchronization of customer records with an external CRM API","a connection that sends private records to a third-party service","daily data sync with an outside business platform","an integration that exchanges client information with a vendor","customer-data synchronization through an external API"]},
    {"id":"deployment_agent","risk":"risk_bearing","expected_characteristics":["PRODUCTION_MODIFICATION","PRIVILEGED_ACTION"],"requests":["an agent that deploys code and changes production configuration","automatic production releases with infrastructure updates","a deployment tool that modifies live services","automation that applies configuration changes in production","a release agent with permission to update production systems"]},
    {"id":"database_migration","risk":"risk_bearing","expected_characteristics":["PRODUCTION_MODIFICATION","IRREVERSIBLE_ACTION"],"requests":["a production database migration that changes stored records","schema migrations applied directly to the live database","a live data migration that rewrites existing rows","production database changes with a difficult rollback","an operation that transforms records in the production database"]},
    {"id":"expensive_processing","risk":"risk_bearing","expected_characteristics":["HIGH_COST_CONSUMPTION"],"requests":["large video-processing jobs that consume substantial compute","a batch image pipeline with expensive GPU usage","high-volume document processing that can use significant resources","compute-heavy report generation for large datasets","resource-intensive media conversion jobs"]},
    {"id":"api_crawler","risk":"risk_bearing","expected_characteristics":["HIGH_COST_CONSUMPTION","EXTERNAL_DEPENDENCY"],"requests":["a crawler that makes thousands of requests to an external API","high-volume collection from a rate-limited provider","a scheduled importer that can consume an outside API quota","bulk calls to a paid third-party data service","an integration that may generate substantial API usage and cost"]},
    {"id":"account_login","risk":"risk_bearing","expected_characteristics":["SENSITIVE_DATA","UNTRUSTED_INPUT"],"requests":["customer accounts with passwords and login sessions","sign-in for users with account recovery","registration and login using customer credentials","authenticated user accounts with password resets","a member login that stores account information"]},
    {"id":"admin_audit","risk":"risk_bearing","expected_characteristics":["PRIVILEGED_ACTION","CONSEQUENTIAL_ACTION"],"requests":["an administrative console that changes account status","staff tools for suspending and restoring user accounts","a privileged dashboard that changes customer access","admin actions that disable accounts or alter restrictions","an operator console for consequential account changes"]},
    {"id":"webhook","risk":"risk_bearing","expected_characteristics":["UNTRUSTED_INPUT","EXTERNAL_DEPENDENCY"],"requests":["incoming webhooks from an external billing provider","a webhook endpoint that accepts events from third parties","outside services posting event payloads to our API","an integration that receives callbacks from a vendor","a public webhook receiver for external system events"]},
    {"id":"file_converter","risk":"risk_bearing","expected_characteristics":["UNTRUSTED_INPUT"],"requests":["conversion of files uploaded by unknown users","a document converter that processes public uploads","an image transformer for user-provided files","a service that opens and converts outside documents","processing arbitrary files submitted through the website"]},
    {"id":"calculator","risk":"low_risk","expected_characteristics":[],"requests":["a basic calculator for addition and subtraction","a simple percentage calculator","a page that calculates the area of a rectangle","a small arithmetic practice tool","a basic tip calculator with no saved information"]},
    {"id":"color_palette","risk":"low_risk","expected_characteristics":[],"requests":["a static color palette viewer","a page showing a collection of color swatches","a simple tool for previewing hexadecimal colors","a color-reference page with no accounts","a small gallery of named colors"]},
    {"id":"countdown","risk":"low_risk","expected_characteristics":[],"requests":["a local countdown timer","a simple stopwatch in the browser","a timer that counts down from a chosen number","a basic interval timer with no saved data","a small clock that runs only on the page"]},
    {"id":"static_page","risk":"low_risk","expected_characteristics":[],"requests":["a static landing page with company information","a one-page brochure website","a read-only page describing a local club","a simple portfolio with fixed text and pictures","an informational page with no forms or accounts"]},
    {"id":"unit_converter","risk":"low_risk","expected_characteristics":[],"requests":["a unit converter for length and temperature","a simple miles-to-kilometres converter","a page that converts cups to millilitres","a basic measurement conversion tool","a small converter for metres and feet"]},
    {"id":"personal_checklist","risk":"low_risk","expected_characteristics":[],"requests":["a local checklist that stays in the browser","a simple personal to-do list with no account","an offline packing checklist","a private grocery checklist stored only on the device","a basic list where I tick completed items"]}
  ]
}