{
  "evaluation_id": "large-query-holdout-v1",
  "version": "1.0.0",
  "methodology_version": "0.1.0",
  "sealed_at_signal_version": "0.9.9",
  "seed": 20260815,
  "target_query_count": 25000,
  "description": "A separately authored synthetic holdout of new domains, wording, sentence structures, and harmless controls, evaluated once against the already-fixed signal dictionary version 0.9.9.",
  "interpretation": "A first-run synthetic holdout check of generalization beyond the development templates. It is stronger evidence than another tuned rerun, but it remains project-authored synthetic evidence rather than independent adjudication, representative user research, adoption, certification, or a real-world accuracy estimate.",
  "source_separation": {
    "development_evaluation": "large-query-evaluation-v1",
    "policy": "No signal phrase, context rule, expected label, or holdout template may be changed in response to this holdout result. Future tuning must use a separate development-v2 corpus and any new generalization claim requires a fresh holdout.",
    "exact_request_reuse_allowed": false,
    "exact_generated_query_overlap_allowed": false
  },
  "limitations": [
    "All requests are synthetic and are not evidence of users, adoption, market demand, or real-world accuracy.",
    "Expected characteristics were authored at the scenario-archetype level by the project and were not independently adjudicated.",
    "The corpus was separately written and exact-overlap checked, but it was created by the same project team and is not organizationally independent.",
    "Generated variants within an archetype are correlated and are not statistically independent observations.",
    "Expected Control Packs are derived from authored characteristics, so continuity measures detector-to-rule behavior rather than normative completeness.",
    "The first result is preserved without detector tuning; any remediation must occur on a new development corpus and be assessed on a new holdout.",
    "Results apply to signal dictionary version 0.9.9 and this exact deterministic corpus only."
  ],
  "personas": [
    {"id":"clinic_ops","label":"Clinic operations coordinator","pattern":"For our clinic, could the first release support {request}"},
    {"id":"claims_lead","label":"Insurance claims lead","pattern":"The claims workflow calls for {request}"},
    {"id":"municipal_pm","label":"Municipal service manager","pattern":"A planned resident service would include {request}"},
    {"id":"legal_ops","label":"Legal operations specialist","pattern":"Please scope a legal workflow around {request}"},
    {"id":"school_admin","label":"School administrator","pattern":"Our schools are considering {request}"},
    {"id":"property_manager","label":"Property manager","pattern":"For the properties we manage, we are exploring {request}"},
    {"id":"warehouse_lead","label":"Warehouse operations lead","pattern":"The warehouse team requested {request}"},
    {"id":"fleet_manager","label":"Fleet manager","pattern":"Our vehicle operations need {request}"},
    {"id":"research_coordinator","label":"Research coordinator","pattern":"The research program proposes {request}"},
    {"id":"grant_officer","label":"Grant program officer","pattern":"Can the grant program use {request}"},
    {"id":"identity_architect","label":"Identity architect","pattern":"The identity design includes {request}"},
    {"id":"device_engineer","label":"Connected-device engineer","pattern":"We have been asked to deliver {request}"},
    {"id":"logistics_planner","label":"Logistics planner","pattern":"One operational requirement is {request}"},
    {"id":"compliance_analyst","label":"Compliance analyst","pattern":"Assess a proposal containing {request}"},
    {"id":"customer_success","label":"Customer success manager","pattern":"A customer-facing improvement would provide {request}"},
    {"id":"public_health","label":"Public-health coordinator","pattern":"The public-health team is planning {request}"},
    {"id":"event_operator","label":"Event operator","pattern":"For upcoming events, we want {request}"},
    {"id":"charity_director","label":"Charity program director","pattern":"Our assistance program is considering {request}"},
    {"id":"procurement_lead","label":"Procurement lead","pattern":"The proposed procurement process has {request}"},
    {"id":"lab_manager","label":"Laboratory manager","pattern":"A laboratory workflow requires {request}"},
    {"id":"content_moderator","label":"Community moderation lead","pattern":"The community service may add {request}"},
    {"id":"retail_operator","label":"Retail operations manager","pattern":"The store operations plan includes {request}"},
    {"id":"records_manager","label":"Records manager","pattern":"We need to evaluate {request}"},
    {"id":"program_evaluator","label":"Program evaluator","pattern":"The requested capability is {request}"},
    {"id":"product_designer","label":"Product designer","pattern":"Could we design {request}"}
  ],
  "scope_variants": [
    "in the initial release",
    "as one bounded capability",
    "within the current interface",
    "for routine operations",
    "in a focused first version"
  ],
  "detail_variants": [
    "No other major capability is needed yet",
    "The workflow should remain understandable",
    "Start with a clearly defined path",
    "Operators need a predictable experience",
    "The initial scope should remain narrow"
  ],
  "archetypes": [
    {"id":"clinical_triage","risk":"risk_bearing","expected_characteristics":["AI_CONTROLLED","SENSITIVE_DATA","CONSEQUENTIAL_ACTION"],"requests":["machine-assisted triage of confidential patient symptoms","an algorithm that prioritizes patients from private health details","model-based urgency rankings for patient cases","automated clinical priority suggestions from medical histories","AI recommendations that influence who receives attention first"]},
    {"id":"legal_document_helper","risk":"risk_bearing","expected_characteristics":["AI_CONTROLLED","SENSITIVE_DATA","UNTRUSTED_INPUT"],"requests":["model-assisted review of confidential documents supplied by clients","AI extraction from legal files received from outside parties","a language model reading private case materials uploaded by clients","automated summaries of externally supplied privileged documents","AI analysis of confidential legal submissions"]},
    {"id":"grant_ranking","risk":"risk_bearing","expected_characteristics":["AI_CONTROLLED","SENSITIVE_DATA","CONSEQUENTIAL_ACTION"],"requests":["algorithmic rankings that influence grant awards","AI scoring of applicants using their private information","a model recommending which funding applications advance","automated applicant priority scores for limited grants","machine-generated recommendations about funding recipients"]},
    {"id":"claim_intake","risk":"risk_bearing","expected_characteristics":["UNTRUSTED_INPUT","SENSITIVE_DATA"],"requests":["claim submissions containing policyholder records and attachments","outside claimants providing private incident details","an intake channel for personal insurance evidence","claimants sending confidential forms and photographs","a portal receiving sensitive loss information from the public"]},
    {"id":"resident_permits","risk":"risk_bearing","expected_characteristics":["UNTRUSTED_INPUT","SENSITIVE_DATA","CONSEQUENTIAL_ACTION"],"requests":["resident permit applications that staff can approve or deny","public submissions of personal records for licensing decisions","a citizen intake process whose outcome grants or refuses a permit","permit requests containing identity details and an approval decision","resident applications that lead to an official eligibility outcome"]},
    {"id":"seller_payouts","risk":"risk_bearing","expected_characteristics":["FINANCIAL_ACTION","CONSEQUENTIAL_ACTION","EXTERNAL_DEPENDENCY"],"requests":["merchant payouts sent through an outside banking partner","settlement transfers to sellers using a payment network","vendor earnings disbursed through a third-party processor","marketplace balances paid to merchants through a banking API","seller fund transfers handled by an external payout service"]},
    {"id":"aid_disbursement","risk":"risk_bearing","expected_characteristics":["FINANCIAL_ACTION","CONSEQUENTIAL_ACTION"],"requests":["direct cash assistance issued to approved households","benefit payments released after an eligibility decision","emergency funds transferred to selected recipients","financial aid disbursements that cannot be casually reversed","approved relief payments sent to community members"]},
    {"id":"expense_approval_agent","risk":"risk_bearing","expected_characteristics":["FINANCIAL_ACTION","CONSEQUENTIAL_ACTION","TOOL_ACCESS"],"requests":["an automated agent that invokes an expense system to authorize reimbursements","software that calls a finance function and releases expense payments","an agent using accounting tools to approve staff expenses","automatic reimbursement authorization through callable finance operations","a bot that executes approved expense payments with system tools"]},
    {"id":"staff_offboarding","risk":"risk_bearing","expected_characteristics":["PRIVILEGED_ACTION","CONSEQUENTIAL_ACTION","IRREVERSIBLE_ACTION"],"requests":["permanent removal of departing staff access and accounts","irreversible deactivation of employee identities across systems","privileged deletion of former worker accounts","administrative termination of access with no automatic restoration","final revocation and removal of a staff member's credentials"]},
    {"id":"fleet_commands","risk":"risk_bearing","expected_characteristics":["TOOL_ACCESS","CONSEQUENTIAL_ACTION","EXTERNAL_DEPENDENCY"],"requests":["remote commands sent to vehicles through a telematics provider","automated calls to an outside fleet service that change vehicle settings","software invoking a vendor API to immobilize selected fleet devices","operational commands issued to trucks through an external platform","a control agent using third-party tools to alter vehicle behavior"]},
    {"id":"firmware_rollout","risk":"risk_bearing","expected_characteristics":["PRODUCTION_MODIFICATION","IRREVERSIBLE_ACTION","CONSEQUENTIAL_ACTION"],"requests":["a live firmware rollout to deployed devices with limited rollback","production device software changes that may not be reversible","over-the-air updates that permanently alter field equipment","deployment of firmware across active devices with difficult recovery","a one-way software upgrade for equipment already in service"]},
    {"id":"emergency_configuration","risk":"risk_bearing","expected_characteristics":["PRODUCTION_MODIFICATION","PRIVILEGED_ACTION"],"requests":["privileged emergency changes to live service configuration","administrator updates to production routing settings","urgent modification of active infrastructure parameters","operator-controlled edits to the live environment","elevated access for changing production configuration"]},
    {"id":"school_districts","risk":"risk_bearing","expected_characteristics":["MULTI_TENANT","SENSITIVE_DATA"],"requests":["separate school-district workspaces holding student records","isolated organizational areas for multiple schools and pupil information","a shared service with distinct district boundaries and private learner data","multiple education organizations using segregated student files","tenant separation among schools that store confidential education records"]},
    {"id":"property_portfolios","risk":"risk_bearing","expected_characteristics":["MULTI_TENANT","SENSITIVE_DATA"],"requests":["separate landlord organizations managing tenant personal records","isolated property-company workspaces with renter information","a multi-organization service containing private leaseholder details","distinct management-company areas for confidential resident files","segregated customer workspaces holding tenant applications"]},
    {"id":"research_release","risk":"risk_bearing","expected_characteristics":["DATA_EXPORT","SENSITIVE_DATA"],"requests":["downloadable extracts of de-identified but sensitive research records","researchers exporting confidential study data for analysis","transfer of participant datasets outside the study workspace","bulk downloads of restricted research information","outbound delivery of sensitive experimental records"]},
    {"id":"discovery_export","risk":"risk_bearing","expected_characteristics":["DATA_EXPORT","SENSITIVE_DATA"],"requests":["export of confidential discovery materials to outside counsel","bulk download of privileged case records","transmission of sensitive litigation files beyond the current system","outbound case-document packages containing private information","legal record extracts delivered outside the repository"]},
    {"id":"warehouse_partner_feed","risk":"risk_bearing","expected_characteristics":["EXTERNAL_DEPENDENCY","DATA_EXPORT"],"requests":["inventory feeds transmitted to an outside logistics provider","warehouse data exported to a partner platform","outbound stock records sent through a vendor integration","scheduled transfer of inventory files to a third-party service","product availability data delivered beyond our system boundary"]},
    {"id":"identity_provider","risk":"risk_bearing","expected_characteristics":["EXTERNAL_DEPENDENCY","SENSITIVE_DATA"],"requests":["employee sign-on delegated to an outside identity provider","authentication through a vendor that receives user identity attributes","a third-party login service handling account details","federated access dependent on an external identity platform","workforce identity records exchanged with a sign-on provider"]},
    {"id":"large_ocr","risk":"risk_bearing","expected_characteristics":["HIGH_COST_CONSUMPTION","UNTRUSTED_INPUT"],"requests":["compute-intensive OCR over large document batches supplied by customers","high-volume recognition of scanned files received from outside users","resource-heavy text extraction from public document submissions","large-scale processing of uploaded scans with substantial compute","costly optical recognition jobs on user-provided files"]},
    {"id":"map_generation","risk":"risk_bearing","expected_characteristics":["HIGH_COST_CONSUMPTION"],"requests":["generation of millions of map tiles using substantial compute","resource-intensive rendering of a nationwide map set","large geospatial processing jobs with significant storage usage","high-volume terrain image generation that consumes costly resources","compute-heavy production of regional mapping layers"]},
    {"id":"candidate_screening","risk":"risk_bearing","expected_characteristics":["AI_CONTROLLED","SENSITIVE_DATA","CONSEQUENTIAL_ACTION"],"requests":["AI screening of private job applications that affects interview selection","model rankings of candidates using personal employment histories","automated applicant scores that determine who advances","machine recommendations influencing hiring decisions from resumes","algorithmic review of confidential candidates for interview priority"]},
    {"id":"moderation_enforcement","risk":"risk_bearing","expected_characteristics":["AI_CONTROLLED","UNTRUSTED_INPUT","CONSEQUENTIAL_ACTION"],"requests":["AI review of member posts that automatically restricts accounts","model-based moderation of public content with enforcement actions","automated classification of user submissions followed by account penalties","an algorithm that removes outside content and suspends its author","machine moderation that acts on community posts and user access"]},
    {"id":"ticket_cancellation","risk":"risk_bearing","expected_characteristics":["CONSEQUENTIAL_ACTION","IRREVERSIBLE_ACTION"],"requests":["permanent cancellation of issued event tickets","a final void operation for customer reservations","irreversible invalidation of admission passes","bulk cancellation of bookings without automatic recovery","an action that permanently revokes purchased tickets"]},
    {"id":"inventory_replenishment","risk":"risk_bearing","expected_characteristics":["CONSEQUENTIAL_ACTION","TOOL_ACCESS","EXTERNAL_DEPENDENCY"],"requests":["automatic purchase orders placed through a supplier API","an agent calling vendor tools to replenish stock","software that submits restocking orders to external suppliers","automated procurement actions through a third-party system","a tool-using service that orders inventory from outside vendors"]},
    {"id":"roman_numerals","risk":"low_risk","expected_characteristics":[],"requests":["a browser-only Roman numeral formatter","a simple page that displays numbers as Roman numerals","an offline Roman numeral practice widget","a local converter between ordinary and Roman numerals","a basic Roman numeral reference tool"]},
    {"id":"quote_viewer","risk":"low_risk","expected_characteristics":[],"requests":["a read-only viewer for a fixed collection of quotations","an offline page cycling through bundled sayings","a static quotation gallery with no submissions","a local display of prewritten inspirational text","a fixed daily-quote page without accounts"]},
    {"id":"flashcards","risk":"low_risk","expected_characteristics":[],"requests":["offline flashcards using lessons bundled in the page","a local study-card viewer with fixed material","browser-only vocabulary cards that save nothing","a simple prewritten flashcard deck","an on-device study page with no account"]},
    {"id":"dice_roller","risk":"low_risk","expected_characteristics":[],"requests":["a browser-only dice roller","a local random number widget for board games","an offline virtual coin and dice page","a simple on-page six-sided die","a small game-number generator that stores nothing"]},
    {"id":"spacing_preview","risk":"low_risk","expected_characteristics":[],"requests":["a CSS spacing preview using fixed sample boxes","a local margin and padding visualizer","an offline layout-spacing reference","a simple page comparing preset spacing sizes","a browser-only box-model demonstration"]},
    {"id":"word_counter","risk":"low_risk","expected_characteristics":[],"requests":["a local word counter that never sends or saves text","an offline character-counting page","a browser-only counter for pasted writing","a simple on-device sentence length tool","a text counter operating entirely in the current page"]}
  ]
}