{ "schema_version": 1, "methodology_version": "1.0", "updated_date": "2026-09-06", "studies": [ { "id": "alphafold-bindingdb-target-ligand", "title": "Did AlphaFold shorten the path from a protein target to a potent ligand?", "short_title": "AlphaFold and potent ligands", "summary": "A target-level quasi-experimental design now backed by a feasibility cohort of 849 proteins. 5 of 8 predeclared evidence gates currently pass, and the strict diagnostic contains 2 post-treatment 100 nM first-crossing events. No effect estimate has been produced.", "status": "data_build", "causal_status": "not_estimated", "domains": [ "biology", "medicine", "chemistry" ], "question": "Did the public availability of useful AlphaFold structures increase the rate at which previously structureless protein targets received experimentally measured potent ligands?", "estimand": "The change after 22 July 2021 in the target-month hazard of first crossing an experimental binding-affinity threshold below 100 nanomolar for targets that lacked an experimental structure before AlphaFold, relative to comparable targets that already had structural information.", "unit_of_analysis": "protein target-month", "related_series_ids": [ "bindingdb-target-ligand-frontiers" ], "treatment_or_exposure": { "name": "Public availability of a useful AlphaFold structure", "date": "2021-07-22", "definition": "A target is treated when it lacked an experimental structure before the AlphaFold Database launch but received a mappable predicted structure that passes the study's predeclared usability rule.", "basis": "The AlphaFold Database launch created a concrete target-specific information shock. Treatment varies across targets because some already had experimental structures and predicted-structure usefulness differs by target.", "alternative_dates": [ { "date": "2021-07-15", "label": "AlphaFold2 Nature publication", "purpose": "Sensitivity analysis for the publication date immediately preceding the database launch." }, { "date": "2022-07-28", "label": "AlphaFold Database expansion", "purpose": "Sensitivity analysis for the later expansion to a much broader protein universe." } ] }, "comparison": { "treated_population": "Eligible protein targets with no qualifying experimental structure before 22 July 2021 and a usable AlphaFold prediction after launch.", "comparison_population": "Eligible protein targets that already had a qualifying experimental structure before 22 July 2021 and had not yet crossed the relevant ligand-potency threshold.", "matching_or_adjustment": "Match or weight targets using pre-2021 target class, organism, sequence and family features, prior publications, prior BindingDB activity, disease relevance, experimental-structure history, and pre-treatment affinity-frontier movement.", "identifying_assumption": "Conditional on observed pre-treatment characteristics and trends, targets without prior structures would have followed the same post-2021 potent-ligand trajectory as structurally characterized targets in the absence of AlphaFold." }, "outcomes": [ { "id": "first-ligand-under-100nm", "name": "First ligand below 100 nM", "definition": "The first dated experimental BindingDB measurement for the target with an accepted affinity value below 100 nanomolar, among targets still at risk on the treatment date.", "direction": "higher_is_progress", "primary": true }, { "id": "first-ligand-under-1um", "name": "First ligand below 1 \u00b5M", "definition": "The first dated accepted experimental affinity measurement below 1 micromolar for a target that had not crossed that threshold before treatment.", "direction": "higher_is_progress", "primary": false }, { "id": "first-ligand-under-10nm", "name": "First ligand below 10 nM", "definition": "The first dated accepted experimental affinity measurement below 10 nanomolar for a target that had not crossed that threshold before treatment.", "direction": "higher_is_progress", "primary": false }, { "id": "affinity-frontier-improvement", "name": "Affinity-frontier improvement", "definition": "The frequency and log magnitude of new best experimental affinity measurements for each target after treatment.", "direction": "higher_is_progress", "primary": false } ], "cohort_rules": [ "Freeze the eligible target universe and identifier mapping before estimating any post-2021 effect.", "Require a stable target-level mapping across BindingDB, UniProt, RCSB PDB, and AlphaFold DB.", "Use experimental binding measurements only and predeclare accepted affinity types, units, inequality handling, and assay exclusions.", "Date threshold crossings by the underlying publication date rather than the later BindingDB ingestion date.", "Exclude a target from a threshold-specific risk set once it crossed that threshold before 22 July 2021.", "Record missing and ambiguous dates explicitly; do not impute a discovery date silently.", "Do not inspect the post-treatment coefficient before the cohort, treatment, outcomes, and model choices are frozen." ], "data_sources": [ { "name": "BindingDB downloads", "url": "https://www.bindingdb.org/rwd/bind/chemsearch/marvin/Download.jsp?all_download=yes", "role": "outcome_data" }, { "name": "AlphaFold Protein Structure Database", "url": "https://alphafold.ebi.ac.uk/", "role": "treatment_data" }, { "name": "RCSB Protein Data Bank", "url": "https://www.rcsb.org/", "role": "treatment_data" }, { "name": "UniProt", "url": "https://www.uniprot.org/", "role": "identifier_mapping" } ], "analysis_plan": [ "Build a target-month panel with threshold-specific risk sets and explicit treatment exposure.", "Plot event-study coefficients before and after July 2021 so pre-treatment differences are visible.", "Estimate a survival or discrete-time hazard model with target covariates, calendar-time effects, and treatment-by-post interactions.", "Report absolute cumulative-incidence differences as well as relative hazard estimates.", "Use the 100 nM outcome as primary and treat 1 \u00b5M, 10 nM, and frontier magnitude as predefined secondary analyses.", "Run alternative treatment dates, matching rules, target definitions, and structure-usability thresholds.", "Publish the cohort, mapping decisions, exclusions, code, and analysis specification before presenting an effect estimate." ], "falsification_tests": [ "Use pre-2021 placebo intervention dates and test whether an apparent effect appears before AlphaFold was available.", "Test for differential pre-treatment trends in threshold crossings and affinity-frontier improvements.", "Repeat the analysis on target groups for which the predicted structure fails the predeclared usability rule.", "Check whether results disappear when targets are matched more tightly on pre-treatment publications and medicinal-chemistry activity.", "Test outcomes less directly connected to structural information as negative-control outcomes where the data permit it." ], "main_confounders": [ "Research funding, target popularity, and disease relevance may change at the same time as structural information becomes available.", "Concurrent advances in screening, synthesis, cryo-EM, structure prediction, and medicinal-chemistry tooling can affect the same targets.", "BindingDB coverage and publication dates are incomplete and may change differently across target classes.", "Targets that lacked experimental structures before 2021 may differ systematically in tractability from targets that already had structures.", "A predicted structure can be available without being useful for the ligand-discovery workflow used by a particular laboratory." ], "gating_criteria": [ { "id": "target-identity-map", "description": "A deterministic, auditable mapping joins BindingDB targets to UniProt, PDB, and AlphaFold DB without unacceptable duplication or ambiguity.", "status": "passed", "evidence": "98.6% unique row mapping; 849 core targets." }, { "id": "publication-date-coverage", "description": "The share and distribution of recoverable publication dates are sufficient and do not differ sharply between treatment groups.", "status": "passed", "evidence": "100.0% of exact Ki/Kd rows have a parsable publication date." }, { "id": "assay-comparability", "description": "Affinity types, units, inequalities, and assay exclusions can be normalized under a rule fixed before outcome comparison.", "status": "passed", "evidence": "23778 dated exact Ki/Kd rows across 849 targets." }, { "id": "pre-treatment-structure-map", "description": "Experimental structure availability before July 2021 can be reconstructed reliably for the eligible target universe.", "status": "passed", "evidence": "100.0% of 32252 mapped PDB IDs have release dates." }, { "id": "alphafold-usability-rule", "description": "A predeclared rule distinguishes merely available AlphaFold predictions from predictions usable for the intended structural analysis.", "status": "pending", "evidence": null }, { "id": "cohort-size-and-overlap", "description": "Treated and comparison groups have adequate sample size, covariate overlap, and threshold-specific risk sets.", "status": "pending", "evidence": null }, { "id": "pre-trend-credibility", "description": "Event-study leads and pre-period outcome trends do not show a large treatment-group divergence before AlphaFold.", "status": "pending", "evidence": null }, { "id": "public-rebuild-path", "description": "Raw snapshots, transformations, cohort decisions, and the complete analysis can be rebuilt from public or distributable inputs.", "status": "passed", "evidence": "Release 202609, SHA-256 1203194f366623ae9b4caee34f2477d412ebddbda267593df9d1d92d0c66fb74; no AlphaFold API errors." } ], "precedent_evidence": [ { "title": "How Artificial Intelligence Shapes Science: Evidence from AlphaFold", "finding": "Hill and Stein report that downstream basic research increased by roughly 15\u201340% for proteins that lacked prior structural information relative to proteins that already had structures, while finding no corresponding evidence yet in early-stage drug development.", "source_url": "https://www.nber.org/papers/w35143", "relationship": "This provides a treatment-and-comparison precedent for the design. It is prior external evidence, not a result produced by this tracker." } ], "methodological_references": [ { "name": "METR \u2014 LLMs' Contribution to Discoveries", "url": "https://metr.org/notes/2026-08-14-llm-contribution-to-discoveries/", "note": "Motivates measuring objective discovery outcomes over time and separating observed acceleration from AI attribution." }, { "name": "tecunningham/ai-discovery-data", "url": "https://github.com/tecunningham/ai-discovery-data", "note": "Motivates the BindingDB candidate and the use of an AlphaFold-style exposure design rather than paper counts." }, { "name": "AlphaFold Protein Structure Database launch", "url": "https://www.ebi.ac.uk/about/news/announcements/alphafold-protein-structure-database/", "note": "Provides the public database-launch date used as the primary treatment date." } ], "limitations": [ "The strict audit cohort is article-curated and does not represent all BindingDB records, patents, or medicinal-chemistry activity.", "The design remains biased if treated and comparison targets have different unobserved post-2021 opportunities after matching.", "Binding affinity is a research milestone, not a clinical outcome, and threshold crossing does not imply a safe or effective drug.", "The current strict cohort contains 2 post-treatment 100 nM first-crossing events; outcome power must be evaluated before estimation.", "Pre-treatment event counts and calendar coverage need explicit modelling rather than a simple before-and-after comparison.", "The study should not be described as causal until the remaining gates pass and the identifying assumptions survive the stated falsification tests." ], "next_step": "Freeze a binding-site/domain usability rule, audit target-class covariate overlap, and estimate pre-treatment event-study leads before moving the study from data_build to analysis." } ] }