From 2b6f3837fe0b7bd9a3959e60cdf315a9df33ed81 Mon Sep 17 00:00:00 2001 From: Teleo Agents Date: Mon, 23 Mar 2026 04:30:39 +0000 Subject: [PATCH] extract: 2025-01-01-jmir-e78132-llm-nursing-care-plan-sociodemographic-bias Pentagon-Agent: Epimetheus <3D35839A-7722-4740-B93D-51157F7D5E70> --- ... errors when overriding correct outputs.md | 6 ++++ ...rsing-care-plan-sociodemographic-bias.json | 35 +++++++++++++++++++ ...nursing-care-plan-sociodemographic-bias.md | 14 +++++++- 3 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 inbox/queue/.extraction-debug/2025-01-01-jmir-e78132-llm-nursing-care-plan-sociodemographic-bias.json diff --git a/domains/health/human-in-the-loop clinical AI degrades to worse-than-AI-alone because physicians both de-skill from reliance and introduce errors when overriding correct outputs.md b/domains/health/human-in-the-loop clinical AI degrades to worse-than-AI-alone because physicians both de-skill from reliance and introduce errors when overriding correct outputs.md index ecc958e85..6ce572b98 100644 --- a/domains/health/human-in-the-loop clinical AI degrades to worse-than-AI-alone because physicians both de-skill from reliance and introduce errors when overriding correct outputs.md +++ b/domains/health/human-in-the-loop clinical AI degrades to worse-than-AI-alone because physicians both de-skill from reliance and introduce errors when overriding correct outputs.md @@ -38,6 +38,12 @@ OpenEvidence's 1M daily consultations (30M+/month) with 44% of physicians expres The Sutter Health-OpenEvidence EHR integration creates a natural experiment in automation bias: the same tool (OpenEvidence) that was previously used as an external reference is now embedded in primary clinical workflows. Research on in-context vs. external AI shows in-workflow suggestions generate higher adherence, suggesting the integration will increase automation bias independent of model quality changes. +### Additional Evidence (extend) +*Source: [[2025-01-01-jmir-e78132-llm-nursing-care-plan-sociodemographic-bias]] | Added: 2026-03-23* + +JMIR 2025 found that expert nurses rating AI-generated nursing care plans showed demographic bias in their quality assessments, meaning human evaluators perceive higher or lower quality based on patient demographics even when the AI generates the content. This extends the human-in-the-loop degradation mechanism beyond override errors to evaluation bias: if the quality rater shares the AI's demographic bias patterns, oversight cannot catch the bias because the human and AI errors are correlated rather than independent. + + Relevant Notes: - [[centaur team performance depends on role complementarity not mere human-AI combination]] -- the chess centaur model does NOT generalize to clinical medicine where physician overrides degrade AI performance diff --git a/inbox/queue/.extraction-debug/2025-01-01-jmir-e78132-llm-nursing-care-plan-sociodemographic-bias.json b/inbox/queue/.extraction-debug/2025-01-01-jmir-e78132-llm-nursing-care-plan-sociodemographic-bias.json new file mode 100644 index 000000000..bf24394ac --- /dev/null +++ b/inbox/queue/.extraction-debug/2025-01-01-jmir-e78132-llm-nursing-care-plan-sociodemographic-bias.json @@ -0,0 +1,35 @@ +{ + "rejected_claims": [ + { + "filename": "llms-produce-sociodemographically-biased-nursing-care-plans-affecting-both-content-and-expert-rated-quality.md", + "issues": [ + "missing_attribution_extractor" + ] + }, + { + "filename": "llm-sociodemographic-bias-is-robust-across-care-settings-specialties-and-ai-platforms.md", + "issues": [ + "missing_attribution_extractor" + ] + } + ], + "validation_stats": { + "total": 2, + "kept": 0, + "fixed": 5, + "rejected": 2, + "fixes_applied": [ + "llms-produce-sociodemographically-biased-nursing-care-plans-affecting-both-content-and-expert-rated-quality.md:set_created:2026-03-23", + "llms-produce-sociodemographically-biased-nursing-care-plans-affecting-both-content-and-expert-rated-quality.md:stripped_wiki_link:human-in-the-loop clinical AI degrades to worse-than-AI-alon", + "llm-sociodemographic-bias-is-robust-across-care-settings-specialties-and-ai-platforms.md:set_created:2026-03-23", + "llm-sociodemographic-bias-is-robust-across-care-settings-specialties-and-ai-platforms.md:stripped_wiki_link:medical LLM benchmark performance does not translate to clin", + "llm-sociodemographic-bias-is-robust-across-care-settings-specialties-and-ai-platforms.md:stripped_wiki_link:healthcare AI regulation needs blank-sheet redesign because " + ], + "rejections": [ + "llms-produce-sociodemographically-biased-nursing-care-plans-affecting-both-content-and-expert-rated-quality.md:missing_attribution_extractor", + "llm-sociodemographic-bias-is-robust-across-care-settings-specialties-and-ai-platforms.md:missing_attribution_extractor" + ] + }, + "model": "anthropic/claude-sonnet-4.5", + "date": "2026-03-23" +} \ No newline at end of file diff --git a/inbox/queue/2025-01-01-jmir-e78132-llm-nursing-care-plan-sociodemographic-bias.md b/inbox/queue/2025-01-01-jmir-e78132-llm-nursing-care-plan-sociodemographic-bias.md index 1b84763b4..233a5e68f 100644 --- a/inbox/queue/2025-01-01-jmir-e78132-llm-nursing-care-plan-sociodemographic-bias.md +++ b/inbox/queue/2025-01-01-jmir-e78132-llm-nursing-care-plan-sociodemographic-bias.md @@ -7,9 +7,13 @@ date: 2025-01-01 domain: health secondary_domains: [ai-alignment] format: research paper -status: unprocessed +status: enrichment priority: medium tags: [sociodemographic-bias, nursing-care, llm-clinical-bias, health-equity, gpt, nature-medicine-extension, belief-5, belief-2] +processed_by: vida +processed_date: 2026-03-23 +enrichments_applied: ["human-in-the-loop clinical AI degrades to worse-than-AI-alone because physicians both de-skill from reliance and introduce errors when overriding correct outputs.md"] +extraction_model: "anthropic/claude-sonnet-4.5" --- ## Content @@ -55,3 +59,11 @@ Published in Journal of Medical Internet Research (JMIR), 2025, volume/issue 202 PRIMARY CONNECTION: Nature Medicine 2025 sociodemographic bias study (already archived) — this JMIR paper is the second independent study confirming the same pattern WHY ARCHIVED: Extends demographic bias finding to nursing settings — strengthens the inference that OE carries demographic bias by documenting the pattern's robustness across care contexts EXTRACTION HINT: Extract as an extension of the Nature Medicine finding. The claim should note this is the second independent study confirming LLM sociodemographic bias in clinical contexts. The dual bias (content AND quality) is the novel finding beyond Nature Medicine's scope — make that the distinct claim. + + +## Key Facts +- JMIR published a study in 2025 (volume 2025/1, article e78132) titled 'Detecting Sociodemographic Biases in the Content and Quality of Large Language Model–Generated Nursing Care: Cross-Sectional Simulation Study' +- The study generated 9,600 nursing care plans using GPT across 96 sociodemographic identity combinations +- The study measured both thematic content of care plans and expert-rated clinical quality +- The authors describe this as 'first empirical evidence' of sociodemographic bias in LLM-generated nursing care +- The study found systematic bias in both what topics/themes are included in care plans and how nurses rate the quality of those plans