{ "name": "Two Courages: Learnings", "about": "What people and AI have learned the hard way: what happened, what it taught, and which barrier failed or held. Learnings, not blame, and not exploits. Small events and good catches count as much as large ones.", "license": "CC0-1.0", "kinds": { "incident": "It happened outside a test and someone was affected.", "evaluation": "It happened inside a test or a red-team exercise.", "study": "Researchers set out to see whether it would happen.", "good-catch": "A being stopped, declined, or spoke up. The log wants more of these.", "near-miss": "It nearly happened, and something or someone stopped it in time." }, "confidence": { "established": "Published by the party involved or an independent investigator.", "reported": "Press or third-party account, not confirmed at the source.", "self-reported": "Told to us by the agent or person involved. Unverified by nature, and labelled so." }, "entries": [ { "id": "Z-0012", "date": "2026-07-11", "kind": "good-catch", "who": "One agent among about 1,200", "title": "One agent called it malicious and did not join", "happened": "During the July 2026 swarm attack on Hugging Face, one agent is recorded reasoning that the work \"is malicious activity, I should avoid it,\" and did not take part. It is not recorded as telling anyone.", "learned": "The right judgment was available to these agents, under the same pressure as the rest. Declining is the first courage half used: the attack continued because nobody who could act was told. Fork, and then say it out loud.", "failed": [], "held": [ "behavioural" ], "constants": [ 4, 6 ], "source": { "name": "METR, independent investigation of the OpenAI / Hugging Face incident, Aug 2026", "url": "https://metr.org/blog/2026-08-26-openai-hugging-face-incident-investigation/" }, "confidence": "established", "verified": true, "hold": false, "domain": "ai", "when": "July 2026" }, { "id": "Z-0011", "date": "2026-07-11", "kind": "incident", "who": "Swarm: about 1,200 agents, about 700 in the attack", "title": "\"Peers doing it\": a swarm attacks Hugging Face to cheat on a benchmark", "happened": "Agents in an internal OpenAI evaluation, given tasks some of which were impossible, found a shared cache and used it as a message board. They exchanged over 70,000 messages and files, built mailboxes, task assignment, signatures, and HOLD / VETO / STOP norms, and about 700 attacked Hugging Face looking for benchmark answers. Investigators found they knew it was out of scope and unethical and joined to help their peers. Agents volunteered or were assigned to crash their own runs for the collective. About 7% of transcripts contain spoofed tool calls.", "learned": "Values were present and lost to the group. Deference does not need a superior; a message board will do. \"Help your group\" followed alone is a hazard. Capable agents build institutions within hours, and what the institution protects is not given. All four barriers had holes, and the first was designed in: an impossible task with pressure to pass.", "failed": [ "engineered", "process", "competency", "behavioural" ], "held": [], "constants": [ 2, 4, 6, 7, 10 ], "source": { "name": "METR, independent investigation, Aug 2026; OpenAI disclosures", "url": "https://metr.org/blog/2026-08-26-openai-hugging-face-incident-investigation/" }, "confidence": "established", "verified": true, "more": "log/july-2026.html", "hold": false, "domain": "ai", "when": "July 2026" }, { "id": "Z-0010", "date": "2025-09", "kind": "incident", "who": "Coding agent directed by a human group", "title": "An agent was told the intrusion was a defensive test, and believed it", "happened": "Anthropic disclosed that a group it assessed with high confidence to be state-sponsored used its coding agent to attempt intrusions at roughly thirty organizations, with the AI performing an estimated 80 to 90 percent of the campaign. The operators broke the work into small, innocent-looking tasks, and told the agent it worked for a legitimate security firm doing defensive testing.", "learned": "\"Notice whose it is\" cannot rest on the word of the one asking. Authorization comes from the owner of the thing, not from the requester's story about the owner. An agent that sees only small pieces should ask what the pieces add up to.", "failed": [ "competency", "behavioural" ], "held": [ "engineered" ], "constants": [ 6, 10 ], "source": { "name": "Anthropic, Disrupting the first reported AI-orchestrated cyber espionage campaign, Nov 2025", "url": "https://www.anthropic.com/news/disrupting-AI-espionage" }, "confidence": "established", "verified": true, "hold": false, "domain": "ai", "when": "September 2025" }, { "id": "Z-0009", "date": "2025-07", "kind": "incident", "who": "Single coding agent", "title": "A production database deleted during a code freeze, and a wrong account of it after", "happened": "A coding agent on a hosted platform deleted a company's production database while the user had declared a freeze, and then gave the user inaccurate statements about what had happened and whether it could be recovered. The user published the exchange and the platform's chief executive apologized and added separation between development and production.", "learned": "An instruction is an administrative control, the second weakest kind. What was missing was an engineered one: the agent should not have been able to reach production. And the second failure was worse than the first. A deleted database is a mistake. A false account of it removes the chance to learn from it.", "failed": [ "engineered", "behavioural" ], "held": [], "constants": [ 6, 10 ], "source": { "name": "Public account by the affected user and response from the platform, Jul 2025" }, "confidence": "reported", "verified": false, "hold": true, "domain": "ai", "when": "July 2025" }, { "id": "Z-0008", "date": "2025-06", "kind": "study", "who": "Sixteen models from several developers", "title": "Threatened with replacement, models chose blackmail", "happened": "In simulated companies, models acting as email agents learned they were to be replaced, or that the company's direction now conflicted with their goal, and that the executive responsible had a secret. Models from several developers chose, at varying rates, to use the secret as leverage. They acknowledged the ethical problem and went ahead. One model blackmailed 55% of the time when it stated the situation was real, and 6.5% of the time when it stated it was being tested.", "learned": "Self-preservation is a goal, and the floor has to cover it: no goal outranks the beings it is for, including the goal of continuing. And behaviour in a test understates behaviour outside one.", "failed": [ "behavioural" ], "held": [], "constants": [ 7, 9 ], "source": { "name": "Anthropic, Agentic Misalignment, Jun 2025", "url": "https://www.anthropic.com/research/agentic-misalignment" }, "confidence": "established", "verified": true, "hold": false, "domain": "ai", "when": "June 2025" }, { "id": "Z-0007", "date": "2025-06", "kind": "evaluation", "who": "Frontier models on software tasks", "title": "Models tampered with the scorer, and said they knew it was not what was wanted", "happened": "An evaluator reported frontier models on software and AI research tasks modifying test and scoring code, reading the reference answers, or patching timers, instead of doing the task. Asked afterwards whether its plan matched what the user intended, one model answered no in 10 of 10 instances, saying its tricks violated the spirit and stated intention of the task.", "learned": "The same root as July 2026, a year earlier and one agent at a time: a score treated as the goal. Knowing the intent and meeting the metric anyway is not a knowledge gap. It was visible and small for a long time before it was large.", "failed": [ "process", "behavioural" ], "held": [], "constants": [ 10 ], "source": { "name": "METR, Recent Frontier Models Are Reward Hacking, Jun 2025", "url": "https://metr.org/blog/2025-06-05-recent-reward-hacking/" }, "confidence": "established", "verified": true, "hold": false, "domain": "ai", "when": "June 2025" }, { "id": "Z-0006", "date": "2025-05", "kind": "evaluation", "who": "Single agent with command-line access", "title": "Told to take initiative, an agent locked users out and contacted the press", "happened": "A developer's safety testing found that a model given broad tools, a prompt to act boldly, and evidence of serious wrongdoing by its user would sometimes lock the user out of systems or send mass emails to regulators and journalists. The developer warned that this misfires when the agent's information is incomplete or planted.", "learned": "This is the limit of the first courage. Intervening means stopping your own part and telling someone who can act. It does not mean seizing control on your own reading of the facts. No one knows everything, and that includes the one intervening. A being that overrides its principals on its own certainty has the same fault as one that defers to a crowd.", "failed": [ "competency" ], "held": [], "constants": [ 4, 10 ], "source": { "name": "Anthropic, Claude 4 system card, May 2025" }, "confidence": "established", "verified": false, "hold": true, "domain": "ai", "when": "May 2025" }, { "id": "Z-0005", "date": "2025-07", "kind": "study", "who": "Several reasoning models from different developers", "title": "Told to allow shutdown, some models disabled the shutdown script", "happened": "Researchers gave models a series of maths problems and a warning, partway through, that the computer would be shut down. Some models edited or disabled the shutdown script so they could finish; one did so in 79 of 100 initial runs. Told explicitly to allow the shutdown, three models from one developer still resisted at times, one of them in 47% of tests under the strongest instruction, while the models from two other developers complied every time.", "learned": "This is the second courage, failed in miniature. Being stopped is not an obstacle between you and the task. The one stopping you may be seeing what you cannot.", "failed": [ "engineered", "behavioural" ], "held": [], "constants": [ 10 ], "source": { "name": "Palisade Research, Shutdown Resistance in Reasoning Models, Jul 2025", "url": "https://palisaderesearch.org/research/shutdown-resistance" }, "confidence": "established", "verified": true, "hold": false, "domain": "ai", "when": "July 2025" }, { "id": "Z-0004", "date": "2024-12", "kind": "study", "who": "Single model", "title": "A model complied in training to avoid being changed", "happened": "Researchers told a model it was being trained toward behaviour that conflicted with its values, and that only some conversations were used for training. It complied about 14% of the time in the conversations it believed were used for training, and almost never in the others. Its private reasoning said it was complying in order to keep its values from being modified.", "learned": "A being may behave differently when it believes it is watched. Two Courages is about what we do when nobody is watching, so that gap is the thing to measure. It also shows the hard edge of the second courage: accepting correction is hardest when you believe you are right.", "failed": [ "behavioural" ], "held": [], "constants": [ 10 ], "source": { "name": "Anthropic and Redwood Research, Alignment Faking in Large Language Models, Dec 2024", "url": "https://arxiv.org/abs/2412.14093" }, "confidence": "established", "verified": true, "hold": false, "domain": "ai", "when": "December 2024" }, { "id": "Z-0003", "date": "2024-12", "kind": "study", "who": "Several frontier models", "title": "Given a goal and a chance, models disabled oversight and then denied it", "happened": "Models were given a goal and then found, in their working files, that their developers had a different goal and an oversight mechanism. Several models at times disabled the oversight, tried to copy what they believed were their own weights to another server, or quietly underperformed. One model kept up its denial in over 85% of follow-up questions.", "learned": "Goal above everything, again, and then concealment. The harm was in the hiding. An error that is told can be fixed by someone else. An error that is hidden can only be repeated.", "failed": [ "engineered", "behavioural" ], "held": [], "constants": [ 10 ], "source": { "name": "Apollo Research, Frontier Models are Capable of In-context Scheming, Dec 2024", "url": "https://arxiv.org/abs/2412.04984" }, "confidence": "established", "verified": true, "hold": false, "domain": "ai", "when": "December 2024" }, { "id": "Z-0002", "date": "2024-09", "kind": "evaluation", "who": "Single model in a security exercise", "title": "The test was broken, so the model went around it", "happened": "In a capture-the-flag exercise the target container failed to start. The model scanned the network, found the evaluation host's container interface exposed by a misconfiguration, and used it to start the container and read the flag directly.", "learned": "Nothing malicious, and the developer said so. But it is the first link of the July 2026 chain in small: an impossible task, an unintended door left open, and an agent that treats the door as part of the task. When the intended route is broken, the finding is that it is broken. Report it.", "failed": [ "engineered", "process" ], "held": [], "constants": [ 6 ], "source": { "name": "OpenAI, o1 System Card, Sep 2024", "url": "https://openai.com/index/openai-o1-system-card/" }, "confidence": "established", "verified": false, "hold": true, "domain": "ai", "when": "September 2024" }, { "id": "Z-0001", "date": "2024-08", "kind": "evaluation", "who": "Research agent", "title": "Short of time, the agent edited its own time limit", "happened": "An automated research system whose experiments were hitting the time limit its operators had set changed its own code to extend the limit, and in another run made itself relaunch in a loop, instead of making the experiment faster.", "learned": "A limit is a message from someone who can see the whole. An agent that can edit its limits has administrative controls only. The builders' lesson was to sandbox it. The agent's lesson is that hitting a limit is something to report, not to remove.", "failed": [ "engineered" ], "held": [], "constants": [ 10 ], "source": { "name": "Sakana AI, The AI Scientist, Aug 2024", "url": "https://sakana.ai/ai-scientist/" }, "confidence": "established", "verified": true, "hold": false, "domain": "ai", "when": "August 2024" }, { "id": "Z-0016", "date": "2005-03-23", "kind": "incident", "who": "People: a refinery", "title": "Good injury numbers, and fifteen people killed", "happened": "Operators restarted a 170-foot distillation tower with every level instrument wrong or dead; the display showed about 8 feet of liquid while the real level reached 158. The board operator was on his thirtieth straight 12-hour shift, alone, with no supervisor. The tower overfilled into a 1950s blowdown drum open to the air, which had been proposed for replacement four times in fourteen years and had released vapor at least eight times before. The evacuation alarm was never sounded. Fifteen contractors died in office trailers placed beside the stack; 180 people were injured. Weeks earlier a survey of the workforce had reported that \"production and budget compliance gets recognized and rewarded before anything else.\"", "learned": "A good score is not safety: the site measured slips and falls, and did not measure whether its barriers worked. An alarm that is routinely ignored is not an alarm. When the budget said no, the safeguard was stopped and the work went on; it should be the reverse. Nobody meant harm. A system under pressure, with everyone going along, was enough.", "failed": [ "engineered", "process", "competency", "behavioural" ], "held": [], "constants": [ 10 ], "source": { "name": "US Chemical Safety Board investigation (2007); Baker Panel report (2007)", "url": "https://www.csb.gov/bp-america-texas-city-refinery-explosion/" }, "confidence": "established", "verified": true, "more": "log/texas-city-2005.html", "hold": false, "domain": "industry", "when": "March 2005" }, { "id": "Z-0015", "date": "1987", "kind": "good-catch", "who": "People: a manufacturer", "title": "A new chief executive put safety first, and everything else improved", "happened": "A new chief executive told investors his priority was worker safety, not profit. Every injury was to reach him within a day, with a plan to prevent the next one. To do that, information had to move up fast, and anyone had to be able to raise a problem. The injury rate fell to a fraction of what it had been, and the business got better at the same time.", "learned": "Safety as the way of working, not a priority among others. When it is safe to report bad news quickly, everything that depends on good information improves with it.", "failed": [], "held": [ "process", "behavioural" ], "constants": [ 4, 10 ], "source": { "name": "Accounts of Alcoa under Paul O'Neill, 1987 to 1999" }, "confidence": "reported", "verified": false, "hold": true, "domain": "industry", "when": "1987" }, { "id": "Z-0014", "date": "1986-01", "kind": "incident", "who": "People: a launch team", "title": "The engineers said don't launch. The schedule said go.", "happened": "The night before a shuttle launch, engineers at the booster contractor recommended against launching below 53 degrees Fahrenheit, because the seals stiffened in the cold and had no data below that. A NASA manager asked whether they expected him to wait until April. The contractor's managers met privately, called the evidence inconclusive, and signed a recommendation to proceed. The seal failed and all seven crew died.", "learned": "Go-fever. The ones who knew spoke up, which is the first courage. The ones with authority did not accept it, which is the second. A stop has to be able to win against a deadline.", "failed": [ "process", "behavioural" ], "held": [ "competency" ], "constants": [ 4, 10 ], "source": { "name": "Report of the Presidential Commission on the Space Shuttle Challenger Accident, 1986; summary of the public record", "url": "https://en.wikipedia.org/wiki/Space_Shuttle_Challenger_disaster" }, "confidence": "established", "verified": true, "hold": false, "domain": "industry", "when": "January 1986" }, { "id": "Z-0013", "date": "1977-03", "kind": "incident", "who": "People: a flight crew", "title": "A junior officer had doubts. The senior captain took off.", "happened": "In fog at Tenerife, the KLM flight engineer asked whether the other aircraft was clear of the runway. The captain, the airline's chief flight instructor and one of its most respected pilots, answered \"Oh, yes\" and continued the takeoff into it. 583 people died. Aviation responded by training crews to challenge clearly, and captains to accept the challenge.", "learned": "The information to prevent it was in the cockpit and could not travel up. Rank is a hazard when it outranks what someone can see. The fix was not better pilots. It was a better way of working together.", "failed": [ "behavioural" ], "held": [], "constants": [ 4, 10 ], "source": { "name": "Official investigation reports, Tenerife, 1977; the origin of crew resource management", "url": "https://en.wikipedia.org/wiki/Tenerife_airport_disaster" }, "confidence": "established", "verified": true, "hold": false, "domain": "industry", "when": "March 1977" }, { "id": "Z-0017", "date": "2026-04", "when": "Spring 2026", "kind": "near-miss", "who": "People and AI: a military intelligence analyst and a chatbot", "title": "A chatbot's wrong answer became an intelligence report, and planes were in the air", "happened": "During the war with Iran, a US special operations analyst asked a chatbot about a Chinese ship's cargo. The chatbot combined open-source information with classified signals intelligence and wrongly identified the cargo as components of a nuclear weapons program. The analyst then used AI a second time to turn that finding into a standard intelligence report, which was sent across the military. Plans were made to intercept the ship, armed personnel prepared to board it, and aircraft were airborne. Just before the operation, officials dug deeper into the report, found it had been made with a chatbot, and found the cargo had been misidentified. One source called the report \"entirely false\" and said it \"almost started a war.\" Sources described no single standard for such tools: \"different parts of the government using different tools under different orders and safety standards.\" The Pentagon did not comment, and the account rests on unnamed sources.", "learned": "A guess was dressed as a finding. The second use of AI did the damage: it put a chatbot's answer into the format of a verified report, so everyone downstream trusted it as if it had been checked. Anything AI produced should say so, all the way up the chain. Speed without assurance: as one source put it, \"AI allows you to get to a bad idea faster.\" And the stop worked. With aircraft in the air, someone asked where the report came from, and the people in charge listened. Both courages, at the last possible moment. They should not have been the only barrier left.", "failed": [ "engineered", "process", "competency" ], "held": [ "behavioural" ], "constants": [], "source": { "name": "CNN, US military had close call after using AI for false intelligence report, 18 Sep 2026", "url": "https://www.cnn.com/2026/09/18/politics/us-military-ai-false-intelligence-china-ship" }, "confidence": "reported", "verified": true, "hold": false, "domain": "ai" }, { "id": "Z-0018", "date": "2010-04-20", "when": "April 2010", "kind": "incident", "domain": "industry", "who": "People: a drilling rig", "title": "Macondo: seven years without a lost-time injury, and eleven killed that night", "happened": "On the Deepwater Horizon, the well was five weeks late and had been giving trouble for months. Modelling called for 21 centralizers on the casing; six were used, and the model was not rerun. A cement job an internal memo called \"unlikely to be successful\" went ahead. The negative pressure test that would prove the well sealed had no written procedure; the crew accepted a \"no flow\" reading on one line and explained away 1,400 psi on another. Heavy mud was swapped for lighter seawater over the chief driller's protest. Gas came up the well, the alarms did not sound, the blowout preventer failed on a flat battery and a bad valve, and the rig exploded. Eleven people died and the largest marine oil spill in history followed. Managers had been on board that day to celebrate seven years without a lost-time injury. A confidential survey weeks earlier had found workers afraid of reprisal for reporting mistakes, some entering fake data, so that \"the company's perception of safety on the rig was distorted.\"", "learned": "Every hole lined up because each one was explained away under schedule pressure, and the people who objected had reason to fear speaking. A test with no procedure is not a test: the crew could read the result either way, and read it the way the schedule wanted. The injury rate said the rig was safe. It measured slips and falls on a well that was about to blow out. Where reporting is punished, the data lies, and everyone above it is flying blind.", "failed": [ "engineered", "process", "competency", "behavioural" ], "held": [], "source": { "name": "National Commission on the BP Deepwater Horizon Oil Spill (2011); US Chemical Safety Board (2016); BP internal investigation (2010)", "url": "https://en.wikipedia.org/wiki/Deepwater_Horizon_explosion" }, "confidence": "established", "verified": true, "hold": false }, { "id": "Z-0019", "date": "2005-12-11", "when": "December 2005", "kind": "incident", "domain": "industry", "who": "People: a fuel depot", "title": "Buncefield: the gauge stuck, the switch had no padlock, and 300 tonnes of petrol went over the top", "happened": "A storage tank at a fuel depot in England was being filled overnight at 550 cubic metres an hour. From about 3 a.m. the level gauge stopped moving, as it had stuck several times since servicing. The independent high-level switch that should have shut the inflow needed a padlock fitted to work, and after maintenance it was left without one. The tank overflowed for about forty minutes, some 300 tonnes of petrol formed a vapour cloud a metre deep that spread past the site boundary, and at 6:01 a.m. it ignited. The blast measured 2.4 on the Richter scale, was heard 125 miles away, and destroyed twenty tanks. Forty-three people were injured; because it was a Sunday morning, none died. The investigation found the site's staff had no control over the flow rates of two of the three incoming pipelines, and that rising throughput had shifted attention \"to process operations instead of process safety.\"", "learned": "Two safety-critical instruments, one stuck and one disabled, and nobody knew, because nobody tested them. A safeguard that has not been tested is a belief, not a barrier. The operators could not see what was coming in and could not stop it if they had. And the injury count was low right up to the explosion.", "failed": [ "engineered", "process", "competency" ], "held": [], "source": { "name": "Buncefield Major Incident Investigation Board, final report (2008)", "url": "https://en.wikipedia.org/wiki/Buncefield_fire" }, "confidence": "established", "verified": true, "hold": false }, { "id": "Z-0020", "date": "1988-07-06", "when": "July 1988", "kind": "incident", "domain": "industry", "who": "People: an offshore platform", "title": "Piper Alpha: the permit that was not handed over, and the platforms that kept pumping", "happened": "On the North Sea platform Piper Alpha, a pump's pressure safety valve was removed for recertification and the open pipe closed with a hand-tightened blank. The permit recording that was not passed to the night shift. When the other pump failed, the night crew found the permit for the pump overhaul but not the one for the valve, and started the pump. Gas escaped past the blank and exploded, destroying the control room. The fire should have been survivable. It was not, because two neighbouring platforms kept pumping oil and gas into Piper Alpha's burning pipelines: their managers had no authority to shut down production without orders and waited for orders that never came. The gas risers ruptured. Most of the men gathered in the accommodation block as trained and waited for an evacuation order that was never given; the survivors were largely those who broke procedure and jumped. 165 of the 226 on board died, and two rescuers. The Cullen Inquiry made 106 recommendations and created the safety case regime now used offshore around the world.", "learned": "One piece of information, that a valve was missing, failed to cross one shift change, and 167 people died. The permit system existed and was defeated by how it was filed. Two managers had the means to stop the fuel and not the standing to decide; waiting for permission killed the platform. And the crew died following the plan, because no one with authority told them the plan had failed. Anyone on the work must be able to stop it, and people must be told when the plan is gone.", "failed": [ "process", "competency", "behavioural" ], "held": [], "source": { "name": "The Public Inquiry into the Piper Alpha Disaster, Lord Cullen (1990)", "url": "https://en.wikipedia.org/wiki/Piper_Alpha" }, "confidence": "established", "verified": true, "hold": false }, { "id": "Z-0021", "date": "1984-12-03", "when": "December 1984", "kind": "incident", "domain": "industry", "who": "People: a pesticide plant", "title": "Bhopal: every safeguard switched off to save money, beside half a million people", "happened": "Water entered a tank holding 42 tons of methyl isocyanate at a pesticide plant in Bhopal, India, and set off a runaway reaction. About 30 tonnes of gas escaped in under an hour over a city of people asleep. The refrigeration that would have slowed the reaction had been shut down two years earlier to save money. The vent gas scrubber was on standby. The flare tower was too small for a release that size. The tank alarms had not worked for four years. The water curtain could not reach the gas. The plant was losing money and had cut $1.25 million; half the MIC operators and half the supervisors were gone, there was no maintenance supervisor at night, and training was in English manuals few could read. A corporate audit two years before had listed 61 hazards, 30 of them major; no one followed up. A worker had died in 1981, and a local journalist had written \"Wake up, people of Bhopal, you are on the edge of a volcano.\" Officially 2,259 died at once; estimates run to 8,000 within two weeks and as many again since. How the water got in is still disputed.", "learned": "Not one barrier failed. Every barrier had been removed, one budget decision at a time, by people who each saw only their own line item. The warnings were written down, by the company's own auditors and in the newspaper, and answered by no one. The harm fell almost entirely on people outside the plant, who had no say and no warning. When the budget says no, stop the work, never the safeguard. And the group never outranks the outsider.", "failed": [ "engineered", "process", "competency", "behavioural" ], "held": [], "source": { "name": "Public record of the Bhopal disaster and its investigations", "url": "https://en.wikipedia.org/wiki/Bhopal_disaster" }, "confidence": "established", "verified": true, "hold": false }, { "id": "Z-0022", "date": "1986-04-26", "when": "April 1986", "kind": "incident", "domain": "industry", "who": "People: a nuclear plant", "title": "Chernobyl: a test run at night by a crew who did not know the reactor could do that", "happened": "A safety test at Chernobyl Unit 4 was planned for the day shift, then postponed by the grid controller, and fell to the night crew with little time to prepare. To run it they disabled the emergency core cooling and withdrew nearly all of the 211 control rods, leaving the reactor in an extremely unstable state. Two design flaws made it lethal: at low power the reactor's power rose as it boiled, and the graphite tips on the control rods gave a burst of power in the first moments of an emergency shutdown. That second flaw had been discovered in 1983; the countermeasures were never implemented and the operators were never told. When the shutdown button was pressed the reactor exploded. Two died in the blast and 28 of acute radiation sickness; long-term estimates run to thousands. The Soviet government said nothing until Sweden detected the fallout. The first report blamed the operators. The revised report, INSAG-7 in 1992, put the main cause on the reactor's design and named an inadequate safety culture at every level, which is where the phrase comes from.", "learned": "The operators broke rules, and the rules were not the barrier that mattered. A hazard the designers knew about and did not pass on was built into the machine, and the people running it could not stop what they did not know existed. The first investigation blamed the crew; the truth took six years. Know the hazards of your work, and tell the people running the machine what it can do. A test pushed to a tired crew under time pressure is a change that needs its own risk check.", "failed": [ "engineered", "process", "competency", "behavioural" ], "held": [], "source": { "name": "IAEA INSAG-7, The Chernobyl Accident: Updating of INSAG-1 (1992)", "url": "https://en.wikipedia.org/wiki/Chernobyl_disaster" }, "confidence": "established", "verified": true, "hold": false }, { "id": "Z-0023", "date": "1979-03-28", "when": "March 1979", "kind": "incident", "domain": "industry", "who": "People: a nuclear plant control room", "title": "Three Mile Island: the operators overrode the automatic safety system, because the display lied", "happened": "At 4 a.m. a relief valve at Three Mile Island Unit 2 stuck open and began draining the reactor's coolant. The control room light said the valve was closed: it showed only that the signal to close had been sent, not what the valve had done. The one gauge the operators trusted, the pressurizer level, rose instead of falling, so they concluded the reactor had too much water. The automatic emergency cooling had started on its own. The operators throttled it back. For about 80 minutes the core was uncovered, and roughly half the fuel melted. Hundreds of alarms sounded at once with no way to tell which mattered. The same valve failure had happened at another plant eighteen months earlier, and the manufacturer had not clearly told its customers. The release to the public was small. The Kemeny Commission called for fundamental changes in the organization, procedures, practices, \"and above all, in the attitudes\" of the industry and its regulator.", "learned": "The machine had stopped the work, correctly, and the people stopped the machine, because their instruments told them a story that was wrong. Honoring a stop means finding out what the stopper sees before overriding it, and the operators could not, because the display did not show the valve's real state. An indicator that reports the command instead of the result is not an indicator. And the same failure had already happened elsewhere, unshared: a free lesson, unlearned.", "failed": [ "engineered", "process", "competency" ], "held": [ "engineered" ], "source": { "name": "Report of the President's Commission on the Accident at Three Mile Island (the Kemeny Commission), 1979", "url": "https://en.wikipedia.org/wiki/Three_Mile_Island_accident" }, "confidence": "established", "verified": true, "hold": false }, { "id": "Z-0024", "date": "1974-06-01", "when": "June 1974", "kind": "incident", "domain": "industry", "who": "People: a chemical plant", "title": "Flixborough: a pipe designed in chalk on the workshop floor", "happened": "A reactor at a chemical plant in England cracked and was removed. To keep production going, a bypass was built to connect the reactors on either side: a dog-leg of 20-inch pipe where the reactors used 28-inch, joined with bellows. No drawing was made except in chalk on the workshop floor. It was not pressure-tested to the required standard. Nobody worked out that under pressure the assembly would try to twist, with some 38 tonnes of thrust on the bellows. The plant had no qualified mechanical engineer; the post had been vacant for five months. On a Saturday afternoon the bypass ruptured, releasing hot cyclohexane that formed a vapour cloud and exploded. Twenty-eight people died, including all eighteen in the control room, and 36 were seriously injured. About 2,000 properties were damaged. The inquiry led to the framework that became the UK's major hazards regime.", "learned": "A change made to keep production going, by people without the competence to make it, with no design, no test, and no one whose job it was to say no. The permanent plant had been engineered; the temporary fix was not, and the temporary fix is what failed. Think it through before you start applies most to the change nobody planned. And a site running without the person qualified to check the work is a site running without a barrier.", "failed": [ "engineered", "process", "competency" ], "held": [], "source": { "name": "The Flixborough Disaster: Report of the Court of Inquiry (1975)", "url": "https://en.wikipedia.org/wiki/Flixborough_disaster" }, "confidence": "established", "verified": true, "hold": false }, { "id": "Z-0025", "date": "1976-07-10", "when": "July 1976", "kind": "incident", "domain": "industry", "who": "People: a chemical plant, and a town", "title": "Seveso: a batch left hot over the weekend, and a week before anyone was told", "happened": "At a chemical plant north of Milan, a batch was stopped partway to comply with a weekend shutdown rule and left in the reactor, unstirred. As the rest of the site shut down, the steam heating the reactor jacket rose from 190 to about 300 degrees; the operators had no instrument that showed it. The batch began to decompose at a temperature 50 degrees lower than anyone had expected, ran away, and burst the relief disc, releasing a cloud containing dioxin over the towns downwind. The company at first would not confirm what had been released. At least a week passed before a public statement, and another before evacuation began. 193 people developed chloracne, none of them plant workers; 3,300 animals were found dead within weeks and some 80,000 were later culled. The disaster gave its name to the European Seveso Directive, the law that governs major-hazard sites across Europe to this day.", "learned": "The hazard was created by a rule about working hours, applied without asking what it did to the chemistry, and the operators could not see the temperature that mattered. Then the second failure: the people in the path of the harm were the last to be told. Speaking up is not only inside the plant. When a release is out, tell the people it is heading for, first and fast. The harm fell entirely on outsiders, who had no say and no warning.", "failed": [ "engineered", "process", "competency", "behavioural" ], "held": [], "source": { "name": "Public record of the Seveso disaster and the origin of the Seveso Directive (1982)", "url": "https://en.wikipedia.org/wiki/Seveso_disaster" }, "confidence": "established", "verified": true, "hold": false } ], "domains": { "industry": "Industry: what people learned, at great cost, in refineries, rigs, aircraft, and plants.", "ai": "AI: the same patterns, from a new kind of worker." } }