{"schema_version":"onlylabs.public_signal.v1","title":"AI21 Labs Writing: Evaluating Coding Agents On The Work Developers Actually Do","description":"AI21 Labs writing signal with public source context, captured evidence pages, related signals, and category-scoped analysis context.","url":"https://onlylabs.fyi/signals/8e54ab28-8aed-4f53-ab84-3ddfc5248ff6","json_url":"https://onlylabs.fyi/signals/8e54ab28-8aed-4f53-ab84-3ddfc5248ff6/signal.json","generated_at":"2026-10-24T20:59:55.574Z","evidence_latest_fetched_at":"2026-10-14T12:02:37.568267+00:00","signal_first_seen_at":"2026-10-14T12:00:50.119522+00:00","org":{"slug":"ai21","name":"AI21 Labs","category":"neolab","category_label":"Neolab","dossier_url":"https://onlylabs.fyi/labs/ai21","dossier_json_url":"https://onlylabs.fyi/labs/ai21/dossier.json"},"related_urls":{"signal":"https://onlylabs.fyi/signals/8e54ab28-8aed-4f53-ab84-3ddfc5248ff6","signal_json":"https://onlylabs.fyi/signals/8e54ab28-8aed-4f53-ab84-3ddfc5248ff6/signal.json","source":"https://www.ai21.com/blog/evaluating-coding-agents-on-the-work-developers-actually-do/","lab_dossier":"https://onlylabs.fyi/labs/ai21","lab_dossier_json":"https://onlylabs.fyi/labs/ai21/dossier.json","analysis":"https://onlylabs.fyi/analysis/ai21","analysis_json":"https://onlylabs.fyi/analysis/ai21/analysis.json","analysis_evidence_json":"https://onlylabs.fyi/analysis/ai21/evidence.json","category":"https://onlylabs.fyi/neolabs","category_json":"https://onlylabs.fyi/neolabs.json","category_feed":"https://onlylabs.fyi/neolabs/feed.xml","category_signals_json":"https://onlylabs.fyi/signals.json?category=neolab","topic":"https://onlylabs.fyi/topics/talking","topic_signals_json":"https://onlylabs.fyi/topics/talking/signals.json?category=neolab","topic_feed":"https://onlylabs.fyi/topics/talking/feed.xml?category=neolab","data_business":null},"answer_pack":{"answer":"AI21 Labs published Evaluating Coding Agents On The Work Developers Actually Do. This talking signal gives public context for research themes, product direction, policy, or launch framing. High-signal details: Substantive research post on coding agent evaluation · The hidden mismatch: Evaluating coding agents on the work developers actually do | AI21 Skip to Main Menu Skip to Main Content Skip to Footer Back to Blog --> Back to.... onlylabs links this event to 1 captured evidence page and 6 related writing signals.","signal_desk":"talking","source_context":{"source_url":"https://www.ai21.com/blog/evaluating-coding-agents-on-the-work-developers-actually-do/","source_host":"ai21.com","occurred_at":"2026-10-14T10:59:51+00:00","first_seen_at":"2026-10-14T12:00:50.119522+00:00","date_source":"sitemap.lastmod","context":null},"context_markers":[{"label":"Lab","value":"AI21 Labs","source":"signal"},{"label":"Signal desk","value":"talking","source":"signal"},{"label":"Source host","value":"ai21.com","source":"source"},{"label":"Notability","value":"Substantive research post on coding agent evaluation","source":"signal"},{"label":"Watch term","value":"Eval methodology","source":"evidence"},{"label":"Watch term","value":"Data pipeline","source":"evidence"},{"label":"Watch term","value":"Infrastructure","source":"evidence"},{"label":"Watch term","value":"Agents and tool use","source":"evidence"}],"evidence_coverage":{"target_pages":1,"captured_pages":1,"readable_pages":1,"capture_methods":["plain"],"missing_page_urls":[],"failed_page_urls":[],"blocked_page_urls":[],"page_urls":["https://www.ai21.com/blog/evaluating-coding-agents-on-the-work-developers-actually-do/"],"related_signals":6,"has_source_url":true,"latest_page_fetched_at":"2026-10-14T12:02:37.568267+00:00"},"data_business":{"matches":false,"lanes":[],"matched_terms":[],"score":null,"reason":null},"agent_handoff":{"signal_json":"https://onlylabs.fyi/signals/8e54ab28-8aed-4f53-ab84-3ddfc5248ff6/signal.json","dossier_json":"https://onlylabs.fyi/labs/ai21/dossier.json","analysis_json":"https://onlylabs.fyi/analysis/ai21/analysis.json","analysis_evidence_json":"https://onlylabs.fyi/analysis/ai21/evidence.json","topic_signals_json":"https://onlylabs.fyi/topics/talking/signals.json?category=neolab","topic_feed":"https://onlylabs.fyi/topics/talking/feed.xml?category=neolab","category_signals_json":"https://onlylabs.fyi/signals.json?category=neolab","data_radar_json":null,"opportunities_json":null},"analysis_playbook":{"objective":"Turn public writing and discussion into a readable map of research themes, product framing, policy posture, launch narratives, and market attention.","evidence_focus":["post title","source URL","captured page text","HN traction","linked model or paper references","publication date"],"extraction_questions":["Which themes are labs choosing to explain publicly?","Which posts are attracting outside discussion?","Which writing reframes a recent release, model, hiring wave, or policy stance?","Which posts mention data, evals, infrastructure, safety, or deployment workflows?"],"signal_questions":["What public theme, launch framing, or research direction does this writing signal expose?","Which themes are labs choosing to explain publicly?","Which posts are attracting outside discussion?","Do the 6 related writing signals show a repeated pattern?"],"output_fields":["org","theme","public_framing","traction","evidence_url"],"data_business_relevance":"Data-business lane extraction is scoped to frontier labs; for this category, keep conclusions tied to category-specific strategy, source evidence, and follow-up questions.","required_sources":[{"label":"signal_json","url":"https://onlylabs.fyi/signals/8e54ab28-8aed-4f53-ab84-3ddfc5248ff6/signal.json","required":true},{"label":"source","url":"https://www.ai21.com/blog/evaluating-coding-agents-on-the-work-developers-actually-do/","required":true},{"label":"dossier_json","url":"https://onlylabs.fyi/labs/ai21/dossier.json","required":true},{"label":"analysis_evidence_json","url":"https://onlylabs.fyi/analysis/ai21/evidence.json","required":true},{"label":"topic_signals_json","url":"https://onlylabs.fyi/topics/talking/signals.json?category=neolab","required":false},{"label":"data_radar_json","url":null,"required":false}],"expected_output":["one-paragraph source-grounded interpretation","category-specific implication","confidence and missing evidence","recommended next source to inspect"],"prompt_seed":"Using only the linked onlylabs JSON, captured source context, and cited evidence, analyze AI21 Labs's writing signal \"Evaluating Coding Agents On The Work Developers Actually Do\" for neolab strategy."},"semantic_triples":[{"subject":"AI21 Labs","predicate":"published","object":"Evaluating Coding Agents On The Work Developers Actually Do","text":"AI21 Labs published Evaluating Coding Agents On The Work Developers Actually Do."},{"subject":"Evaluating Coding Agents On The Work Developers Actually Do","predicate":"is classified as","object":"writing signal","text":"Evaluating Coding Agents On The Work Developers Actually Do is classified as writing signal."},{"subject":"Evaluating Coding Agents On The Work Developers Actually Do","predicate":"belongs to","object":"talking desk","text":"Evaluating Coding Agents On The Work Developers Actually Do belongs to talking desk."},{"subject":"Evaluating Coding Agents On The Work Developers Actually Do","predicate":"has evidence coverage","object":"1 captured evidence page","text":"Evaluating Coding Agents On The Work Developers Actually Do has evidence coverage 1 captured evidence page."},{"subject":"Evaluating Coding Agents On The Work Developers Actually Do","predicate":"has captured page count","object":"1","text":"Evaluating Coding Agents On The Work Developers Actually Do has captured page count 1."},{"subject":"Evaluating Coding Agents On The Work Developers Actually Do","predicate":"has readable page count","object":"1","text":"Evaluating Coding Agents On The Work Developers Actually Do has readable page count 1."},{"subject":"Evaluating Coding Agents On The Work Developers Actually Do","predicate":"has related signal count","object":"6","text":"Evaluating Coding Agents On The Work Developers Actually Do has related signal count 6."},{"subject":"Evaluating Coding Agents On The Work Developers Actually Do","predicate":"has analysis playbook objective","object":"Turn public writing and discussion into a readable map of research themes, product framing, policy posture, launch narratives, and market attention.","text":"Evaluating Coding Agents On The Work Developers Actually Do has analysis playbook objective Turn public writing and discussion into a readable map of research themes, product framing, policy posture, launch narratives, and market attention.."},{"subject":"Evaluating Coding Agents On The Work Developers Actually Do","predicate":"has source host","object":"ai21.com","text":"Evaluating Coding Agents On The Work Developers Actually Do has source host ai21.com."},{"subject":"Evaluating Coding Agents On The Work Developers Actually Do","predicate":"has lab","object":"AI21 Labs","text":"Evaluating Coding Agents On The Work Developers Actually Do has lab AI21 Labs."},{"subject":"Evaluating Coding Agents On The Work Developers Actually Do","predicate":"has signal desk","object":"talking","text":"Evaluating Coding Agents On The Work Developers Actually Do has signal desk talking."},{"subject":"Evaluating Coding Agents On The Work Developers Actually Do","predicate":"has source host","object":"ai21.com","text":"Evaluating Coding Agents On The Work Developers Actually Do has source host ai21.com."},{"subject":"Evaluating Coding Agents On The Work Developers Actually Do","predicate":"has notability","object":"Substantive research post on coding agent evaluation","text":"Evaluating Coding Agents On The Work Developers Actually Do has notability Substantive research post on coding agent evaluation."},{"subject":"Evaluating Coding Agents On The Work Developers Actually Do","predicate":"has watch term","object":"Eval methodology","text":"Evaluating Coding Agents On The Work Developers Actually Do has watch term Eval methodology."},{"subject":"Evaluating Coding Agents On The Work Developers Actually Do","predicate":"has watch term","object":"Data pipeline","text":"Evaluating Coding Agents On The Work Developers Actually Do has watch term Data pipeline."},{"subject":"Evaluating Coding Agents On The Work Developers Actually Do","predicate":"has watch term","object":"Infrastructure","text":"Evaluating Coding Agents On The Work Developers Actually Do has watch term Infrastructure."},{"subject":"Evaluating Coding Agents On The Work Developers Actually Do","predicate":"has watch term","object":"Agents and tool use","text":"Evaluating Coding Agents On The Work Developers Actually Do has watch term Agents and tool use."}]},"intelligence":{"signal_desk":"talking","answer":"AI21 Labs published Evaluating Coding Agents On The Work Developers Actually Do. This talking signal gives public context for research themes, product direction, policy, or launch framing. High-signal details: Substantive research post on coding agent evaluation · The hidden mismatch: Evaluating coding agents on the work developers actually do | AI21 Skip to Main Menu Skip to Main Content Skip to Footer Back to Blog --> Back to.... onlylabs links this event to 1 captured evidence page and 6 related writing signals.","semantic_triples":[{"subject":"AI21 Labs","predicate":"published","object":"Evaluating Coding Agents On The Work Developers Actually Do","text":"AI21 Labs published Evaluating Coding Agents On The Work Developers Actually Do."},{"subject":"Evaluating Coding Agents On The Work Developers Actually Do","predicate":"is classified as","object":"writing signal","text":"Evaluating Coding Agents On The Work Developers Actually Do is classified as writing signal."},{"subject":"Evaluating Coding Agents On The Work Developers Actually Do","predicate":"belongs to","object":"talking desk","text":"Evaluating Coding Agents On The Work Developers Actually Do belongs to talking desk."},{"subject":"Evaluating Coding Agents On The Work Developers Actually Do","predicate":"has evidence coverage","object":"1 captured evidence page","text":"Evaluating Coding Agents On The Work Developers Actually Do has evidence coverage 1 captured evidence page."}]},"signal":{"id":"8e54ab28-8aed-4f53-ab84-3ddfc5248ff6","url":"https://onlylabs.fyi/signals/8e54ab28-8aed-4f53-ab84-3ddfc5248ff6","json_url":"https://onlylabs.fyi/signals/8e54ab28-8aed-4f53-ab84-3ddfc5248ff6/signal.json","source_url":"https://www.ai21.com/blog/evaluating-coding-agents-on-the-work-developers-actually-do/","title":"Evaluating Coding Agents On The Work Developers Actually Do","summary":"AI21 Labs published a writing signal. onlylabs watches public writing for research themes, product direction, and model-launch context.","context":null,"kind":{"key":"post_published","label":"Writing"},"org":{"slug":"ai21","name":"AI21 Labs","category":"neolab"},"occurred_at":"2026-10-14T10:59:51+00:00","first_seen_at":"2026-10-14T12:00:50.119522+00:00","date_source":"sitemap.lastmod","evidence_coverage":{"target_pages":1,"captured_pages":1,"readable_pages":1,"capture_methods":["plain"],"missing_page_urls":[],"failed_page_urls":[],"blocked_page_urls":[],"page_urls":["https://www.ai21.com/blog/evaluating-coding-agents-on-the-work-developers-actually-do/"]},"facets":{},"traction":{"github_stars":null,"hn_points":null,"hn_comments":null,"hn_story_id":null,"hf_downloads":null,"hf_likes":null},"data_radar":null},"primary_evidence_page":{"is_primary":true,"source_match":true,"url":"https://www.ai21.com/blog/evaluating-coding-agents-on-the-work-developers-actually-do/","final_url":"https://www.ai21.com/blog/evaluating-coding-agents-on-the-work-developers-actually-do/","title":"Evaluating Coding Agents On The Work Developers Actually Do","http_status":200,"content_type":"text/html; charset=UTF-8","capture_method":"plain","fetched_at":"2026-10-14T12:02:37.568267+00:00","bytes":69636,"raw_path":"28427f6f42ba5b808012fffa6493b508a738c873e2a62465e8c510750883c1ef.html","content_hash":"84c6d1df92832ee5d7b6f658134ca2dea1c6f2e3fcb47f15b013d0958abb93d1","excerpt_chars":1200,"truncated":true,"excerpt":"The hidden mismatch: Evaluating coding agents on the work developers actually do | AI21 Skip to Main Menu Skip to Main Content Skip to Footer Back to Blog --> Back to Blog Back to Blog --> In brief We’ve all heard it before: your AI system – whether an agent, harness, or routing layer – is only as good as the evaluation behind it. We’ve also heard: public benchmarks are poor proxies for real developer traffic. In terms of what they ask for, how they phrase it, task difficulty, and how trajectories unfold over the session. We saw this discrepancy up close when we surveyed our internal developer traffic. The results challenged our assumptions about agentic coding tasks, and we used these lessons to build a robust benchmark that preserves quality on real-world coding tasks in our intelligent gateway. This post covers that journey. The real distribution of developer work Luckily, to understand the gap between public benchmarks and real usage, we have the benefit of real live data from our internal team of developers. To make sense of this traffic, we developed an internal coding task classifier and ran seven weeks of real developer traffic through it. What we..."},"evidence_pages":[],"related_signals":[{"id":"87e25e7d-7e62-427c-b015-700a0f7f99fd","url":"https://onlylabs.fyi/signals/87e25e7d-7e62-427c-b015-700a0f7f99fd","source_url":"https://www.ai21.com/blog/gitlab-case-study/","title":"Gitlab Case Study","context":null,"kind":{"key":"post_published","label":"Writing"},"org":{"slug":"ai21","name":"AI21 Labs","category":"neolab"},"occurred_at":"2026-10-22T11:46:01+00:00","first_seen_at":"2026-10-22T16:00:51.253352+00:00","date_source":"sitemap.lastmod"},{"id":"bd82846b-1464-41ec-96da-dfa629a7de57","url":"https://onlylabs.fyi/signals/bd82846b-1464-41ec-96da-dfa629a7de57","source_url":"https://www.ai21.com/blog/ai21-post-trains-nvidia-nemotron-35-super-vl/","title":"Ai21 Post Trains Nvidia Nemotron 35 Super Vl","context":null,"kind":{"key":"post_published","label":"Writing"},"org":{"slug":"ai21","name":"AI21 Labs","category":"neolab"},"occurred_at":"2026-10-21T14:31:28+00:00","first_seen_at":"2026-10-15T16:00:49.904373+00:00","date_source":"sitemap.lastmod"},{"id":"c1c6fce4-418d-446f-8d65-85c4078099f2","url":"https://onlylabs.fyi/signals/c1c6fce4-418d-446f-8d65-85c4078099f2","source_url":"https://www.ai21.com/blog/the-verifiability-litmus-test-for-agent-design/","title":"The Verifiability Litmus Test For Agent Design","context":null,"kind":{"key":"post_published","label":"Writing"},"org":{"slug":"ai21","name":"AI21 Labs","category":"neolab"},"occurred_at":"2026-10-15T11:57:59+00:00","first_seen_at":"2026-10-06T12:00:50.033093+00:00","date_source":"sitemap.lastmod"},{"id":"cde2da8b-28fa-41ae-86d4-d2945c919051","url":"https://onlylabs.fyi/signals/cde2da8b-28fa-41ae-86d4-d2945c919051","source_url":"https://www.ai21.com/blog/how-ai21-manages-its-gpu-fleet-with-kueue/","title":"How Ai21 Manages Its Gpu Fleet With Kueue","context":null,"kind":{"key":"post_published","label":"Writing"},"org":{"slug":"ai21","name":"AI21 Labs","category":"neolab"},"occurred_at":"2026-10-13T08:30:56+00:00","first_seen_at":"2026-10-04T16:00:50.34925+00:00","date_source":"sitemap.lastmod"},{"id":"34553295-d3bc-46e7-aaaa-114772922ebe","url":"https://onlylabs.fyi/signals/34553295-d3bc-46e7-aaaa-114772922ebe","source_url":"https://www.ai21.com/blog/you-need-a-verifier/","title":"You Need A Verifier","context":null,"kind":{"key":"post_published","label":"Writing"},"org":{"slug":"ai21","name":"AI21 Labs","category":"neolab"},"occurred_at":"2026-08-20T10:41:58+00:00","first_seen_at":"2026-08-19T12:00:50.061172+00:00","date_source":"sitemap.lastmod"},{"id":"bd5a5a6f-9216-49af-a6a8-ed150d4acb05","url":"https://onlylabs.fyi/signals/bd5a5a6f-9216-49af-a6a8-ed150d4acb05","source_url":"https://www.ai21.com/blog/mind-the-gap/","title":"Mind The Gap","context":null,"kind":{"key":"post_published","label":"Writing"},"org":{"slug":"ai21","name":"AI21 Labs","category":"neolab"},"occurred_at":"2026-08-18T14:51:43+00:00","first_seen_at":"2026-06-26T20:26:30.018378+00:00","date_source":"sitemap.lastmod"}]}