[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"glossary-llm-as-judge::en":3,"gloss-cluster-llm-as-judge::en":20,"gloss-next-llm-as-judge::en":9},{"slug":4,"category":5,"name":6,"definition":7,"meta_desc":8,"faq":9,"schema_markup":9,"related":10},"llm-as-judge","prompt-eng","LLM-as-Judge","LLM-as-judge is an evaluation technique where a large language model itself is used to score, critique, or compare the outputs of another model call (or of two different prompt versions applied to the same input), following a defined rubric provided in the judge's own prompt — a specialized application of the meta prompt pattern, used specifically to automate quality evaluation at a scale manual human review can't practically match. This has become one of the most important tools in production prompt-engineering and AI-product-development workflows because traditional software testing (exact-match assertions) doesn't work well for free-text, non-deterministic LLM output — two correct summaries of the same document can be worded completely differently, so you can't simply assert \"output equals expected string.\" LLM-as-judge fills that gap by using a model's own language-understanding capability to assess more nuanced qualities: factual accuracy against a reference, adherence to a specified tone or format, helpfulness, conciseness, or a head-to-head comparison between two candidate outputs (\"Response A vs Response B: which better addresses the user's question? Respond A, B, or TIE\"). It's used for pairwise comparison (deciding whether a new prompt version is better than the current production version, across an evaluation set), absolute scoring (rating a single output 1-10 against a rubric, useful for tracking quality trends over time), and reference-based grading (comparing a model's answer against a known correct answer for factual-accuracy tasks). A well-known limitation is that LLM judges have their own biases — a tendency toward favoring longer responses (verbosity bias), favoring responses that resemble the judge's own writing style (self-preference bias when judge and generator are the same model family), and positional bias in pairwise comparisons (favoring whichever response is presented first) — so mature evaluation setups randomize response order, occasionally validate judge scores against a sample of real human ratings, and sometimes use a different, typically stronger model as judge than the one being evaluated, to reduce self-preference bias. Concrete worked example: a team maintaining an AI customer-email-response generator wants to compare a new prompt version against the current production version across 300 historical support tickets before deciding whether to ship it. Manually reading 600 response pairs isn't practical for a routine prompt change, so they build an LLM-as-judge evaluation: \"Given this customer ticket and two candidate agent responses, score each 1-10 on helpfulness and professionalism, then state which is better overall. TICKET: {{ticket}}. RESPONSE A: {{response_a}}. RESPONSE B: {{response_b}}.\" Run across all 300 pairs with response order randomized to control positional bias, the new version wins 214-86 — a result that gives the team quantitative, scalable confidence to ship, spot-checked against a 20-ticket human-reviewed sample to confirm the judge's scoring aligns with actual human quality judgment.","LLM-as-judge uses one AI model to evaluate, score, or compare the outputs of another AI model (or prompt) against a defined rubric.",null,[11,14,17],{"slug":12,"name":13},"meta-prompt","Meta Prompt",{"slug":15,"name":16},"prompt-versioning","Prompt Versioning",{"slug":18,"name":19},"self-consistency","Self-Consistency",[21,25,28,31,35,38,41,44,47,50,53,56],{"slug":22,"category":5,"name":23,"updated_at":24},"analogical-prompting","Analogical Prompting","2026-08-24T02:46:37+00:00",{"slug":26,"category":5,"name":27,"updated_at":24},"automatic-prompt-optimization","Automatic Prompt Optimization",{"slug":29,"category":5,"name":30,"updated_at":24},"chain-of-density","Chain of Density (CoD)",{"slug":32,"category":5,"name":33,"updated_at":34},"chain-of-thought-prompting","Chain-of-Thought Prompting","2026-08-24T02:46:36+00:00",{"slug":36,"category":5,"name":37,"updated_at":24},"chain-of-verification","Chain-of-Verification",{"slug":39,"category":5,"name":40,"updated_at":34},"chunking","Chunking",{"slug":42,"category":5,"name":43,"updated_at":34},"constrained-decoding","Constrained Decoding",{"slug":45,"category":5,"name":46,"updated_at":34},"context-stuffing","Context Stuffing",{"slug":48,"category":5,"name":49,"updated_at":34},"delimiter","Delimiter",{"slug":51,"category":5,"name":52,"updated_at":24},"directional-stimulus-prompting","Directional Stimulus Prompting",{"slug":54,"category":5,"name":55,"updated_at":24},"emotion-prompting","Emotion Prompting",{"slug":57,"category":5,"name":58,"updated_at":34},"few-shot-prompting","Few-Shot Prompting"]