Runs the golden-question sets (core + paraphrase) against each mode and reports hit@5 and tokens-per-query. Run from the repo root (the corpus paths are repo-relative).This is the gate for every retrieval feature: a mode or ranking stage only becomes the default when it beats the current default here. Current standing: BM25 default; hybrid ships opt-in after tying (not beating) it.
{"docsConfig":{"theme":"mint","$schema":"https://mintlify.com/docs.json","name":"DuckBrain","description":"Local-first DuckDB/DuckLake brain for AI agents","colors":{"primary":"#F5A623","light":"#FFC966","dark":"#B87A00"},"footer":{"socials":{"github":"https://github.com/autonoco/duckbrain"}},"styling":{"latex":false},"navigation":{"pages":[]}},"deployment":{"deploymentMetadataValue":{"subdomain":"duckbrain","hasAudioNarration":false,"actualSubdomain":"duckbrain","customDomains":["docs.duckbrain.sh"],"gitSource":{"type":"github","owner":"autonoco","repo":"duckbrain","deployBranch":"main","contentDirectory":"docs-site","isPrivate":true,"mountPath":"","label":""},"feedback":{"agentFeedback":true},"entitlements":{},"buildId":"6ae95c4adbcba01b9ab87102:success","clientVersion":"0.0.4102","searchFilterCounts":[{"language":"default","tag":"Ingest Sources","count":21},{"language":"default","tag":"Commands","count":7},{"language":"default","tag":"Guide","count":3},{"language":"default","tag":"Get Started","count":2}]},"assistantConfig":{"enableAskAiButton":false,"enableStarterQuestions":false,"enableContextualStarterQuestions":false,"enableAskAiSearchBar":true},"basePath":"","hasChatPermissions":false,"hasWebMcpPermissions":false,"toggles":[{"name":"kill-isr","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"custom-rbac-roles","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"agent-mintlify-slack","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"agent-integrations-settings","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"rbac-expansion","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"automations-pr-grouping","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"custom-prefix","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"analytics-v7","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"user-flows-analytics","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"no-publish-flow","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"workos-onboarding","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"llms-txt-v2","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"snapshot-serving-live","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"stripTemperatureMiddleware","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"support-platform-pylon-zendesk","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"snapshot-serving","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"instant-rollback","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"granular-notifications-server","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"granular-notifications","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"automations-fable","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"fixed-ai-action-pricing","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"automation-resolve-merge-conflict","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"scoped-publishing","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"mintql","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"page-level-analytics","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"editor-version-history-v2","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"billing-page-v2","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"full-page-search","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"workspace-projects","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"assistant-v5","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"fork-pr-previews","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"revalidation-jwt","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"workos-sso-scim","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"hobby-abuse-scan","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"dashboard-inbox","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"workos-signup","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"turbopuffer-dual-write","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"workos-enabled","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"approve-review-flow","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"editor-metadata","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"sso-setup-deadline-banner","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"turbopuffer-serve","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"live-agents","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false}],"googleFontsInlined":false},"customCss":[],"page":{"pagePath":"commands/eval","slug":"/commands/eval","pageMetadata":{"title":"eval","description":"The thesis harness: golden questions vs baselines","href":"/commands/eval"},"description":{"compiledSource":"\"use strict\";\nconst {jsx: _jsx} = arguments[0];\nconst {useMDXComponents: _provideComponents} = arguments[0];\nfunction _createMdxContent(props) {\n const _components = {\n p: \"p\",\n ..._provideComponents(),\n ...props.components\n };\n return _jsx(_components.p, {\n children: \"The thesis harness: golden questions vs baselines\"\n });\n}\nfunction MDXContent(props = {}) {\n const {wrapper: MDXLayout} = {\n ..._provideComponents(),\n ...props.components\n };\n return MDXLayout ? _jsx(MDXLayout, {\n ...props,\n children: _jsx(_createMdxContent, {\n ...props\n })\n }) : _createMdxContent(props);\n}\nreturn {\n default: MDXContent\n};\n","frontmatter":{},"scope":{}},"pageType":"default","mdx":{"mdxSource":{"compiledSource":"\"use strict\";const{Fragment:_Fragment,jsx:_jsx,jsxs:_jsxs}=arguments[0];const{useMDXComponents:_provideComponents}=arguments[0];function _createMdxContent(props){const _components={code:`code`,p:`p`,pre:`pre`,span:`span`,strong:`strong`,..._provideComponents(),...props.components},{CodeBlock}=_components;if(!CodeBlock)_missingMdxReference(`CodeBlock`,true);return _jsxs(_Fragment,{children:[_jsx(CodeBlock,{filename:``,numberOfLines:`1`,language:`shellscript`,children:_jsx(_components.pre,{className:`shiki shiki-themes github-light-default dark-plus`,style:{backgroundColor:`#ffffff`,\"--shiki-dark-bg\":`#0B0C0E`,color:`#1f2328`,\"--shiki-dark\":`#D4D4D4`},language:`shellscript`,children:_jsxs(_components.code,{language:`shellscript`,numberOfLines:`1`,children:[_jsxs(_components.span,{className:`line`,children:[_jsx(_components.span,{style:{color:`#953800`,\"--shiki-dark\":`#DCDCAA`},children:`duckbrain`}),_jsx(_components.span,{style:{color:`#0A3069`,\"--shiki-dark\":`#CE9178`},children:` eval`}),_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:` [--modes `}),_jsx(_components.span,{style:{color:`#0A3069`,\"--shiki-dark\":`#CE9178`},children:`bm25,vec,hybrid]`})]}),`\n`]})})}),`\n`,_jsx(_components.p,{children:`Runs the golden-question sets (core + paraphrase) against each mode and reports hit@5 and tokens-per-query. Run from the repo root (the corpus paths are repo-relative).`}),`\n`,_jsxs(_components.p,{children:[`This is the gate for every retrieval feature: `,_jsx(_components.strong,{children:`a mode or ranking stage only becomes the default when it beats the current default here.`}),` Current standing: BM25 default; hybrid ships opt-in after tying (not beating) it.`]})]})}function MDXContent(props={}){const{wrapper:MDXLayout}={..._provideComponents(),...props.components};return MDXLayout?_jsx(MDXLayout,{...props,children:_jsx(_createMdxContent,{...props})}):_createMdxContent(props)}return{default:MDXContent};function _missingMdxReference(id,component){throw new Error(`Expected `+(component?`component`:`object`)+\" `\"+id+\"` to be defined: you likely forgot to import, pass, or provide it.\")}","frontmatter":{},"scope":{"config":{},"pageMetadata":{"title":"eval","description":"The thesis harness: golden questions vs baselines","href":"/commands/eval"}}},"mdxSourceWithNoJs":null},"mdxExtracts":{"tableOfContents":[],"codeExamples":{}},"jsLinks":[],"subdomain":"duckbrain","siteName":"DuckBrain","title":"eval - DuckBrain","navMaps":{"firstHrefInVersion":{},"firstHrefInLanguage":{}},"lastModified":"2026-07-16T15:50:30.173Z","originalFileLocation":"commands/eval.mdx"},"headHtml":"\u003cscript data-mint-head>(function(){var d=document.documentElement,p=null;if(!p){try{p=localStorage.getItem(\"isDarkMode\")}catch(e){}if(p==='true')p='dark';else if(p==='false')p='light';if(p!=='light'\u0026\u0026p!=='dark'\u0026\u0026p!=='system')p=\"system\"}var r=p==='system'?(matchMedia(\"(prefers-color-scheme: dark)\").matches?'dark':'light'):p;d.classList.remove(r==='dark'?'light':'dark');d.classList.add(r);d.setAttribute(\"data-theme-preference\",p)})();\u003c/script>\u003cmeta data-mint-head name=\"description\" content=\"The thesis harness: golden questions vs baselines\">\u003cmeta data-mint-head name=\"application-name\" content=\"DuckBrain\">\u003cmeta data-mint-head name=\"generator\" content=\"Mintlify\">\u003clink data-mint-head rel=\"apple-touch-icon\" type=\"image/png\" sizes=\"180x180\" href=\"https://mintlify.s3.us-west-1.amazonaws.com/duckbrain/_generated/favicon/apple-touch-icon.png?v=3\">\u003clink data-mint-head rel=\"icon\" type=\"image/png\" sizes=\"32x32\" href=\"https://mintlify.s3.us-west-1.amazonaws.com/duckbrain/_generated/favicon/favicon-32x32.png?v=3\">\u003clink data-mint-head rel=\"icon\" type=\"image/png\" sizes=\"16x16\" href=\"https://mintlify.s3.us-west-1.amazonaws.com/duckbrain/_generated/favicon/favicon-16x16.png?v=3\">\u003clink data-mint-head rel=\"shortcut icon\" type=\"image/x-icon\" href=\"https://mintlify.s3.us-west-1.amazonaws.com/duckbrain/_generated/favicon/favicon.ico?v=3\">\u003cmeta data-mint-head property=\"og:title\" content=\"eval - DuckBrain\">\u003cmeta data-mint-head property=\"og:description\" content=\"The thesis harness: golden questions vs baselines\">\u003cmeta data-mint-head property=\"og:site_name\" content=\"DuckBrain\">\u003cmeta data-mint-head property=\"og:url\" content=\"https://docs.duckbrain.sh/commands/eval\">\u003cmeta data-mint-head property=\"og:type\" content=\"website\">\u003cmeta data-mint-head property=\"og:image\" content=\"https://duckbrain.mintlify.site/_mintlify/api/og/v2?division=Commands\u0026amp;title=eval\u0026amp;description=The+thesis+harness%3A+golden+questions+vs+baselines\u0026amp;theme=586da6f3a2f51b1623084080\u0026amp;signature=909e57339cfc203286761bbde55227f6c59b797c24e31a24bd89f8a01a863701\">\u003cmeta data-mint-head property=\"og:image:width\" content=\"1200\">\u003cmeta data-mint-head property=\"og:image:height\" content=\"630\">\u003cmeta data-mint-head name=\"twitter:card\" content=\"summary_large_image\">\u003cmeta data-mint-head name=\"twitter:title\" content=\"eval - DuckBrain\">\u003cmeta data-mint-head name=\"twitter:description\" content=\"The thesis harness: golden questions vs baselines\">\u003cmeta data-mint-head name=\"twitter:image\" content=\"https://duckbrain.mintlify.site/_mintlify/api/og/v2?division=Commands\u0026amp;title=eval\u0026amp;description=The+thesis+harness%3A+golden+questions+vs+baselines\u0026amp;theme=586da6f3a2f51b1623084080\u0026amp;signature=909e57339cfc203286761bbde55227f6c59b797c24e31a24bd89f8a01a863701\">\u003cmeta data-mint-head name=\"twitter:image:width\" content=\"1200\">\u003cmeta data-mint-head name=\"twitter:image:height\" content=\"630\">\u003cmeta data-mint-head name=\"msapplication-config\" content=\"https://mintlify.s3.us-west-1.amazonaws.com/duckbrain/_generated/favicon/browserconfig.xml?v=3\">\u003cmeta data-mint-head name=\"apple-mobile-web-app-title\" content=\"DuckBrain\">\u003cmeta data-mint-head name=\"msapplication-TileColor\" content=\"#F5A623\">\u003clink data-mint-head rel=\"canonical\" href=\"https://docs.duckbrain.sh/commands/eval\">\u003clink data-mint-head rel=\"alternate\" type=\"application/xml\" href=\"/sitemap.xml\">\u003clink data-mint-head rel=\"alternate\" type=\"text/markdown\" href=\"/commands/eval.md\">\u003cscript data-mint-head type=\"application/ld+json\">{\"@context\":\"https://schema.org\",\"@graph\":[{\"@type\":\"Organization\",\"@id\":\"https://docs.duckbrain.sh/#organization\",\"name\":\"DuckBrain\",\"url\":\"https://docs.duckbrain.sh\"},{\"@type\":\"WebSite\",\"@id\":\"https://docs.duckbrain.sh#website\",\"name\":\"DuckBrain\",\"url\":\"https://docs.duckbrain.sh\",\"publisher\":{\"@id\":\"https://docs.duckbrain.sh/#organization\"}},{\"@type\":\"WebPage\",\"@id\":\"https://docs.duckbrain.sh/commands/eval#webpage\",\"url\":\"https://docs.duckbrain.sh/commands/eval\",\"name\":\"eval\",\"description\":\"The thesis harness: golden questions vs baselines\",\"dateModified\":\"2026-07-16T15:50:30.173Z\",\"isPartOf\":{\"@id\":\"https://docs.duckbrain.sh#website\"},\"breadcrumb\":{\"@id\":\"https://docs.duckbrain.sh/commands/eval#breadcrumb\"}},{\"@type\":\"BreadcrumbList\",\"@id\":\"https://docs.duckbrain.sh/commands/eval#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"Commands\",\"item\":\"https://docs.duckbrain.sh/commands/init\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"eval\",\"item\":\"https://docs.duckbrain.sh/commands/eval\"}]},{\"@type\":[\"Article\",\"TechArticle\"],\"@id\":\"https://docs.duckbrain.sh/commands/eval#article\",\"headline\":\"eval\",\"name\":\"eval\",\"description\":\"The thesis harness: golden questions vs baselines\",\"url\":\"https://docs.duckbrain.sh/commands/eval\",\"mainEntityOfPage\":{\"@id\":\"https://docs.duckbrain.sh/commands/eval#webpage\"},\"dateModified\":\"2026-07-16T15:50:30.173Z\",\"publisher\":{\"@id\":\"https://docs.duckbrain.sh/#organization\"},\"isPartOf\":{\"@id\":\"https://docs.duckbrain.sh#website\"}}]}\u003c/script>\u003cscript data-mint-head type=\"application/ld+json\">{\"@context\":\"https://schema.org\",\"@type\":\"WebSite\",\"name\":\"DuckBrain\",\"publisher\":{\"@type\":\"Organization\",\"name\":\"DuckBrain\"},\"provider\":{\"@type\":\"Organization\",\"name\":\"Mintlify\",\"url\":\"https://mintlify.com\"}}\u003c/script>","agentPointerHref":"/commands/eval.md","siteName":"DuckBrain","nav":{"global":null,"groups":[{"group":"Get Started","pages":[{"title":"DuckBrain","href":"/index"},{"title":"Install","href":"/install"}]},{"group":"Commands","pages":[{"title":"init","href":"/commands/init"},{"title":"connect","href":"/commands/connect"},{"title":"search","href":"/commands/search"},{"title":"context","href":"/commands/context"},{"title":"sync","description":"The one freshness verb: refresh now, one source, or stay synced continuously","href":"/commands/sync"},{"title":"eval","href":"/commands/eval"},{"title":"doctor","description":"Health check + repair: daemon, locks, temp, FDA, versions, row counts","href":"/commands/doctor"}]},{"group":"Ingest Sources","pages":[{"title":"Sources Overview","href":"/sources/overview"},{"title":"Apps via Pipedream","href":"/sources/pipedream"},{"title":"Slack","href":"/sources/slack"},{"title":"Claude Code Sessions","href":"/sources/claude-sessions"},{"title":"OpenClaw","href":"/sources/openclaw"},{"title":"Codex","href":"/sources/codex"},{"title":"Markdown \u0026 Notes","href":"/sources/markdown"},{"title":"iMessage","href":"/sources/imessage"},{"title":"WhatsApp","href":"/sources/whatsapp"},{"title":"Telegram","href":"/sources/telegram"},{"title":"Discord","href":"/sources/discord"},{"title":"Notion","href":"/sources/notion"},{"title":"Granola","href":"/sources/granola"},{"title":"Apple Photos","href":"/sources/apple-photos"},{"title":"GitHub issues/PRs (gitcrawl)","href":"/sources/git"},{"title":"Contacts","href":"/sources/contacts"},{"title":"Gmail \u0026 Calendar","href":"/sources/gmail"},{"title":"Google Drive","href":"/sources/google-drive"},{"title":"GitHub","href":"/sources/github"},{"title":"Fireflies","href":"/sources/fireflies"},{"title":"RevenueCat","href":"/sources/revenuecat"}]},{"group":"Guide","pages":[{"title":"Search \u0026 Facets","href":"/search"},{"title":"Instances","href":"/instances"},{"title":"For Agents","href":"/agents"}]}]},"basePath":"","title":"eval - DuckBrain","buildId":"dpl_596ePT575z4qnAFksq1biRW97aLY","deploymentId":"dpl_596ePT575z4qnAFksq1biRW97aLY","dynamics":[],"chunks":[]}