From 8edff8988a8bf5683c0e46d3135afedc465c7e7b Mon Sep 17 00:00:00 2001 From: Benjamin Toby Date: Thu, 22 Oct 2026 04:56:10 +0100 Subject: [PATCH] Run models on sensible generation defaults out of the box A fresh install had no generation settings until someone set one, so every model ran on whatever its provider happened to use. Each harness now has a default (text 0.3 temperature, vision 0.2, image the provider's own), shown in the model settings modal so what a model runs on is never a guess. The defaults are only sent to a model whose catalog lists the field, so a provider that says nothing about its models still gets no parameter it might reject; anything saved on a harness or bot wins over the default. --- src/dict/harness-parameter-dict.test.ts | 7 ++ src/dict/harness-parameter-dict.ts | 70 +++++++++++++++++++ src/harness/text/grab-chat-params.ts | 24 +++++-- .../(partials)/harness-parameter-fields.tsx | 22 +++++- .../(partials)/harness-parameter-modal.tsx | 39 +++++++++-- .../(partials)/harness-setting-group.tsx | 1 + 6 files changed, 150 insertions(+), 13 deletions(-) diff --git a/src/dict/harness-parameter-dict.test.ts b/src/dict/harness-parameter-dict.test.ts index 0bd5913..6c274b4 100644 --- a/src/dict/harness-parameter-dict.test.ts +++ b/src/dict/harness-parameter-dict.test.ts @@ -1,5 +1,6 @@ import { describe, expect, test } from "bun:test"; import { + harnessParameterDefault, HarnessParameters, harnessParametersFromOptions, modelTakesParameter, @@ -40,6 +41,12 @@ describe("harness parameters", () => { ).toBe(false); }); + test("ships a literal text default and a lookup vision default", () => { + expect(harnessParameterDefault({ harness: "text", key: "temperature" })).toBe(0.3); + expect(harnessParameterDefault({ harness: "vision", key: "temperature" })).toBe(0); + expect(harnessParameterDefault({ harness: "text", key: "seed" })).toBeUndefined(); + }); + test("a model that lists no parameters takes them all", () => { expect( modelTakesParameter({ parameter: temperature, supported: [] }), diff --git a/src/dict/harness-parameter-dict.ts b/src/dict/harness-parameter-dict.ts index 46f079b..299e991 100644 --- a/src/dict/harness-parameter-dict.ts +++ b/src/dict/harness-parameter-dict.ts @@ -118,6 +118,76 @@ export const HarnessParameterHarnesses = ["text", "vision"] as const; export const HarnessParameterKeys = HarnessParameters.map((parameter) => parameter.key); +/** + * What each chat harness uses when nobody set anything, so a fresh install + * is ready to go: bots are literal and repeatable (`Cobble`'s job is doing + * the work it was asked for, not improvising), while media reading is a + * lookup, so it stays at 0. + */ +export const HarnessParameterDefaults: Partial< + Record< + (typeof HarnessParameterHarnesses)[number], + Partial> + > +> = { + text: { temperature: 0.3 }, + vision: { temperature: 0 }, +}; + +/** + * One setting's value when the harness's own has none. + */ +export function harnessParameterDefault({ + harness, + key, +}: { + harness: (typeof HarnessParameterHarnesses)[number]; + key: HarnessParameterKey; +}) { + return HarnessParameterDefaults[harness]?.[key]; +} + +/** + * A setting's value for a harness: what it was saved as, else the app's + * default. `saved` says which, since a default is only used on a model + * that says it takes the field (a saved one is sent to any model whose + * catalog doesn't rule it out). + */ +export function harnessParameterValue({ + harness, + params, + key, +}: { + harness: (typeof HarnessParameterHarnesses)[number]; + params?: Record; + key: HarnessParameterKey; +}): { value?: HarnessParameterValue; saved: boolean } { + const saved = params?.[key]; + + if (typeof saved == "number" || (typeof saved == "string" && saved)) { + return { value: saved, saved: true }; + } + + return { value: harnessParameterDefault({ harness, key }), saved: false }; +} + +/** + * Whether a model's provider says it takes a setting (no guessing). + */ +export function modelAdvertisesParameter({ + parameter, + supported, +}: { + parameter: HarnessParameter; + supported?: string[]; +}) { + return Boolean( + supported?.includes( + "advertised" in parameter ? parameter.advertised : parameter.key, + ), + ); +} + /** * The generation settings a saved `options` object holds. */ diff --git a/src/harness/text/grab-chat-params.ts b/src/harness/text/grab-chat-params.ts index 46a3412..092d3d6 100644 --- a/src/harness/text/grab-chat-params.ts +++ b/src/harness/text/grab-chat-params.ts @@ -1,5 +1,7 @@ import { HarnessParameters, + harnessParameterValue, + modelAdvertisesParameter, modelTakesParameter, } from "@/src/dict/harness-parameter-dict"; import grabHarnessSetting from "@/src/functions/backend/db/harness/grab-harness-setting"; @@ -13,10 +15,10 @@ type Params = { /** * A model's generation settings for one call: what is saved in the - * harness's setting (the workspace's, the bot's, else the default), - * keyed by the field sent on the wire, keeping only what the model says it - * takes. Empty when none are set, so the model gets its provider's - * defaults. + * harness's setting (the workspace's, the bot's, else the default), else + * `HarnessParameterDefaults` so a fresh install is ready to go. Keyed by + * the field sent on the wire, keeping only what the model says it takes + * (a default only where the model explicitly advertises it). */ export default async function grabChatParams({ harness, @@ -34,7 +36,11 @@ export default async function grabChatParams({ return (model) => Object.fromEntries( HarnessParameters.flatMap((parameter) => { - const value = setting.options[parameter.key]; + const { value, saved } = harnessParameterValue({ + harness, + params: setting.options, + key: parameter.key, + }); if (typeof value != "number" && typeof value != "string") return []; if (!value && value !== 0) return []; @@ -45,6 +51,14 @@ export default async function grabChatParams({ }) ) return []; + if ( + !saved && + !modelAdvertisesParameter({ + parameter, + supported: model.supported_parameters, + }) + ) + return []; const key = "wire" in parameter ? parameter.wire : parameter.key; diff --git a/src/pages/admin/settings/models/(partials)/harness-parameter-fields.tsx b/src/pages/admin/settings/models/(partials)/harness-parameter-fields.tsx index a35e1fb..4ccc4b4 100644 --- a/src/pages/admin/settings/models/(partials)/harness-parameter-fields.tsx +++ b/src/pages/admin/settings/models/(partials)/harness-parameter-fields.tsx @@ -1,12 +1,20 @@ import { + HarnessParameterHarnesses, + harnessParameterValue, HarnessParameters, modelTakesParameter, + type HarnessParameterKey, } from "@/src/dict/harness-parameter-dict"; +import type { HarnessKind } from "@/src/dict/harness-dict"; import type { HarnessModelOption } from "@/src/types"; import HarnessParameterControl from "./harness-parameter-control"; type Props = { id: string; + /** + * The harness the settings are for: its default fills an empty one. + */ + harness: HarnessKind; /** * The model whose settings these are: only what it takes is shown. */ @@ -18,10 +26,14 @@ type Props = { /** * The generation settings the picked model takes: temperature, sampling * limits, penalties, seed and reasoning effort. A model whose provider - * lists no parameters (anything but OpenRouter) gets them all. + * lists no parameters (anything but OpenRouter) gets them all. An empty + * one shows the value it actually runs on (the harness's default). A model + * whose provider lists nothing gets the app's default in its box, but the + * call only sends a default to a model that advertises the field. */ export default function HarnessParameterFields({ id, + harness, model, params, onChange, @@ -42,7 +54,13 @@ export default function HarnessParameterFields({ key={parameter.key} id={`${id}-${parameter.key}`} parameter={parameter} - value={params?.[parameter.key]} + value={ + harnessParameterValue({ + harness: harness as (typeof HarnessParameterHarnesses)[number], + params, + key: parameter.key, + }).value + } onChange={(value) => { const next = { ...params }; diff --git a/src/pages/admin/settings/models/(partials)/harness-parameter-modal.tsx b/src/pages/admin/settings/models/(partials)/harness-parameter-modal.tsx index 0629dd5..429fa92 100644 --- a/src/pages/admin/settings/models/(partials)/harness-parameter-modal.tsx +++ b/src/pages/admin/settings/models/(partials)/harness-parameter-modal.tsx @@ -3,12 +3,23 @@ import Modal from "@moduletrace/twui/elements/Modal"; import H2 from "@moduletrace/twui/layout/H2"; import Span from "@moduletrace/twui/layout/Span"; import Stack from "@moduletrace/twui/layout/Stack"; -import { HarnessParameters } from "@/src/dict/harness-parameter-dict"; +import { + HarnessParameterHarnesses, + harnessParameterValue, + HarnessParameterKeys, + HarnessParameters, + type HarnessParameterKey, +} from "@/src/dict/harness-parameter-dict"; +import type { HarnessKind } from "@/src/dict/harness-dict"; import type { HarnessModelOption } from "@/src/types"; import HarnessParameterFields from "./harness-parameter-fields"; type Props = { id: string; + /** + * The harness the settings are for: its default fills an empty one. + */ + harness: HarnessKind; /** * The model the settings are for: only what it takes is offered. */ @@ -24,13 +35,27 @@ type Props = { */ export default function HarnessParameterModal({ id, + harness, model, params, onChange, }: Props) { const [open, setOpen] = useState(false); + /** + * Nothing saved yet: every value shown is the app's default for it. + */ + const using_defaults = !Object.entries(params || {}).some( + ([key, value]) => + value !== undefined && + value !== "" && + HarnessParameterKeys.includes(key as HarnessParameterKey), + ); const set = HarnessParameters.flatMap((parameter) => { - const value = params?.[parameter.key]; + const { value } = harnessParameterValue({ + harness: harness as (typeof HarnessParameterHarnesses)[number], + params, + key: parameter.key, + }); if (value === undefined || value === "") return []; @@ -52,7 +77,7 @@ export default function HarnessParameterModal({ {set.length - ? set.join(" · ") + ? `${using_defaults ? "Default: " : ""}${set.join(" · ")}` : "Temperature, reasoning effort, sampling limits"} @@ -70,14 +95,16 @@ export default function HarnessParameterModal({ {model.name} settings - How this model answers. Empty leaves the provider's own - default; a lower temperature keeps a bot literal and - repeatable. + How this model answers. The filled-in values are what it + runs on now (the app's default for this harness, else the + provider's own); a lower temperature keeps a bot literal + and repeatable. onChange({ ...value, params })}