Add per-model generation settings to the model selector
Each text and vision harness setting now carries the generation settings its model takes (temperature, sampling limits, penalties, seed, reasoning effort), edited in a modal under the model pick so the harness block stays short. Nothing is set by default, so models keep their provider's own values until a super admin or admin changes them. A setting is only offered and only sent when the catalog says the model takes it: OpenRouter's supported_parameters is now stored on harness_models and refreshed by the hourly sync. Other providers list no parameters, so their models get them all.
This commit is contained in:
@@ -0,0 +1,51 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import {
|
||||
HarnessParameters,
|
||||
harnessParametersFromOptions,
|
||||
modelTakesParameter,
|
||||
withHarnessParameters,
|
||||
} from "./harness-parameter-dict";
|
||||
|
||||
const temperature = HarnessParameters.find((p) => p.key == "temperature")!;
|
||||
const reasoning = HarnessParameters.find((p) => p.key == "reasoning_effort")!;
|
||||
|
||||
describe("harness parameters", () => {
|
||||
test("reads only the known settings out of options", () => {
|
||||
expect(
|
||||
harnessParametersFromOptions({
|
||||
options: { voice: "alloy", temperature: 0.2, seed: 0, junk: "x" },
|
||||
}),
|
||||
).toEqual({ temperature: 0.2, seed: 0 });
|
||||
});
|
||||
|
||||
test("replaces the settings and keeps voice", () => {
|
||||
expect(
|
||||
withHarnessParameters({
|
||||
options: { voice: "alloy", temperature: 1, top_p: 0.5 },
|
||||
params: { temperature: 0.2 },
|
||||
}),
|
||||
).toEqual({ voice: "alloy", temperature: 0.2 });
|
||||
});
|
||||
|
||||
test("only offers what the model advertises", () => {
|
||||
const supported = ["tools", "temperature", "reasoning"];
|
||||
|
||||
expect(modelTakesParameter({ parameter: temperature, supported })).toBe(true);
|
||||
expect(modelTakesParameter({ parameter: reasoning, supported })).toBe(true);
|
||||
expect(
|
||||
modelTakesParameter({
|
||||
parameter: temperature,
|
||||
supported: ["tools", "top_p"],
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
test("a model that lists no parameters takes them all", () => {
|
||||
expect(
|
||||
modelTakesParameter({ parameter: temperature, supported: [] }),
|
||||
).toBe(true);
|
||||
expect(
|
||||
modelTakesParameter({ parameter: temperature, supported: undefined }),
|
||||
).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,174 @@
|
||||
/**
|
||||
* Generation settings a harness can ask its model for. Each is saved in the
|
||||
* setting's `options` (next to `voice`) and only sent to a model whose
|
||||
* catalog entry advertises it: a fallback model that rejects a field would
|
||||
* fail otherwise.
|
||||
*
|
||||
* - `key`: the option's key in `options`, and the field sent on the wire
|
||||
* - `advertised`: the name a provider's model list uses for it, when it
|
||||
* differs (OpenRouter's `supported_parameters`)
|
||||
* - `wire`: how the field is written, for providers that nest it
|
||||
*/
|
||||
export const HarnessParameters = [
|
||||
{
|
||||
key: "temperature",
|
||||
title: "Temperature",
|
||||
description: `Lower keeps a bot literal and repeatable, higher makes it more varied. Most models default to 1, which makes small bots invent details.`,
|
||||
value_type: "number",
|
||||
min: 0,
|
||||
max: 2,
|
||||
step: 0.05,
|
||||
},
|
||||
{
|
||||
key: "top_p",
|
||||
title: "Top P",
|
||||
description: `Nucleus sampling: how much of the model's probability mass it may pick from. Change this or temperature, rarely both.`,
|
||||
value_type: "number",
|
||||
min: 0,
|
||||
max: 1,
|
||||
step: 0.05,
|
||||
},
|
||||
{
|
||||
key: "top_k",
|
||||
title: "Top K",
|
||||
description: `How many of the likeliest tokens it may pick from. 0 turns it off. Not every provider takes it.`,
|
||||
value_type: "number",
|
||||
min: 0,
|
||||
max: 200,
|
||||
step: 1,
|
||||
},
|
||||
{
|
||||
key: "min_p",
|
||||
title: "Min P",
|
||||
description: `Drops tokens less likely than this share of the best token. Alternative to top_p.`,
|
||||
value_type: "number",
|
||||
min: 0,
|
||||
max: 1,
|
||||
step: 0.01,
|
||||
},
|
||||
{
|
||||
key: "top_a",
|
||||
title: "Top A",
|
||||
description: `Drops tokens the model is only weakly considering. 0 turns it off.`,
|
||||
value_type: "number",
|
||||
min: 0,
|
||||
max: 1,
|
||||
step: 0.01,
|
||||
},
|
||||
{
|
||||
key: "frequency_penalty",
|
||||
title: "Frequency penalty",
|
||||
description: `Positive values make it repeat words less. Negative values make it repeat more.`,
|
||||
value_type: "number",
|
||||
min: -2,
|
||||
max: 2,
|
||||
step: 0.1,
|
||||
},
|
||||
{
|
||||
key: "presence_penalty",
|
||||
title: "Presence penalty",
|
||||
description: `Positive values push it toward new topics, negative values back to what was already said.`,
|
||||
value_type: "number",
|
||||
min: -2,
|
||||
max: 2,
|
||||
step: 0.1,
|
||||
},
|
||||
{
|
||||
key: "repetition_penalty",
|
||||
title: "Repetition penalty",
|
||||
description: `1 leaves repetition alone, higher penalizes it. Some providers take this instead of the two penalties above.`,
|
||||
value_type: "number",
|
||||
min: 0,
|
||||
max: 2,
|
||||
step: 0.05,
|
||||
},
|
||||
{
|
||||
key: "seed",
|
||||
title: "Seed",
|
||||
description: `Asks for repeatable answers. Providers only do their best: the same seed usually gives the same answer, not always.`,
|
||||
value_type: "number",
|
||||
min: 0,
|
||||
step: 1,
|
||||
},
|
||||
{
|
||||
key: "reasoning_effort",
|
||||
title: "Reasoning effort",
|
||||
description: `How long a reasoning model thinks before it answers. "Default" leaves the provider's own setting alone. Models without reasoning ignore it.`,
|
||||
value_type: "select",
|
||||
advertised: "reasoning",
|
||||
wire: "reasoning",
|
||||
options: [
|
||||
{ value: "minimal", title: "Minimal" },
|
||||
{ value: "low", title: "Low" },
|
||||
{ value: "medium", title: "Medium" },
|
||||
{ value: "high", title: "High" },
|
||||
],
|
||||
},
|
||||
] as const;
|
||||
|
||||
export type HarnessParameter = (typeof HarnessParameters)[number];
|
||||
export type HarnessParameterKey = HarnessParameter["key"];
|
||||
export type HarnessParameterValue = number | string;
|
||||
|
||||
/**
|
||||
* The harnesses whose models are chat models, so the only ones the
|
||||
* generation settings apply to.
|
||||
*/
|
||||
export const HarnessParameterHarnesses = ["text", "vision"] as const;
|
||||
|
||||
export const HarnessParameterKeys = HarnessParameters.map((parameter) => parameter.key);
|
||||
|
||||
/**
|
||||
* The generation settings a saved `options` object holds.
|
||||
*/
|
||||
export function harnessParametersFromOptions({
|
||||
options,
|
||||
}: {
|
||||
options: Record<string, unknown>;
|
||||
}): Partial<Record<HarnessParameterKey, HarnessParameterValue>> {
|
||||
return Object.fromEntries(
|
||||
HarnessParameters.flatMap((parameter) => {
|
||||
const value = options[parameter.key];
|
||||
|
||||
return typeof value == "number" || (typeof value == "string" && value)
|
||||
? [[parameter.key, value]]
|
||||
: [];
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* A `options` object with the given generation settings replaced (and the
|
||||
* ones missing from it removed), keeping everything else (`voice`).
|
||||
*/
|
||||
export function withHarnessParameters({
|
||||
options,
|
||||
params,
|
||||
}: {
|
||||
options: Record<string, unknown>;
|
||||
params: Partial<Record<HarnessParameterKey, HarnessParameterValue>>;
|
||||
}): Record<string, unknown> {
|
||||
const rest: Record<string, unknown> = Object.fromEntries(
|
||||
Object.entries(options).filter(([key]) => !HarnessParameterKeys.includes(key as HarnessParameterKey)),
|
||||
);
|
||||
|
||||
return { ...rest, ...params };
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a model advertises a setting. A model whose provider lists no
|
||||
* parameters (anything but OpenRouter) is taken to accept them all.
|
||||
*/
|
||||
export function modelTakesParameter({
|
||||
parameter,
|
||||
supported,
|
||||
}: {
|
||||
parameter: HarnessParameter;
|
||||
supported?: string[];
|
||||
}) {
|
||||
if (!supported?.length) return true;
|
||||
|
||||
return supported.includes(
|
||||
"advertised" in parameter ? parameter.advertised : parameter.key,
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user