Repository navigation
Expand file tree
/
Copy pathContracts.cs
More file actions
316 lines (292 loc) · 15.5 KB
/
Copy pathContracts.cs
File metadata and controls
316 lines (292 loc) · 15.5 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
using Microsoft.Extensions.AI;
using Microsoft.Extensions.AI.Evaluation;
namespace ManagedCode.SkillOpt;
/// <summary>A stable task case in one of the explicit data splits.</summary>
public sealed record SkillOptCase
{
/// <summary>Unique caller-supplied identity within and across all splits.</summary>
public required string Id { get; init; }
/// <summary>Stable identity for the complete task and evaluation context, including non-text inputs.</summary>
public required string ContentFingerprint { get; init; }
/// <summary>Messages sent to the target model before the skill text is added.</summary>
public required IReadOnlyList<ChatMessage> Messages { get; init; }
/// <summary>Typed Microsoft.Extensions.AI evaluation context for this case.</summary>
public IReadOnlyList<EvaluationContext> EvaluationContexts { get; init; } = [];
}
/// <summary>Supported text-space update strategies.</summary>
public enum SkillUpdateMode
{
/// <summary>Apply selected bounded add, insert, replace, or delete edits directly.</summary>
Patch,
/// <summary>Ask the optimizer model to return a complete replacement skill.</summary>
FullRewrite,
/// <summary>Apply selected edits, then ask the optimizer model to rewrite the complete skill.</summary>
RewriteFromSuggestions
}
/// <summary>Controls the edit budget at each optimization step.</summary>
public enum SkillOptRunPhase
{
/// <summary>Establishing the initial selection score.</summary>
Initializing,
/// <summary>Executing bounded optimization steps.</summary>
Training,
/// <summary>Updating epoch-level meta memory.</summary>
EpochMetaUpdate,
/// <summary>Applying and validating the epoch-level slow update.</summary>
EpochSlowUpdate,
/// <summary>Producing optional final held-out reports.</summary>
FinalReporting,
/// <summary>All requested optimization and reporting work finished.</summary>
Completed
}
/// <summary>Schedules the per-step edit count across the run.</summary>
public enum SkillEditBudgetSchedule
{
/// <summary>Use the maximum edit count at every step.</summary>
Constant,
/// <summary>Decrease linearly from the maximum toward the minimum.</summary>
Linear,
/// <summary>Decrease on a cosine schedule from the maximum toward the minimum.</summary>
Cosine
}
/// <summary>Optimization direction for the selected numeric metric.</summary>
public enum SkillOptScoreDirection
{
/// <summary>Higher raw metric values are better.</summary>
HigherIsBetter,
/// <summary>Lower raw metric values are better.</summary>
LowerIsBetter
}
/// <summary>All explicit limits and selection settings for one run.</summary>
public sealed record SkillOptOptions
{
/// <summary>Number of training epochs.</summary>
public required int Epochs { get; init; }
/// <summary>Optimization steps per epoch.</summary>
public required int StepsPerEpoch { get; init; }
/// <summary>Maximum task cases sampled into a rollout batch.</summary>
public required int BatchSize { get; init; }
/// <summary>Maximum target-model rollouts, including initialization and held-out reports.</summary>
public required int MaxRollouts { get; init; }
/// <summary>Maximum optimizer-model calls.</summary>
public required int MaxOptimizerCalls { get; init; }
/// <summary>Maximum evaluator calls.</summary>
public required int MaxEvaluationCalls { get; init; }
/// <summary>Maximum selected edits applied in one step.</summary>
public required int MaxEditsPerStep { get; init; }
/// <summary>Maximum rejected candidate summaries retained in run memory.</summary>
public required int MaxRejectedEdits { get; init; }
/// <summary>Lower bound for scheduled edit counts.</summary>
public int MinimumEditsPerStep { get; init; } = 1;
/// <summary>Schedule used to calculate each step's edit limit.</summary>
public SkillEditBudgetSchedule EditBudgetSchedule { get; init; } = SkillEditBudgetSchedule.Constant;
/// <summary>Text-space candidate update strategy.</summary>
public SkillUpdateMode UpdateMode { get; init; } = SkillUpdateMode.Patch;
/// <summary>When true, reflection includes failure trajectories only.</summary>
public bool FailureOnly { get; init; }
/// <summary>Maximum number of candidate patches merged in one aggregation prompt.</summary>
public int MergeBatchSize { get; init; } = 8;
/// <summary>Seed for deterministic case-batch selection.</summary>
public int RandomSeed { get; init; }
/// <summary>Name of the numeric metric returned by the evaluator.</summary>
public required string ScoreMetricName { get; init; }
/// <summary>Whether larger or smaller raw values of the selected metric are preferred.</summary>
public SkillOptScoreDirection ScoreDirection { get; init; } = SkillOptScoreDirection.HigherIsBetter;
/// <summary>Inclusive minimum allowed metric value.</summary>
public double MetricMinimum { get; init; }
/// <summary>Inclusive maximum allowed metric value.</summary>
public double MetricMaximum { get; init; } = 1;
/// <summary>Minimum selection-score gain required to accept a candidate.</summary>
public double MinimumImprovement { get; init; }
internal void Validate()
{
if (Epochs <= 0 || StepsPerEpoch <= 0 || BatchSize <= 0 || MaxRollouts <= 0 ||
MaxOptimizerCalls <= 0 || MaxEvaluationCalls <= 0 || MaxEditsPerStep <= 0 || MaxRejectedEdits <= 0 ||
MinimumEditsPerStep <= 0 || MinimumEditsPerStep > MaxEditsPerStep || MergeBatchSize < 2)
{
throw new ArgumentOutOfRangeException(nameof(SkillOptOptions), "All run limits must be positive and internally consistent.");
}
ArgumentException.ThrowIfNullOrWhiteSpace(ScoreMetricName);
if (!double.IsFinite(MetricMinimum) || !double.IsFinite(MetricMaximum) || MetricMaximum <= MetricMinimum)
{
throw new ArgumentException("MetricMaximum must be finite and greater than MetricMinimum.");
}
if (!double.IsFinite(MinimumImprovement) || MinimumImprovement < 0)
{
throw new ArgumentOutOfRangeException(nameof(MinimumImprovement));
}
if (!Enum.IsDefined(UpdateMode) || !Enum.IsDefined(EditBudgetSchedule) || !Enum.IsDefined(ScoreDirection))
{
throw new ArgumentOutOfRangeException(nameof(SkillOptOptions), "UpdateMode, EditBudgetSchedule, and ScoreDirection must be supported values.");
}
if ((long)Epochs * StepsPerEpoch > int.MaxValue)
{
throw new ArgumentOutOfRangeException(nameof(SkillOptOptions), "Epochs multiplied by StepsPerEpoch must fit in a 32-bit step count.");
}
}
}
/// <summary>One exact, supported text edit proposed by the optimizer.</summary>
public sealed record SkillOptEdit
{
/// <summary>Text edit operation to apply.</summary>
public required SkillOptEditOperation Operation { get; init; }
/// <summary>Exact existing text anchor used by insert, replace, or delete.</summary>
public string Target { get; init; } = string.Empty;
/// <summary>New text appended, inserted, or substituted.</summary>
public string Content { get; init; } = string.Empty;
/// <summary>Number of supporting trajectories reported by reflection.</summary>
public int SupportCount { get; init; }
/// <summary>Outcome class from which this edit was reflected, if known.</summary>
public SkillOptOutcome? SourceOutcome { get; init; }
}
/// <summary>Bounded exact-text operations supported by the optimizer.</summary>
public enum SkillOptEditOperation
{
/// <summary>Append content to the end of the skill.</summary>
Append,
/// <summary>Insert content immediately after an exact anchor.</summary>
InsertAfter,
/// <summary>Replace an exact anchor with content.</summary>
Replace,
/// <summary>Delete an exact anchor.</summary>
Delete
}
/// <summary>Trajectory outcome category passed to reflection prompts.</summary>
public enum SkillOptOutcome
{
/// <summary>The target task did not meet the full-score threshold.</summary>
Failure,
/// <summary>The target task met the full-score threshold.</summary>
Success
}
/// <summary>Typed optimizer memory retained across optimization steps and resume.</summary>
public sealed record SkillOptRunState
{
/// <summary>Random seed used for deterministic batch selection.</summary>
public required int Seed { get; init; }
/// <summary>Number of fully completed training steps.</summary>
public required int CompletedSteps { get; init; }
/// <summary>Number of fully completed epochs.</summary>
public required int CompletedEpochs { get; init; }
/// <summary>Current resumable phase or uncertain-operation phase.</summary>
public required SkillOptRunPhase Phase { get; init; }
/// <summary>False means the last checkpoint reserves an operation whose outcome is uncertain; it cannot be resumed.</summary>
public required bool IsResumeSafe { get; init; }
/// <summary>Skill at the start of the active epoch.</summary>
public required string EpochStartSkill { get; init; }
/// <summary>Current accepted skill text.</summary>
public required string CurrentSkill { get; init; }
/// <summary>Best selection-scoring skill observed so far.</summary>
public required string BestSkill { get; init; }
/// <summary>Selection score of the current skill.</summary>
public required double CurrentScore { get; init; }
/// <summary>Best selection score observed so far.</summary>
public required double BestScore { get; init; }
/// <summary>Training step at which the best skill was accepted.</summary>
public required int BestStep { get; init; }
/// <summary>Epoch-level optimizer memory.</summary>
public required string MetaSkill { get; init; }
/// <summary>Bounded rejected-candidate memory.</summary>
public required IReadOnlyList<string> RejectedEdits { get; init; }
/// <summary>Completed step-level evidence.</summary>
public required IReadOnlyList<SkillOptStepRecord> History { get; init; }
/// <summary>Target-model rollouts already reserved or completed.</summary>
public required int RolloutsUsed { get; init; }
/// <summary>Optimizer-model calls already reserved or completed.</summary>
public required int OptimizerCallsUsed { get; init; }
/// <summary>Evaluator calls already reserved or completed.</summary>
public required int EvaluationCallsUsed { get; init; }
/// <summary>Canonical identity of run configuration, splits, and initial skill.</summary>
public required string DatasetFingerprint { get; init; }
}
/// <summary>Evidence recorded for one completed optimization step.</summary>
public sealed record SkillOptStepRecord(
int Step,
int Epoch,
int TrainCaseCount,
int Failures,
int Successes,
int EditBudget,
double BeforeScore,
double CandidateScore,
string Action,
int AppliedEdits,
string FailureSummary,
string OptimizerReasoning);
/// <summary>Measured score report for a data split; mean is null when no cases were supplied.</summary>
public sealed record SkillOptSplitReport(
string Split,
int CaseCount,
double? MeanScore,
IReadOnlyList<SkillOptCaseScore> Cases);
/// <summary>Measured score and evaluator reason for one case.</summary>
public sealed record SkillOptCaseScore(
string CaseId,
double Score,
string? MetricReason,
IReadOnlyList<SkillOptMetricEvidence> Metrics);
/// <summary>Typed, per-case projection of an official Microsoft evaluation metric.</summary>
public sealed record SkillOptMetricEvidence(
string Name,
string MetricType,
double? NumericValue,
bool? BooleanValue,
string? TextValue,
string? Reason,
string? Interpretation,
IReadOnlyList<SkillOptEvaluationDiagnostic> Diagnostics)
{
/// <summary>Whether the official evaluator interpreted this metric as a failed outcome; null means no interpretation was supplied.</summary>
public bool? InterpretationFailed { get; init; }
}
/// <summary>Candidate text plus all official evaluator evidence from its selection split.</summary>
public sealed record SkillOptCandidateEvaluation(string CandidateSkill, SkillOptSplitReport SelectionReport);
/// <summary>Diagnostic emitted by Microsoft.Extensions.AI.Evaluation for a metric.</summary>
public sealed record SkillOptEvaluationDiagnostic(EvaluationDiagnosticSeverity Severity, string Message);
/// <summary>Best and final skill artifacts, split evidence, and resumable state.</summary>
public sealed record SkillOptResult(
string BestSkill,
string CurrentSkill,
SkillOptSplitReport BaselineTest,
SkillOptSplitReport BestTest,
SkillOptRunState State);
/// <summary>One optimization run, including all three data splits and caller-owned AI clients.</summary>
public sealed record SkillOptRequest
{
/// <summary>Initial Markdown skill text to optimize.</summary>
public required string InitialSkill { get; init; }
/// <summary>Cases used for target rollouts and reflection.</summary>
public required IReadOnlyList<SkillOptCase> TrainingCases { get; init; }
/// <summary>Held-out cases used to accept or reject candidates.</summary>
public required IReadOnlyList<SkillOptCase> SelectionCases { get; init; }
/// <summary>Optional final reporting cases, never included in optimizer prompts.</summary>
public required IReadOnlyList<SkillOptCase> TestCases { get; init; }
/// <summary>Fixed target-model client used to execute each case.</summary>
public required IChatClient TargetChatClient { get; init; }
/// <summary>
/// Optional caller-owned message composition for target rollouts. Receives the original case messages and the
/// current candidate skill separately so hosts can keep frozen scenario context intact while supplying skill text.
/// Include the factory's stable behavior/version in <see cref="RunIdentity"/> when using this callback.
/// </summary>
public Func<IReadOnlyList<ChatMessage>, string, IReadOnlyList<ChatMessage>>? TargetMessageFactory { get; init; }
/// <summary>Optimizer client used for reflection and text updates.</summary>
public required IChatClient OptimizerChatClient { get; init; }
/// <summary>Microsoft.Extensions.AI evaluator used to measure model responses.</summary>
public required IEvaluator Evaluator { get; init; }
/// <summary>Chat configuration supplied to the evaluator.</summary>
public required ChatConfiguration EvaluationChatConfiguration { get; init; }
/// <summary>Explicit run limits and selection settings.</summary>
public required SkillOptOptions Options { get; init; }
/// <summary>Caller-defined stable model and evaluator profile identity, used to reject incompatible resumes.</summary>
public required string RunIdentity { get; init; }
/// <summary>Safe completed-step/epoch state to continue from.</summary>
public SkillOptRunState? ResumeState { get; init; }
/// <summary>Persists each reservation and safe boundary before the optimizer continues.</summary>
public Func<SkillOptRunState, CancellationToken, ValueTask>? Checkpoint { get; init; }
/// <summary>Optional application-owned hard gate over complete per-case selection evidence; false rejects regardless of mean score.</summary>
public Func<SkillOptCandidateEvaluation, bool>? CandidateGate { get; init; }
/// <summary>Optional target-model generation settings.</summary>
public ChatOptions? TargetChatOptions { get; init; }
/// <summary>Optional optimizer-model generation settings.</summary>
public ChatOptions? OptimizerChatOptions { get; init; }
}