evaluator

No long description provided.

Installation

dagger install github.com/dagger/dagger/modules/evaluator@89166d06f037ebdf273caaba4ca2976f86d4e9c7

Entrypoint

Return Type

Evaluator !

Arguments

Name	Type	Default Value	Description
docs	File	-	The documentation for the tool calling scheme to generate a prompt for.
initialPrompt	File	-	An initial system prompt to evaluate and use as a starting point.
model	String	-	Model to use for the evaluator agent.

Example

dagger -m github.com/dagger/dagger/modules/evaluator@89166d06f037ebdf273caaba4ca2976f86d4e9c7 call \

func (m *MyModule) Example() *dagger.Evaluator  {
	return dag.
			Evaluator()
}

@function
def example() -> dagger.Evaluator:
	return (
		dag.evaluator()
	)

@func()
example(): Evaluator {
	return dag
		.evaluator()
}

Types

Evaluator 🔗

docs() 🔗

Return Type

File !

Example

dagger -m github.com/dagger/dagger/modules/evaluator@89166d06f037ebdf273caaba4ca2976f86d4e9c7 call \
 docs

func (m *MyModule) Example() *dagger.File  {
	return dag.
			Evaluator().
			Docs()
}

@function
def example() -> dagger.File:
	return (
		dag.evaluator()
		.docs()
	)

@func()
example(): File {
	return dag
		.evaluator()
		.docs()
}

initialPrompt() 🔗

Return Type

File !

Example

dagger -m github.com/dagger/dagger/modules/evaluator@89166d06f037ebdf273caaba4ca2976f86d4e9c7 call \
 initial-prompt

func (m *MyModule) Example() *dagger.File  {
	return dag.
			Evaluator().
			InitialPrompt()
}

@function
def example() -> dagger.File:
	return (
		dag.evaluator()
		.initial_prompt()
	)

@func()
example(): File {
	return dag
		.evaluator()
		.initialPrompt()
}

evaluatorModel() 🔗

Return Type

String !

Example

dagger -m github.com/dagger/dagger/modules/evaluator@89166d06f037ebdf273caaba4ca2976f86d4e9c7 call \
 evaluator-model

func (m *MyModule) Example(ctx context.Context) string  {
	return dag.
			Evaluator().
			EvaluatorModel(ctx)
}

@function
async def example() -> str:
	return await (
		dag.evaluator()
		.evaluator_model()
	)

@func()
async example(): Promise<string> {
	return dag
		.evaluator()
		.evaluatorModel()
}

withEval() 🔗

Return Type

Evaluator !

Arguments

Name	Type	Default Value	Description
eval	Interface !	-	No description provided

Example

dagger -m github.com/dagger/dagger/modules/evaluator@89166d06f037ebdf273caaba4ca2976f86d4e9c7 call \
 with-eval

func (m *MyModule) Example(eval ) *dagger.Evaluator  {
	return dag.
			Evaluator().
			WithEval(eval)
}

@function
def example(eval: ) -> dagger.Evaluator:
	return (
		dag.evaluator()
		.with_eval(eval)
	)

@func()
example(eval: ): Evaluator {
	return dag
		.evaluator()
		.withEval(eval)
}

withEvals() 🔗

Return Type

Evaluator !

Arguments

Name	Type	Default Value	Description
evals	[Interface ! ] !	-	No description provided

Example

dagger -m github.com/dagger/dagger/modules/evaluator@89166d06f037ebdf273caaba4ca2976f86d4e9c7 call \
 with-evals

func (m *MyModule) Example(evals []) *dagger.Evaluator  {
	return dag.
			Evaluator().
			WithEvals(evals)
}

@function
def example(evals: List[]) -> dagger.Evaluator:
	return (
		dag.evaluator()
		.with_evals(evals)
	)

@func()
example(evals: []): Evaluator {
	return dag
		.evaluator()
		.withEvals(evals)
}

evalsAcrossModels() 🔗

Run evals across models.

Models run in parallel, and evals run in series, with all attempts in parallel.

Return Type

EvalsAcrossModels !

Arguments

Name	Type	Default Value	Description
evals	[String ! ]	-	Evals to run. Defaults to all.
models	[String ! ]	-	Models to run evals across. Defaults to all.
attempts	Integer	-	Attempts to run each eval. Defaults to a per-provider value.
systemPrompt	File	-	A system prompt to use.

Example

dagger -m github.com/dagger/dagger/modules/evaluator@89166d06f037ebdf273caaba4ca2976f86d4e9c7 call \
 evals-across-models

func (m *MyModule) Example() *dagger.EvaluatorEvalsAcrossModels  {
	return dag.
			Evaluator().
			EvalsAcrossModels()
}

@function
def example() -> dagger.EvaluatorEvalsAcrossModels:
	return (
		dag.evaluator()
		.evals_across_models()
	)

@func()
example(): EvaluatorEvalsAcrossModels {
	return dag
		.evaluator()
		.evalsAcrossModels()
}

systemPrompt() 🔗

Return Type

String !

Arguments

Name	Type	Default Value	Description
evalName	String	-	Run a particular eval, instead of leaving it open-ended.

Example

dagger -m github.com/dagger/dagger/modules/evaluator@89166d06f037ebdf273caaba4ca2976f86d4e9c7 call \
 system-prompt

func (m *MyModule) Example(ctx context.Context) string  {
	return dag.
			Evaluator().
			SystemPrompt(ctx)
}

@function
async def example() -> str:
	return await (
		dag.evaluator()
		.system_prompt()
	)

@func()
async example(): Promise<string> {
	return dag
		.evaluator()
		.systemPrompt()
}

explore() 🔗

Return Type

[String ! ] !

Example

dagger -m github.com/dagger/dagger/modules/evaluator@89166d06f037ebdf273caaba4ca2976f86d4e9c7 call \
 explore

func (m *MyModule) Example(ctx context.Context) []string  {
	return dag.
			Evaluator().
			Explore(ctx)
}

@function
async def example() -> List[str]:
	return await (
		dag.evaluator()
		.explore()
	)

@func()
async example(): Promise<string[]> {
	return dag
		.evaluator()
		.explore()
}

generateSystemPrompt() 🔗

Return Type

String !

Example

dagger -m github.com/dagger/dagger/modules/evaluator@89166d06f037ebdf273caaba4ca2976f86d4e9c7 call \
 generate-system-prompt

func (m *MyModule) Example(ctx context.Context) string  {
	return dag.
			Evaluator().
			GenerateSystemPrompt(ctx)
}

@function
async def example() -> str:
	return await (
		dag.evaluator()
		.generate_system_prompt()
	)

@func()
async example(): Promise<string> {
	return dag
		.evaluator()
		.generateSystemPrompt()
}

evaluate() 🔗

Return Type

String !

Arguments

Name	Type	Default Value	Description
model	String !	-	No description provided
name	String !	-	No description provided
attempts	Integer !	0	No description provided

Example

dagger -m github.com/dagger/dagger/modules/evaluator@89166d06f037ebdf273caaba4ca2976f86d4e9c7 call \
 evaluate --model string --name string --attempts integer

func (m *MyModule) Example(ctx context.Context, model string, name string, attempts int) string  {
	return dag.
			Evaluator().
			Evaluate(ctx, model, name, attempts)
}

@function
async def example(model: str, name: str, attempts: int) -> str:
	return await (
		dag.evaluator()
		.evaluate(model, name, attempts)
	)

@func()
async example(model: string, name: string, attempts: number): Promise<string> {
	return dag
		.evaluator()
		.evaluate(model, name, attempts)
}

iterate() 🔗

Iterate continuously runs evals across all models.

Return Type

String !

Example

dagger -m github.com/dagger/dagger/modules/evaluator@89166d06f037ebdf273caaba4ca2976f86d4e9c7 call \
 iterate

func (m *MyModule) Example(ctx context.Context) string  {
	return dag.
			Evaluator().
			Iterate(ctx)
}

@function
async def example() -> str:
	return await (
		dag.evaluator()
		.iterate()
	)

@func()
async example(): Promise<string> {
	return dag
		.evaluator()
		.iterate()
}

compare() 🔗

Return Type

String !

Arguments

Name	Type	Default Value	Description
before	File !	-	No description provided
after	File !	-	No description provided

Example

dagger -m github.com/dagger/dagger/modules/evaluator@89166d06f037ebdf273caaba4ca2976f86d4e9c7 call \
 compare --before file:path --after file:path

func (m *MyModule) Example(ctx context.Context, before *dagger.File, after *dagger.File) string  {
	return dag.
			Evaluator().
			Compare(ctx, before, after)
}

@function
async def example(before: dagger.File, after: dagger.File) -> str:
	return await (
		dag.evaluator()
		.compare(before, after)
	)

@func()
async example(before: File, after: File): Promise<string> {
	return dag
		.evaluator()
		.compare(before, after)
}

EvalsAcrossModels 🔗

traceId() 🔗

Return Type

String !

Example

dagger -m github.com/dagger/dagger/modules/evaluator@89166d06f037ebdf273caaba4ca2976f86d4e9c7 call \
 evals-across-models \
 trace-id

func (m *MyModule) Example(ctx context.Context) string  {
	return dag.
			Evaluator().
			EvalsAcrossModels().
			TraceId(ctx)
}

@function
async def example() -> str:
	return await (
		dag.evaluator()
		.evals_across_models()
		.trace_id()
	)

@func()
async example(): Promise<string> {
	return dag
		.evaluator()
		.evalsAcrossModels()
		.traceId()
}

modelResults() 🔗

Return Type

[ModelResult ! ] !

Example

dagger -m github.com/dagger/dagger/modules/evaluator@89166d06f037ebdf273caaba4ca2976f86d4e9c7 call \
 evals-across-models \
 model-results

func (m *MyModule) Example() []*dagger.EvaluatorModelResult  {
	return dag.
			Evaluator().
			EvalsAcrossModels().
			ModelResults()
}

@function
def example() -> List[dagger.EvaluatorModelResult]:
	return (
		dag.evaluator()
		.evals_across_models()
		.model_results()
	)

@func()
example(): EvaluatorModelResult[] {
	return dag
		.evaluator()
		.evalsAcrossModels()
		.modelResults()
}

check() 🔗

Return Type

Void !

Example

dagger -m github.com/dagger/dagger/modules/evaluator@89166d06f037ebdf273caaba4ca2976f86d4e9c7 call \
 evals-across-models \
 check

func (m *MyModule) Example(ctx context.Context)   {
	return dag.
			Evaluator().
			EvalsAcrossModels().
			Check(ctx)
}

@function
async def example() -> None:
	return await (
		dag.evaluator()
		.evals_across_models()
		.check()
	)

@func()
async example(): Promise<void> {
	return dag
		.evaluator()
		.evalsAcrossModels()
		.check()
}

csv() 🔗

Return Type

String !

Arguments

Name	Type	Default Value	Description
noHeader	Boolean !	false	Don't include a header.

Example

dagger -m github.com/dagger/dagger/modules/evaluator@89166d06f037ebdf273caaba4ca2976f86d4e9c7 call \
 evals-across-models \
 csv --no-header boolean

func (m *MyModule) Example(ctx context.Context, noHeader bool) string  {
	return dag.
			Evaluator().
			EvalsAcrossModels().
			Csv(ctx, noHeader)
}

@function
async def example(no_header: bool) -> str:
	return await (
		dag.evaluator()
		.evals_across_models()
		.csv(no_header)
	)

@func()
async example(noHeader: boolean): Promise<string> {
	return dag
		.evaluator()
		.evalsAcrossModels()
		.csv(noHeader)
}

ModelResult 🔗

modelName() 🔗

Return Type

String !

Example

Function EvaluatorModelResult.modelName is not accessible from the evaluator module

Function EvaluatorModelResult.modelName is not accessible from the evaluator module

Function EvaluatorModelResult.modelName is not accessible from the evaluator module

Function EvaluatorModelResult.modelName is not accessible from the evaluator module

spanId() 🔗

Return Type

String !

Example

Function EvaluatorModelResult.spanId is not accessible from the evaluator module

Function EvaluatorModelResult.spanId is not accessible from the evaluator module

Function EvaluatorModelResult.spanId is not accessible from the evaluator module

Function EvaluatorModelResult.spanId is not accessible from the evaluator module

evalReports() 🔗

Return Type

[EvalResult ! ] !

Example

Function EvaluatorModelResult.evalReports is not accessible from the evaluator module

Function EvaluatorModelResult.evalReports is not accessible from the evaluator module

Function EvaluatorModelResult.evalReports is not accessible from the evaluator module

Function EvaluatorModelResult.evalReports is not accessible from the evaluator module

check() 🔗

Return Type

Void !

Example

Function EvaluatorModelResult.check is not accessible from the evaluator module

Function EvaluatorModelResult.check is not accessible from the evaluator module

Function EvaluatorModelResult.check is not accessible from the evaluator module

Function EvaluatorModelResult.check is not accessible from the evaluator module

EvalResult 🔗

name() 🔗

Return Type

String !

Example

Function EvaluatorEvalResult.name is not accessible from the evaluator module

Function EvaluatorEvalResult.name is not accessible from the evaluator module

Function EvaluatorEvalResult.name is not accessible from the evaluator module

Function EvaluatorEvalResult.name is not accessible from the evaluator module

spanId() 🔗

Return Type

String !

Example

Function EvaluatorEvalResult.spanId is not accessible from the evaluator module

Function EvaluatorEvalResult.spanId is not accessible from the evaluator module

Function EvaluatorEvalResult.spanId is not accessible from the evaluator module

Function EvaluatorEvalResult.spanId is not accessible from the evaluator module

error() 🔗

Return Type

String !

Example

Function EvaluatorEvalResult.error is not accessible from the evaluator module

Function EvaluatorEvalResult.error is not accessible from the evaluator module

Function EvaluatorEvalResult.error is not accessible from the evaluator module

Function EvaluatorEvalResult.error is not accessible from the evaluator module

report() 🔗

Return Type

String !

Example

Function EvaluatorEvalResult.report is not accessible from the evaluator module

Function EvaluatorEvalResult.report is not accessible from the evaluator module

Function EvaluatorEvalResult.report is not accessible from the evaluator module

Function EvaluatorEvalResult.report is not accessible from the evaluator module

successRate() 🔗

Return Type

Float !

Example

Function EvaluatorEvalResult.successRate is not accessible from the evaluator module

Function EvaluatorEvalResult.successRate is not accessible from the evaluator module

Function EvaluatorEvalResult.successRate is not accessible from the evaluator module

Function EvaluatorEvalResult.successRate is not accessible from the evaluator module

totalAttempts() 🔗

Return Type

Integer !

Example

Function EvaluatorEvalResult.totalAttempts is not accessible from the evaluator module

Function EvaluatorEvalResult.totalAttempts is not accessible from the evaluator module

Function EvaluatorEvalResult.totalAttempts is not accessible from the evaluator module

Function EvaluatorEvalResult.totalAttempts is not accessible from the evaluator module

inputTokens() 🔗

Return Type

Integer !

Example

Function EvaluatorEvalResult.inputTokens is not accessible from the evaluator module

Function EvaluatorEvalResult.inputTokens is not accessible from the evaluator module

Function EvaluatorEvalResult.inputTokens is not accessible from the evaluator module

Function EvaluatorEvalResult.inputTokens is not accessible from the evaluator module

outputTokens() 🔗

Return Type

Integer !

Example

Function EvaluatorEvalResult.outputTokens is not accessible from the evaluator module

Function EvaluatorEvalResult.outputTokens is not accessible from the evaluator module

Function EvaluatorEvalResult.outputTokens is not accessible from the evaluator module

Function EvaluatorEvalResult.outputTokens is not accessible from the evaluator module

check() 🔗

Return Type

Void !

Example

Function EvaluatorEvalResult.check is not accessible from the evaluator module

Function EvaluatorEvalResult.check is not accessible from the evaluator module

Function EvaluatorEvalResult.check is not accessible from the evaluator module

Function EvaluatorEvalResult.check is not accessible from the evaluator module