#middleware

1 messages · Page 1 of 1 (latest)

brazen mortar
# 
func (s agentSchema) Install() {
    // extend type Query { withLlm(): Query! }
    dagql.Fields[*core.Query]{
        dagql.Func("withLlm", s.withLlm).
            Doc(`Enable LLM integration`).
            ArgDoc("model", "The model to use").
            ArgDoc("key", "The API key for the LLM endpoint"),
    }.Install(s.srv)
    // Install hook
    s.srv.OnInstallObject(func(selfType dagql.ObjectType, install func(dagql.ObjectType)) {
        class := dagql.NewClass[*core.Agent](dagql.ClassOpts[*core.Agent]{
            // Instantiate a throwaway agent instance from the type
            Typed: core.NewAgent(s.srv, s.llmConfig(), nil, selfType),
        })
        class.Install(
            dagql.Func("withPrompt", s.withPrompt).
                Doc("add a prompt to the agent context").
                ArgDoc("prompt", "The prompt. Example: \"make me a sandwich\""),
            dagql.Func("run", s.run).
                Doc("run the agent"),
            dagql.Func("history", s.history).
                Doc("return the agent history: user prompts, agent replies, and tool calls"),
            dagql.Func("as"+selfType.TypeName(), s.asObject).
                Doc("convert the agent back to a "+selfType.TypeName()),
        )
        install(class)
    })
}
# 
time="2025-01-28T05:10:02Z" level=info msg="http: panic serving 10.87.0.128:55746: runtime error: invalid memory address or nil pointer dereference"

goroutine 61 [running]:
net/http.(*conn).serve.func1()
    /usr/lib/go/src/net/http/server.go:1947 +0xbe

panic({0x223d1c0?, 0x3da6380?})
    /usr/lib/go/src/runtime/panic.go:785 +0x132

github.com/dagger/dagger/core.(*Agent).Type(0x0?)
    /app/core/agent.go:73 +0xe

github.com/dagger/dagger/dagql.NodeFuncWithCacheKey(...) 
    ({0xc0005a7630, 0xd}, 0xc000568cf0, 0x0)
    /app/dagql/objects.go:608 +0x1f4

github.com/dagger/dagger/dagql.NodeFunc(...) 
    /app/dagql/objects.go:588

github.com/dagger/dagger/dagql.Func(...) 
    ({0xc0005a7630, 0xd}, 0xc0007832e0)
    /app/dagql/objects.go:569 +0xfb

github.com/dagger/dagger/core/schema.agentSchema.Install.func1(...) 
    ({0x2a88d70, 0xc00054cd00}, 0xc00054ce00)
    /app/core/schema/agent.go:38 +0x65f

github.com/dagger/dagger/dagql.(*Server).InstallObject(0xc00006b980, {...})
    /app/dagql/server.go:221 +0xbf

github.com/dagger/dagger/dagql/introspection.Install(...)
    /app/dagql/introspection/types.go:67 +0xc16

github.com/dagger/dagger/core/schema.(*querySchema).Install(0xc0001c4300)
    /app/core/schema/query.go:37 +0x34

github.com/dagger/dagger/core/schema.(*CoreMod).Install(...) 
    (0x2a729a0?, {0xc000612e10?, 0xc0005459a0?}, 0xc00006b980)
    /app/core/schema/coremod.go:58 +0x592

github.com/dagger/dagger/engine/server.(*Server).initializeDaggerClient(...) 
    (0xc0002e6308, {0x2a7e770, 0xc0000ac4b0}, 0xc0005b2280, ...)
    /app/engine/server/session.go:554 +0x1677

github.com/dagger/dagger/engine/server.(*Server).getOrInitClient(...) 
    (0xc0002e6308, {0x2a7e770, 0xc0000ac4b0}, 0xc000612b40)
    /app/engine/server/session.go:799 +0xbd6

github.com/dagger/dagger/engine/server.(*Server).serveHTTPToClient(...) 
    (0xc0002e6308, {0x2a72c00, 0xc0001635e0}, ...)
    /app/engine/server/session.go:952 +0xeae

github.com/dagger/dagger/engine/server.(*Server).ServeHTTP.httpHandlerFunc[...].func1(...) 
    (0xc000552640)
    /app/engine/server/session.go:1426 +0x31

net/http.HandlerFunc.ServeHTTP(...) 
    /usr/lib/go/src/net/http/server.go:2220

golang.org/x/net/http2/h2c.h2cHandler.ServeHTTP(...) 
    /go/pkg/mod/golang.org/x/net@v0.33.0/http2/h2c/h2c.go:125 +0x673

net/http.serverHandler.ServeHTTP(...) 
    /usr/lib/go/src/net/http/server.go:3210 +0x8e

net/http.(*conn).serve(...) 
    /usr/lib/go/src/net/http/server.go:2092 +0x5d0

created by net/http.(*Server).Serve in goroutine 151
    /usr/lib/go/src/net/http/server.go:3360 +0x485
#

I think I got it

brazen mortar
#

nope 😛

#

I'll push the branch

brazen mortar
#

Going to bed, quick checkpoint @elder isle:

  • core/schema/agent.go I've commented out the class-setup code that you helped me with earlier. Somewhere in there is the cause of my earlier nil-pointer dereference panic
  • Hitting a deadlock probably caused by my middleware.
  • Started added debug logging, but the cycle of engine restart / click to engine logs in dagger cloud is slowing me down somewhat
brazen mortar
#

@quiet hill @mighty oyster @rose crypt in case you have any ideas... 🙏👆

quiet hill
#

I think you mean @mighty oyster 🙂

#

(will check in in a bit, currently reviewing the java sdk)

#

okay looking in now

#

🤔 what's remotefn

#

am i missing something, can't tell if it's wired in

brazen mortar
#

it's via the BBi interface. unrelated to the issue

quiet hill
#

the deadlock is caused by recursion in Install

# 
github.com/dagger/dagger/dagql.(*Server).installObjectNoMiddleware(0xc0003109a0, {0x2a4a330, 0xc00248d740})
    /app/dagql/server.go:231 +0x155 fp=0xc005e0dc20 sp=0xc005e0d9f8 pc=0x1286815
github.com/dagger/dagger/dagql.(*Server).installObject(0x25b6da0?, {0x2a4a330?, 0xc00248d740?})
    /app/dagql/server.go:224 +0x9e fp=0xc005e0dc60 sp=0xc005e0dc20 pc=0x128667e
github.com/dagger/dagger/dagql.Fields[...].findOrInitializeType(0x2a5fc60?, 0xc0003109a0?, {0x26744c0?, 0x5})
    /app/dagql/objects.go:832 +0x185 fp=0xc005e0dd38 sp=0xc005e0dc60 pc=0x1cec445
github.com/dagger/dagger/dagql.Fields[...].Install(0x2a5fc60, 0xc0003109a0)
    /app/dagql/objects.go:798 +0x105 fp=0xc005e0dfe8 sp=0xc005e0dd38 pc=0x1cecb25
github.com/dagger/dagger/core/schema.(*querySchema).Install(0xc007616480)
    /app/core/schema/query.go:63 +0x6ce fp=0xc005e0e780 sp=0xc005e0dfe8 pc=0x1cb6cae
github.com/dagger/dagger/core/schema.(*CoreMod).Install(0x2a33860?, {0xc000142b90?, 0xc001aaa280?}, 0xc0003109a0)
    /app/core/schema/coremod.go:58 +0x592 fp=0xc005e0e928 sp=0xc005e0e780 pc=0x1c7fd72
brazen mortar
#

yes I know there's an install lock

quiet hill
#

(just writing down what i'm doing so others can help out as well - i have to head off soonish)

brazen mortar
#

Also I shouldn't have pushed my frantic debugging changes... 😬 will maybe revert that so that the code tells a cleaner picture

#

fyi @elder isle we're discussing my broken code here

#

(the debugging changes are in the very last commit)

#

Done, I force-pushed the agent branch with less of the debugging changes. Will fix those to a separate branch

#

(pushed an unrelated bugfix)

#

@quiet hill if you want, we can talk live. But maybe too late in the day for you

quiet hill
#

yeah sorry, i picked this up way late in the day 😢

#

i would say, you shouldn't block on this - i don't think the lock there is essential for protecting against anything, there's only thread doing the installing afaict.

#

so can be commented out for now

brazen mortar
quiet hill
#

i am wondering if maybe the middleware signature is the right "shape" - the issue is that we're trying to install-while-installing, which does feel a little confusing

brazen mortar
#

Yeah, honestly I have no idea - I just needed a way to push my code out of dagql, into core

#

this seemed clean enough, for now

#

("for now" as in "for my POC", not for merging)

#

I'm in on rush to merge this, I just need it to work so that I can demo the result and we can decide if it's worth pursuing further

#

will try commenting out the lock now

quiet hill
#

ahhhhh

#

i think i see what's happening

#

i think you just need to move the lock out of the noMiddleware function and back into the top-level InstallObject function

#

yeah, that makes sense to me

brazen mortar
#

In the version I just forced-pushed (with debugging changes) removed I believe that's now the case.

#

Now trying a build with that + lock commented out

#

(dagger cloud wishlist: a way to see only currently running spans - I would use it to quickly zoom in to my dev engine to see its logs 🙂

quiet hill
#

i see

#

if you do that you get a different deadlock lol

#

with the lock commented out it'll work

brazen mortar
#

at the moment it's not working... but having trouble getting debugging messages out

#

sometimes I worry that I'm talking to the wrong nested engine 😬

quiet hill
#

yeee 🎉

#

now at least the schema is installed

brazen mortar
#

No sign of this log line:

    slog.Info("[AGENT] installing middleware")

Event though it's called before anything could deadlock

quiet hill
#

this isnt a deadlock now

#

the handler has crashed

quiet hill
brazen mortar
#

I only see 93 lines

#

well that would explain it 😛

quiet hill
#

what lol

brazen mortar
#

OK I'm starting to understand some of my confusion last night

quiet hill
#

yeah, it looks like selfType is nil

brazen mortar
#

ah, reloading the web page fixed it

#

damn... that definitely threw me off course last night 😭

quiet hill
#

need to head off now 😦
but just a note, before you keep digging, i would add a thing to the middlewares, to skip them if it's an "internal" object, i.e. one that begins with _

#

those internal objects are a bit special, and trying to alter their schemas will definitely cause "fun" at some point (even if it's not the cause of this)

brazen mortar
#

yeah... good idea. I worked around that by installing the middleware after the root query schema (which is where that stuff is installed I think)

#

Ok, this crash is the original problem I encountered - the deadlock appeared later while I was debugging 😛

quiet hill
#

i think there's still a couple cases where internal objects are installed later (e.g. introspection is added last, and introduces internal objects)

brazen mortar
#

Ok will add the check for _

quiet hill
#

good luck 🎉

brazen mortar
#

Thank you @quiet hill !

brazen mortar
#

@elder isle quick dagql question: is it possible to install a type A which references a type B, before the type B is installed? Otherwise I may have a circular dependency problem...

elder isle
#

i think so

#

we would have that problem all over the place otherwise

brazen mortar
#

yeah makes sense

elder isle
#

Container <-> Directory for example

brazen mortar
#

btw I hit a weird Cloud issue above, where my service logs don't render all the way - missing lines at the end

#

refreshing the page fixes it

elder isle
#

known issue - it's hard 😮‍💨

#

it's:

  • Cloud sees that the asService span completed (technically true, it doesn't care about the later effect of the service starting)
  • Cloud sees logs not come in for <grace period> and assumes logs are done
  • Cloud ends stream
brazen mortar
#

Honestly the refreshing is not the issue - it's the not knowing I have to refresh

#

Ah I see, it's specific to looking at it in real time as opposed to after completion

elder isle
#

yeah, that's why a refresh fixes it - you just reconnect and see more data

#

maybe the FE could reconnect since it "knows" it's still running

brazen mortar
#

All I know is: AI agents will fix this

#

😛

elder isle
brazen mortar
#

@elder isle what's the difference between ObjectType.Install and ObjectType.Extend? Not sure which to use in my middleware...

elder isle
#

what a great question

brazen mortar
#

and yes I did read the godoc but it did not help me 🙂

elder isle
#

pretty self explanatory

image.png
brazen mortar
#

something about not having access to the concrete implementation in Extend?

brazen mortar
elder isle
#

honestly it might just be that one is designed for DSL usage + generics, like dagql.Fields[T]{...}.Install(ctx)

brazen mortar
#

So just a cosmetic difference in how you call it, now fundamental difference in the end result?

elder isle
#

either should be fine - just use whichever is more ergonomic, yeah

#

they both just append to cls.fields

brazen mortar
#

(was wondering because graphql has an actual extend syntax. but this Extend seems unrelated)

elder isle
#

it's somewhat related, pretty sure i chose that word because it's conceptually extending the schema in the same way, it just also bringing along the implementation at the same time

brazen mortar
#

OK thanks! Let's see if I can get to the finish line

#

Oh - ObjectType doesn't have only has Extend anyway

elder isle
#

makes sense, since it can't be generic

#

ah that might be the root of it

brazen mortar
#

@elder isle as soon as this works, I'll implement your BBI

#

Was thinking that instead of # setVariable in the schema, you could have the model just pass a setVariable argument to its tool call

elder isle
#

that's how it works already

#

the # setVariable was just ad hoc doc notation

brazen mortar
#

Also I was thinking about the typing of the agent API. For now it's constrained to either 1) returning its own type (eg. mutating itself). or 2) talking to the user (retrieved via history). But what if it wants to return a Container or Directory or other custom type to me?

--> Was thinking that we could walk all the possible types that a pipeline starting from the self object could return - then generate an function returning each type. For example promptDirectory(string) *Directory -> prompt the agent with a task that must return a directory. And so on.

#

dag.SuperBuilder().AsAgent().PromptDirectory("Build me the frontend fully minified, then give me the source code, but with all big assets removed")

elder isle
#

hmmm interesting

#

seems like a lot of tab completion competition

brazen mortar
#

Also I have an idea for improving @white quartz 's "one-one" BBI. Currently it can't deal with chaining. So it can retrieve a file but can't read its contents for example.

But the BBI could walk the object return types, and flatten all available functions in all available types that return a scalar.

brazen mortar
# 
func (a *Agent) Type() *ast.Type {
    return &ast.Type{
        NamedType: a.selfType.TypeName() + "Agent",
        NonNull:   true,
    }
}
#

It makes it seem like Agent.selfType is nil, but there's no codepath where that should be possible...

#

Here is the entire middleware implementation. It has the only two calls to NewAgent:

func (s agentSchema) InstallObject(selfType dagql.ObjectType, install func(dagql.ObjectType)) {
    selfTypeName := selfType.TypeName()
    slog.Info("[agent middleware] installing original type: " + selfTypeName)
    // Install the target type first, so we can reference it in our wrapper type
    if !s.isAgentMaterial(selfType) {
        slog.Info("[agent middleware] not installing agent wrapper on special type " + selfTypeName)
        return
    }
    slog.Info("[agent middleware] installing wrapper type: " + selfTypeName + "Agent")
    defer slog.Info("[agent middleware] installed: " + selfTypeName + "Agent")
    agentType := dagql.NewClass[*core.Agent](dagql.ClassOpts[*core.Agent]{
        // Instantiate a throwaway agent instance from the type
        Typed: core.NewAgent(s.srv, core.LlmConfig{}, nil, selfType),
    })
    agentType.Install(
        dagql.Func("withPrompt", s.withPrompt).
            Doc("add a prompt to the agent context").
            ArgDoc("prompt", "The prompt. Example: \"make me a sandwich\""),
        dagql.Func("run", s.run).
            Doc("run the agent"),
        dagql.Func("history", s.history).
            Doc("return the agent history: user prompts, agent replies, and tool calls"),
        // FIXME
        //        dagql.Func("as"+selfTypeName, s.asObject).
        //            Doc("convert the agent back to a "+selfType.TypeName()),
    )
    selfType.Extend(
        dagql.FieldSpec{
            Name:        "asAgent",
            Description: fmt.Sprintf("convert the agent back to a %s", selfTypeName),
            Type:        agentType.Typed(),
        },
        func(ctx context.Context, self dagql.Object, args map[string]dagql.Input) (dagql.Typed, error) {
            return core.NewAgent(s.srv, s.llmConfig(), self, self.ObjectType()), nil
        },
    )
    // Install the wrapper type
    install(selfType)
    install(agentType)
}
elder isle
#

ah, i think you can't use the dagql.Func helper here, since it's going to allocate a zero-value for the return type - likely one of those returns your *core.Agent

#

or, you can, but you have to re-assign field.Spec.Type to be your instantiated type

brazen mortar
#

yeah withPrompt and run are chainable, so they return the agent type

#

So if I do it all with raw Extend() like I do for asAgent, it should work?

elder isle
#

yep!

brazen mortar
elder isle
#

yea that should work too by the looks of it

brazen mortar
#

When using Extend, can my field function type its self object to the real concrete type?

eg.

agentType.Extend(
  dagql.FieldSpec{...},
  func(ctx context.Context, self *core.Agent /* <-- possible? */, args map[string]dagql.Input) (dagql.Typed, error) { ... },
elder isle
#

nah, Extend is the boring unmagical one

brazen mortar
#

Ok so manual cast should work though

elder isle
#

yep

#

well

#

it'll probably be dagql.Instance[*core.Agent]

brazen mortar
#

a := self.(dagql.Instance[*core.Agent]).Self

elder isle
#

yup yup

brazen mortar
#

Does this look right?

    agentType.Extend(
        dagql.FieldSpec{
            Name:        "withPrompt",
            Description: "add a prompt to the agent context",
            Type:        agentType.Typed(),
            Args: dagql.InputSpecs{
                {
                    Name:        "prompt",
                    Description: "the prompt",
                    Type:        dagql.String(""),
                },
            },
        },
        func(ctx context.Context, self dagql.Object, args map[string]dagql.Input) (dagql.Typed, error) {
            a := self.(dagql.Instance[*core.Agent]).Self
            return a.WithPrompt(args["prompt"].(*dagql.String).String(), nil
        },
    )
#

(Going the Extend route after all, more predictable)

elder isle
#

what does .Typed() do?

#

lgtm generally

#

oh wait

#

it'll be dagql.String, not *dagql.String

#

and there's a ) missing there, otherwise lgtm

brazen mortar
#

I guess I don't need it

#

(predates the class creation I think)

#

@elder isle actually I do need it I think. Without that .Typed() I get:

compiler: cannot use agentType (variable of type dagql.Class[*core.Agent]) as dagql.Typed value in struct literal: dagql.Class[*core.Agent] does not implement dagql.Typed (missing method Type)
elder isle
#

oh makes sense, i didn't realize that's the class, thought it was your NewAgent but that's inside the class, so Typed() is just pulling it back out

brazen mortar
#

a graphql schema error, haven't seen one of those in a while

#

middleware install seems to have worked

#

Looks like I made a type (__Schema) disappear? But:

time="2025-01-28T18:33:41Z" level=info msg="[agent middleware] installing original type: __Schema"
time="2025-01-28T18:33:41Z" level=info msg="[agent middleware] not installing agent wrapper on special type __Schema"
#

oh I see it. oops

#

rebuilding, almost there!

#

We have a successful schema injection!

#

Now will add back the full implementation

#

@elder isle only one catch, everything seems to return null...

⋈ container  | as-agent
null
⋈ container  | as-agent | with-prompt "make me a sandwich"
null
elder isle
#

huh.

#

is it possible that shell / the TUI just has no way of displaying it?

#

i don't even see it executing the query

brazen mortar
#

well it's just a string

#

oh wait no it's not

#

Is there plumbing I should add for this?

elder isle
#

there are spans beneath container but not beneath any as-agent calls - seems like it's not even hitting your API

image.png
#

maybe add a scalar field to it?

#

so the shell/TUI will force evaluation?

brazen mortar
#

but how come it works for regular objects?

#

Ok disregarding for now. First let's get the full implementation plumbed in, and see if it actually works 🙂 I can live with a few null if so

#

How do I specify that my function returns an array of strings in FieldSpec?

elder isle
#

dagql.ArrayInput[dagql.String]{} i think

brazen mortar
#

ah ArrayInput

elder isle
#

ye

elder isle
#

i've run into this weirdness too

#

call and shell seem to rely on the presence of scalar fields to force evaluation

brazen mortar
#

in this context you mean "field" in the raw graphql sense? Without the module-specific distinction between "fields" and "functions" right?

elder isle
#

yeah

#

in the same manner that scalar fields are the only things that "un-lazy" in the SDKs

brazen mortar
#

Ok it all seems to work - next blocker: agent fails to lookup the llm key in the secret store

#

looks like maybe I just fat-fingered the shell command

#

OK I'm stuck on this...

#

@elder isle this is back to the trick we discussed yesterday:

func (s agentSchema) llmConfig() core.LlmConfig {
    // The LLM config is attached to the root query object, as a global configuration.
    // We retrieve it via the low-level dagql server.
    // It can't be retrieved more directly, because the `asAgent` fields are attached
    // to all object types in the schema, and therefore we need a retrieval method
    // that doesn't require access to the parent's concrete type
    //
    // Note: only call this after the `core.Query` has been installed on the server
    return s.srv.Root().(dagql.Instance[*core.Query]).Self.LlmConfig
}
#

so frustrating

elder isle
#

what's the issue?

#

can't get the key?

brazen mortar
#

worked around it for now, with a global var 🙂

#

Now I have actual openai API calls and tool calls 🙂 then another crash

#

ah, not a crash! good old errors

elder isle
#

notsureif i wonder if that's broken on main? the call looks correct

brazen mortar
#

I think the issue is now purely in my code. BBI implementation, translating LLM tool calls to dagql queries etc

#

the error is not on the actual call - but on the nested call that the LLM attempts to make

#

will bring back custom spans!

#

I have to go afk for a bit... If you find an opportunity to play with it I would greatly appreciate it! Trying to get a demo to work for today

elder isle
#

oh, yea it's definitely something in there, phew

brazen mortar
#

looks like it successfully called 2 functions. It just called them wrong 🙂

elder isle
#

@brazen mortar gonna try a less fraught way to convert the args into a Selector, we have a lot of code similar to that already that we can probably reuse

#

e.g. ObjectType.ParseField

brazen mortar
#

FYI everything in bbi_utils.go was AI-generated 🙂

#

so I won't take it personally

#

Actually these errors are not even BBI errors - they're dumber than that, as soon as a tool call fails, I error out of the whole loop. Instead I should just send the error back to the model so that it can adjust

#

that's what these errors are

elder isle
#

i'm looking into the weird 'expected slice, got struct' one

# 
❯ dagger-dev shell --no-mod
Dagger interactive shell. Type ".help" for more information. Press Ctrl+D to exit.
⋈ .core | with-llm gpt-4o OPENAI_API_KEY | git https://github.com/dagger/dagger | as-agent | with-prompt "what is your latest stable release?" | run | history
🧑 💬what is your latest stable release?
🤖 💬
🤖 💻 tags({"patterns":["refs/tags/v*"]})
🤖 💬The latest stable release is version **v0.15.2**.
⋈

there we go

#

@brazen mortar pushed ☝️

brazen mortar
#

oooooh!

#

@elder isle fun fact: you can call with-llm gpt-4o LLM_KEY then start a new chain (in the same session); since the. llm config is in an engine-wide global variable (probably not a good idea to merge it that way)

#

This means you can prompt modules

# 
.core | with-llm gpt-4o LLM_KEY
github.com/dagger/dagger | as-agent | with-prompt "run the docs server, connect it to it over http and see if /quickstart seems to render OK. Also comment on if it the text seems clear" | run
elder isle
elder isle
# brazen mortar ``` .core | with-llm gpt-4o LLM_KEY github.com/dagger/dagger | as-agent | with-p...

doesnt seem to work:

❯ dagger-dev shell --no-mod
Dagger interactive shell. Type ".help" for more information. Press Ctrl+D to exit.
⋈ .core | with-llm gpt-4o OPENAI_API_KEY
defaultPlatform: linux/amd64
version: v0.15.3-010101000000-dev-6cfe7a2336cb
⋈ github.com/dagger/dagger | as-agent | with-prompt "run the docs server, connect it to it over http and see if /quickstart seems to render OK. Also comment on if it the text seems clear" | run
Error: no function "as-agent" in type "DaggerDev"
⋈
brazen mortar
#

Oh you're right! I spoke too soon

#

Maybe module loading doesn't call dagql.Server.installObject?

#

(that's what I hook)

elder isle
#

or your middleware isn't making it to module schemas

#

hmm seems like it should

brazen mortar
#

I'll add an endpoint to dump the tools schema

elder isle
#

@brazen mortar eek i think the problem might be that the CLI is based on introspecting via the Dagger APIs, not the schema, so your asAgent field doesn't show up there

brazen mortar
#

Ah. So that's a different level of middleware I might need?

#

Surprising because I believe the shell also uses Dagger API for introspection

#

But actually maybe not

elder isle
#

no, it does - that's the problem

#

if it were introspecting the graphql schema it'd be fine

#

but it's introspecting based on the module's TypeDefs

brazen mortar
#

I see. And in the case of core types, there's no distinction?

elder isle
#

trying to figure that out

brazen mortar
#

Makes me wonder if it would work from code - trying now

elder isle
#

ah interesting, core module is introspected using the dagger API too for shell, but for the core "module" it actually works via schema introspection under the hood, so that's why it works

brazen mortar
#

I see more middleware in my future... 🔮

#

Getting it to work with modules is key to validating the theory (that Dagger can be a world-class platform for multi-agent, taking advantage of modules calling modules calling modules...)

quiet hill
#

just had a thought 🤔 for some reason i'm genuinely interested in the idea of middleware a lot

#

...because every client (module / cli) etc has a different "view" of the schema, you could imagine them also having different middlewares potentially

#

and if that's the case... then you could distribute middlewares themselves as modules maybe

#

and have them just like dependencies

#

like, it's the natural hook point for llms, but it could also be the natural hook point for doing supply chain attestations

#

or for building on the default telemetry, e.g. to record (and highlight) spans that match certain criteria

#

dunno, it just feels like a potentially very powerful way to metaprogram the engine - but from the perspective of a module, in a way that keeps it neatly boxed

#

(late night musings with jed)

brazen mortar
#

Yeah I agree there's something there

#

They might have to be distributed separately - if you dagger install and expect a neatly sandboxed dependency, and instead all your other dependencies are completely different, that would be... disconcerting.

Kind of installing an app vs. system extension on mac

#

Just pushed a new debug method. Call tools on any agent and it will dump all tools with name, description, and args json schema

quiet hill
#

100% yes, i guess it feels somewhat like installing rust macro crates as well

#

but i do really like the idea that it's still somewhat sandboxed - only my dependencies look different, all of my dependencies dependencies still look the same to them with the power of views

#

in my head it's essentially a graphql extension - but instead of a "module" only adding new types, you're able to programatically extend all the types

#

but ofc in a very type safe way

#

it also feels like a missing piece in how we'd go about making "core" a proper module - core is so fundamental in so many ways, but some of it isn't - if you could build an extension as a module, then we could move huge parts of the core out

brazen mortar
#

@quiet hill @elder isle quick status update:

  • I'm focusing on convenience and polish.
  • Agent.tools() to dump available tools
  • Agent.ask() to ask a question and get a string answer, in one go, without changing the agent state
  • Agent.do() to perform a task and get a new agent state, in one go
  • Agent.lastReply() to get the last message sent by the agent

I really need to get asAgent to work on modules also... But not sure how to attack that

#

Another possible task is to get LLM key by session attachable, would make everything else even more zero-conf

brazen mortar
#

middleware

#

@quiet hill @elder isle I'm re-using this thread for my continued middleware adventures...

#

Here is what @craggy matrix posted to help me out:

most likely this: https://github.com/sipsma/dagger/blob/bc8146a76ed55513c82d208850dd9f9a2073d5b9/cmd/dagger/typedefs.graphql#L68-L68, which is indeed currentTypeDefs

currentTypeDefs impl does:

So yeah ensuring those fields on type Module match what you installed in graphql should do it

#

not sure where to start - my existing middleware (which gets called on new object installed in dagql) would:

  1. determine if the object is defined by a module
  2. look up which module
  3. patch the typedef for the wrapped type (to add asAgent)
  4. add a typedef for the wrapper type

?

elder isle
#

taking a look

brazen mortar
#

Or, maybe I need another middleware, in the module registration lifecycle?

#

but then I would have to duplicate all the type definition logic

#

@elder isle if you want to play with the last branch: all you need is a .env in the workdir with LLM_KEY=<VALUE>

#

you can also set LLM_MODEL, LLM_HOST and LLM_PATH, they should be honored (for other models)

#

And, thanks to the side-loading of LLM config, you can write a module that seamlessly turns objects into agents (but only core objects for now...)

# 
package main

import (
    "context"
    "fmt"
    "os"
)

type VersionWatcher struct{}

func (m *VersionWatcher) Latest(ctx context.Context, repo string) (string, error) {
    return dag.Git(repo).AsAgent().Ask(ctx, "what's your latest stable version? only give the semver string and nothing else")
}

--> not a super impressive example since there's probably a git command to do this 🙂 👆

elder isle
#

trying a quick hack: just have module introspection work the same way core introspection does (introspect actual schema and generate TypeDefs from it)

brazen mortar
#

🙏

#

meanwhile I'm trying to add support for chaining in the BBI

#

(right now the model can call functions that return an ID, but can't do anything with it)

#

that + agentifying modules are the remaining blockers to really trying to build something cool

#

(at the moment no chaining means a directory can't read the contents of files for example)

elder isle
elder isle
#

@brazen mortar gonna take this as a win, think you just changed the API - will push so you can try it out

❯ dagger-dev shell --no-mod
Dagger interactive shell. Type ".help" for more information. Press Ctrl+D to exit.
⋈ github.com/dagger/dagger | as-agent | with-prompt "run the docs server, connect it to it over http and see if /quickstart seems to render OK. Also comment on if it the text seems clear" | run
Error: no function "with-prompt" in type "DaggerDev"
⋈ github.com/dagger/dagger | as-agent | ask "run the docs server, connect it to it over http and see if /quickstart seems to render OK. Also comment on if it the text seems clear" | run
#

super hacky impl but i think it works - the asAgent call worked

brazen mortar
#

@elder isle same API as before, I just added sugar on top: do and ask are built on top of with-prompt + run + history

#

I guess run is kind of like sync 🙂

brazen mortar
# 
⋈ github.com/dagger/dagger | as-agent | ask "run the docs server, connect it to it over http and see if /quickstart seems to render OK. Also comment on if it the text seems clear" | run

Missing part of the snippet?

elder isle
#

will take a look, disclaimer: had to take a call so I had Cline do all the work, so any errors are his(?) fault, not mine

brazen mortar
#

(who's Cline?)

elder isle
#

it's a VScode extension that turns it into an AI engineer

#

supports MCP/tools but also natively supports browsing the web, editing, and using the editor terminal

white quartz
#

the meta tool is one function, e.g. "file". Takes a prompt as input, the handler is just using that prompt with all available tools

#

e.g.

"clone dagger and summarize the readme"

  • assuming clone() returns a File
  • there's a "tool" called file. Signature is file(prompt: string) or something
  • LLM calls file(prompt: "grab the contents of README.md")
  • Built-in handler calls underlying_model("grab the contents of README.md", tools=[all File tools])
#

it's basically a pseudo-router for functions within a module

#

so we avoid the need for flattening the namespace, have pseudo-tools for each module that take care of fine-tuning the individual calls

brazen mortar
#

but isn't that for regular code calling the agent, rather than model calling the tools?

white quartz
#

not sure if it makes a difference?

brazen mortar
brazen mortar
#

but maybe I misunderstood what you meant and What I'm talking about is different

white quartz
#

I meant agent calling code -- I don't fully understand the other direction of code calling agent (as in, how it's different from code calling code)

brazen mortar
#

@elder isle I'm back at it, I just pulled - is it safe to make changes on top of that last commit, or is more refactoring coming?

#

also does it work? 🙂

elder isle
#

still trying to figure out why the heck it's getting the same type back in shell thinkies

elder isle
brazen mortar
#

Ok - I'm focusing on another area anyway

#

Any leads on how to get modules to work? I can try to keep looking if you have to switch to something else

#

ie. if I can help let me know

elder isle
#

figured it out sweating

#

@brazen mortar you can pull - hella messy but should actually work now

brazen mortar
#

ouh yeah

#

necessary messy, or temporary messy?

#

either way, thank you!!

elder isle
brazen mortar
#

brb gonna prompt a module

#

I wonder what happens if I prompt my gpt module... can I tell the core LLM to think about the best prompt for the gpt module - thus implementing chain of thought, kinda?

elder isle
#

err... i must have broken it just before pushing, re-investigating 😅 seeing a new error now

⋈ github.com/vito/daggerverse/apko | as-agent | ask "what is your purpose"
Error: input: apko.asAgent instantiate: cannot instantiate dagql.Class[*github.com/dagger/dagger/core.ModuleObject] with *core.Agent
elder isle
brazen mortar
#

"take a deep breath. think harder."

#

it's the new "sudo make me a sandwich"

elder isle
#

ok it works once I put "WTF" in front of the agent type name 👍 shipit 🚢

mighty oyster
# brazen mortar I wonder what happens if I prompt my gpt module... can I tell the core LLM to th...

i have this horrible tip-of-my-tongue problem where i swear to god there's a phrase from some mathematician or philosopher of the mind for how entropy compounds on itself when 2 algos talk to each other in a loop... it's like somewhere between hofstadters strange loops, searle's chinese room, the telephone game, and round trip translation failure but i cannot find the fucking phrase and im beginning to fear it only exists in my own head

#

but i do see a lot of ppl talking about using LLMs to generate prompts for LLMs, and boy does it seem fraught lol

#

especially when unsupervised... but probably works very well with supervised techniques (a way to benchmark the result and let the model iterate to a locally optimal input)

brazen mortar
#

I mean it's inevitable. At some point it will become wasteful to have a human look at the coding assistants suggestions and manually press "accept" or "try again" all day. So obviously you need a middle manager agent to click "accept" instead

#

then the human developer can focus on the higher-level task of doing quarterly performance review and weekly 1-1 meetings with the AI

#

I can't wait

#

Eventually we'll need a "no meeting thursday policy" to get all the AIs to stop scheduling pointless meetings with each other

elder isle
brazen mortar
elder isle
#

yea lol. something in our namespacing logic/heuristics is misfiring

#

related to the root module object

brazen mortar
#

@elder isle wow you really simplified that fieldArgsToJSONSchema

#

Alex said "not today, skynet"

brazen mortar
#

Another noob question: given a *ast.Definition type that I know is an object, how can I get its ID type?

#

I think I got it

brazen mortar
#

Ok one more... given a *call.ID and object type name (eg. "Directory") how do I load the corresponding Object?

#

ah! dagql.Server.Load 🙂

brazen mortar
#

God (basic) chaining to work. With some optimizations to keep token count low:

  • we only flatten functions for calls that we actually returned an ID for.
  • I send the ID digest, and keep a lookup table of the actual IDs in the state. Then lookup when the model sends the id
#

Still have some weird bugs. Like when the model tries to call Container.withExec it says it can't find the field. But other fields in the same container work fine...

brazen mortar
#

A quick list of outstanding bugs, any help fixing them would be appreciated!

  • agent can't call Container.withExec and I have no idea why...
  • When calling agent | ... | state on a module, I get the text encoding of the state. It's supposed to check whether it's an object type (to return an ID) or a non-object (to string-encode), It looks like it doesn't detect object return types when they're from modules.
  • I have occasional "ID lookup fail" related to my IDs field, not sure why. Possibly the model "eats" parts of the ID, and needs to be prompt-engineered into compliance...
  • errors should be caught and sent back to the model instead of interrupting the loop
brazen mortar
#

@here if anyone could help me with that Container.withExec... it's really hurting the usefuleness...

#

of course it had to be that function

mighty oyster
#

i do like the irony of it being withexec lol, i'm not sure there's a worse single function to be uncooperative

brazen mortar
#

exactly

brazen mortar
#

@mighty oyster sorry getting back to you about that error..

#

@mighty oyster here's the error I hit:

    classField, ok := target.ObjectType().FieldSpec(field.Name)
    if !ok {
        return nil, nil, fmt.Errorf("field %q not found in object type %q", field.Name, s.self.ObjectType().TypeName())
    }

--> I get field withExec not found in object type Container

#

@elder isle this is how I check if a return type is an object or not... From there I either send the model an ID (if object) or the marshalled value (if not object).

--> Core objects are correctly detected
--> But module objects are not. As a result AgentSmith.State() returns a giant string instead of a Smith

https://github.com/shykes/dagger/blob/agent/core/bbi.go#L162-L168

// Return true if the given type is an object
func (s *OneOneBBISession) isObjectType(t *ast.Type) bool {
    objType, ok := s.srv.Schema().Types[t.Name()]
    if !ok {
        return false
    }
    return objType.Kind == ast.Object
}

Does this look wrong?

GitHub
dagger/core/bbi.go at agent · shykes/dagger

A portable devkit for CI/CD pipelines. Contribute to shykes/dagger development by creating an account on GitHub.

elder isle
#

looks fine, but hmm, maybe that *dagql.Server is only the core schema

brazen mortar
#

I feel like ever since the module middleware layer is involved, I lost what little sense I had of knowing what the hell is going on 😭

#

and how the types relate to each other

#

@elder isle but everything goes through dagql.Server.installObject in the end, and presumably that's how the schema is updated no?

elder isle
#

yeah, just a guess

#

do you have a one-liner repro?

brazen mortar
#

github.com/dagger/dagger/modules/wolfi | agent | state

#

(replace that module with any other module)

#

oh interesting. if you do github.com/dagger/dagger/modules/wolfi | agent | .doc state -> ☠️ panic

#

I can't tell if it's the engine or the cli that had the panic

#

engine seems fine

#

probably that type info triggered a bug in the shell .doc implementation?

mighty oyster
# brazen mortar <@430802613848506380> here's the error I hit: ```golang classField, ok := t...

ah, you're fairly far out on a branch, that fieldspec isn't in main lol ```func (class Class[T]) fieldLocked(name string, views ...string) (Field[T], bool) {
fields, ok := class.fields[name]
if !ok {
return Field[T]{}, false
}
for i := len(fields) - 1; i >= 0; i-- {
// iterate backwards to allow last-defined field to have precedence
field := fields[i]

    if field.ViewFilter == nil {
        return *field, true
    }
    for _, view := range views {
        if field.ViewFilter.Contains(view) {
            return *field, true
        }
    }
}
return Field[T]{}, false

}

brazen mortar
#

oh?

mighty oyster
#

or i guess just missing the first bit entirely

brazen mortar
#

I rebased on main just yesterday though

mighty oyster
brazen mortar
#

Oh I see - yeah I haven't looked closely at that 🙂

mighty oyster
#

i just mean the FieldSpec impl is still on a branch, i should blabber less, just catching up

brazen mortar
elder isle
#

sorry back, had to futz around with a scanner, took too long to realize i need to turn it off and on again -_-

mighty oyster
#

lol small world i had a fight with my printer earlier

#

it lost

elder isle
brazen mortar
#

"Have you tried turning yourself off and on again"

brazen mortar
#

I'm still stuck on that "return module object as string" problem... Do you have any leads @elder isle @mighty oyster ?

#

I know everyone is busy, I don't know who else on a reasonable timezone I can ask for help ...

#

cc @twin charm

elder isle
#

taking a look!

#

ah found something dumb

#

@brazen mortar pushed, it was a one-line fix 🙃

# 
diff --git a/core/agent.go b/core/agent.go
index 292f7b968..a44a138c6 100644
--- a/core/agent.go
+++ b/core/agent.go
@@ -598,6 +598,7 @@ func (s AgentMiddleware) ModuleWithObject(ctx context.Context, mod *Module, self
                                Name:         selfTypeName,
                                OriginalName: selfTypeName,
                        },
+                       Valid: true,
                },
        }
brazen mortar
#

nice! thank you!

brazen mortar
#

testing now

#

@elder isle FYI yesterday I cleaned up the BBI interface, now it works exactly like sql drivers in the go stdlib

#

I wanted to port your gql driver, but ran out of steam after 15mn, too many sleepless nights 😅

#

I got a chance to see your use of the mcp package, and was thinking - maybe we should just co-opt that and reuse their code for cumbersome tasks like converting Go-typed arguments to json schema etc :

#

(but would be different from full MCP compatibility, since MCP defines a lot of crap that don't fit in BBI)

elder isle
#

ah yea interesting

brazen mortar
#

ha ha ha ha it's aliiiiive! So much cooler with module state thank you @elder isle

brazen mortar
#

Quick update:

  • Rebased on main (had to resolve a small conflict), agent calls are now correctly cached in the same session! Huge usability win 🙂 Thank you @craggy matrix !

  • Ergonomic win: you can now configure LLM_KEY using a secret provider URI in your .env. For example mine is: LLM_KEY=op://Private/bsvkkuzosh5u5v7thsp36yw524/credential. It feels really nice, awesome work on the secrets providers @white quartz @unreal parcel. Note: this will require building your CLI from the branch (all other features work on a stable CLI)

brazen mortar
#

(force-pushed with the full history restored. I had previously squashed 28 commits together to help me deal with merge conflicts on main rebase. But I I went back and got it to work without losing history)

brazen mortar
#

Next batch of errors...

Bug 1. "Can't instantiate..."

Repro:

⋈ container | from alpine | agent | please "figure out your linux distro; then install git. look at at all available tools" | history
✔ container: Container! 0.0s
○ .from(address: "alpine"): Container! 1.8s
✔ .agent: ContainerAgent! 0.0s
🧑 💬figure out your linux distro; then install git. look at at all available tools
🤖 💻 from({"address": "alpine"})
💻 error calling tool: new object: cannot instantiate dagql.Class[*github.com/dagger/dagger/core.Container] with dagql.Instance[*github.com/dagger/dagger/core.Container]
🤖 💻 from({"address": "ubuntu"})
💻 error calling tool: new object: cannot instantiate dagql.Class[*github.com/dagger/dagger/core.Container] with dagql.Instance[*github.com/dagger/dagger/core.Container]
#

Bug 2. "FIeld "withExec" not found..."

⋈ container | from alpine | agent | please "execute a simple command (picked by you) in the container and show me the result"  | history
🧑 💬execute a simple command (picked by you) in the container and show me the result
🤖 💻 withExec({"args":["echo","Hello, World!"]})
💻 error calling tool: field "withExec" not found in object type "Container"
🤖 💻 withExec({"args":["echo","Hello, World!"]})
💻 error calling tool: field "withExec" not found in object type "Container"
🤖 💻 withExec({"args":["echo","Hello, World!"]})
💻 error calling tool: field "withExec" not found in object type "Container"
🤖 💬It seems there is an issue with executing commands directly. Please verify that the container is correctly initialized or provide more context for me to assist you further.
#

Bug 3. Cannot create int from float64

⋈ directory | agent | please "write a file hello.txt with contents 'hello world' and set the permissions to 0600" | history
🧑 💬write a file hello.txt with contents 'hello world' and set the permissions to 0600
🤖 💻 withNewFile({"path":"hello.txt","contents":"hello world","permissions":384})
💻 error calling tool: decode arg "permissions": cannot create Int from float64
🤖 💻 withNewFile({"path":"hello.txt","contents":"hello world","permissions":600})
💻 error calling tool: decode arg "permissions": cannot create Int from float64
🤖 💻 withNewFile({"path":"hello.txt","contents":"hello world","permissions":384})
💻 error calling tool: decode arg "permissions": cannot create Int from float64
🤖 💻 withNewFile({"path":"hello.txt","contents":"hello world","permissions":384})
💻 error calling tool: decode arg "permissions": cannot create Int from float64
🤖 💻 withNewFile({"path":"hello.txt","contents":"hello world","permissions":384})
💻 error calling tool: decode arg "permissions": cannot create Int from float64
🤖 💻 withNewFile({"path":"hello.txt","contents":"hello world","permissions":384})
💻 error calling tool: decode arg "permissions": cannot create Int from float64
🤖 💻 withNewFile({"path":"hello.txt","contents":"hello world"})
💻 ok
🤖 💬I have created the file `hello.txt` with the contents "hello world". The file was created without setting specific permissions due to an earlier issue, so it has the default permissions. Would you like me to attempt setting the permissions again?
⋈
brazen mortar
#

I found a way to do it without middleware... way simpler 😭

#

Required flipping around the API, which also makes it easier to explain...

# 
llm | with-directory $DIR | with-prompt "find all text files in the directory, and translate them to french" | directory
#

Also opens the possibilities of letting the llm track multiple typed variables in their state:

llm |
 with-directory source $DIR |
 with-container base $CONTAINER |
 with-prompt "you have access to a source directory and a base container. Inspect the source, figure out what toolchain is needed to build it, install the toolchain on the container, and build the software. Save the build directory to the variable 'build'" |
 directory build
#

Unfortunately the refactoring didn't remove any of the 3 bugs above 😢

brazen mortar