Skip to content

Commit

Permalink
add tracing & metrics (#149)
Browse files Browse the repository at this point in the history
  • Loading branch information
@tim-smart
tim-smart committed Apr 9, 2024
1 parent 20ca499 commit 54c1b8c
Show file tree
Hide file tree
Showing 13 changed files with 496 additions and 49 deletions.
7 changes: 7 additions & 0 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,16 @@
"typescript": "^5.4.4"
},
"dependencies": {
"@effect/experimental": "^0.14.12",
"@effect/opentelemetry": "^0.31.28",
"@effect/platform": "^0.48.27",
"@effect/platform-node": "^0.45.29",
"@effect/schema": "^0.64.20",
"@opentelemetry/exporter-metrics-otlp-http": "^0.50.0",
"@opentelemetry/exporter-trace-otlp-http": "^0.50.0",
"@opentelemetry/sdk-metrics": "^1.23.0",
"@opentelemetry/sdk-trace-base": "^1.23.0",
"@opentelemetry/sdk-trace-node": "^1.23.0",
"dfx": "^0.91.1",
"dotenv": "^16.4.5",
"effect": "2.4.18",
Expand Down
332 changes: 332 additions & 0 deletions pnpm-lock.yaml
View file

Large diffs are not rendered by default.

4 changes: 4 additions & 0 deletions src/AutoThreads.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,7 @@ const make = ({ topicKeyword }: { readonly topicKeyword: string }) =>
Effect.logDebug(TreeFormatter.formatIssue(error.error)),
}),
Effect.catchAllCause(Effect.logError),
Effect.withSpan("AutoThreads.handleMessages"),
),
)

Expand Down Expand Up @@ -161,6 +162,7 @@ const make = ({ topicKeyword }: { readonly topicKeyword: string }) =>
}),
),
withEditPermissions,
Effect.withSpan("AutoThreads.edit"),
),
)

Expand All @@ -181,6 +183,7 @@ const make = ({ topicKeyword }: { readonly topicKeyword: string }) =>
type: Discord.InteractionCallbackType.DEFERRED_UPDATE_MESSAGE,
}),
),
Effect.withSpan("AutoThreads.editSubmit"),
),
)

Expand All @@ -197,6 +200,7 @@ const make = ({ topicKeyword }: { readonly topicKeyword: string }) =>
}),
),
withEditPermissions,
Effect.withSpan("AutoThreads.archive"),
),
)

Expand Down
11 changes: 11 additions & 0 deletions src/DocsLookup.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ const make = Effect.gen(function* (_) {
),
Effect.annotateLogs("module", "DocsLookup"),
Effect.annotateLogs("query", query),
Effect.withSpan("DocsLookup.search", { attributes: { query } }),
)
}

Expand Down Expand Up @@ -89,6 +90,12 @@ const make = Effect.gen(function* (_) {
Effect.bind("entry", ({ key, docs }) =>
Effect.fromNullable(docs.map[key]),
),
Effect.tap(({ entry, reveal }) =>
Effect.annotateCurrentSpan({
entry: entry.signature,
public: reveal,
}),
),
Effect.bind("embed", ({ entry }) => entry.embed),
Effect.map(({ embed, reveal }) =>
Ix.response({
Expand All @@ -112,12 +119,14 @@ const make = Effect.gen(function* (_) {
}),
),
}),
Effect.withSpan("DocsLookup.command"),
),
)

const autocomplete = Ix.autocomplete(
Ix.option("docs", "query"),
Ix.focusedOptionValue.pipe(
Effect.tap(query => Effect.annotateCurrentSpan("query", query)),
Effect.filterOrFail(
_ => _.length >= 3,
_ => new QueryTooShort({ actual: _.length, min: 3 }),
Expand Down Expand Up @@ -147,6 +156,7 @@ const make = Effect.gen(function* (_) {
}),
),
}),
Effect.withSpan("DocsLookup.autocomplete"),
),
)

Expand Down Expand Up @@ -282,4 +292,5 @@ const wrapCodeBlock = (code: string) =>
}),
Effect.catchAllCause(_ => Effect.succeed(code)),
Effect.map(_ => "```typescript\n" + _ + "\n```"),
Effect.withSpan("DocsLookup.wrapCodeBlock"),
)
11 changes: 7 additions & 4 deletions src/Github.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,13 @@ const make = ({ token }: { readonly token: Secret.Secret }) => {
type Endpoints = typeof rest

const request = <A>(f: (_: Endpoints) => Promise<A>) =>
Effect.tryPromise({
try: () => f(rest),
catch: reason => new GithubError({ reason }),
})
Effect.withSpan(
Effect.tryPromise({
try: () => f(rest),
catch: reason => new GithubError({ reason }),
}),
"Github.request",
)

const wrap =
<A, Args extends any[]>(
Expand Down
8 changes: 7 additions & 1 deletion src/Issueifier.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ https://discord.com/channels/${channel.guild_id}/${channel.id}
`,
}),
),
Effect.withSpan("Issueifier.createIssue"),
)

const followUp = (
Expand Down Expand Up @@ -106,6 +107,7 @@ https://discord.com/channels/${channel.guild_id}/${channel.id}
),
),
),
Effect.withSpan("Issueifier.followUp"),
)

const command = Ix.global(
Expand All @@ -131,6 +133,7 @@ https://discord.com/channels/${channel.guild_id}/${channel.id}
const context = yield* _(Ix.Interaction)
const repoIndex = yield* _(ix.optionValue("repository"))
const repo = githubRepos[repoIndex]
yield* _(Effect.annotateCurrentSpan({ repo: repo.label }))
const channel = yield* _(
channels.get(context.guild_id!, context.channel_id!),
)
Expand All @@ -147,7 +150,10 @@ https://discord.com/channels/${channel.guild_id}/${channel.id}
type: Discord.InteractionCallbackType
.DEFERRED_CHANNEL_MESSAGE_WITH_SOURCE,
})
}).pipe(Effect.annotateLogs("command", "issueify")),
}).pipe(
Effect.annotateLogs("command", "issueify"),
Effect.withSpan("Issueifier.command"),
),
)

const ix = Ix.builder
Expand Down
4 changes: 3 additions & 1 deletion src/MemberCache.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,9 @@ const make = Effect.gen(function* (_) {

return {
get: (guildId: Discord.Snowflake, userId: Discord.Snowflake) =>
cache.get(new GetMember({ guildId, userId })),
cache
.get(new GetMember({ guildId, userId }))
.pipe(Effect.withSpan("MemberCache.get", { attributes: { userId } })),
} as const
})

Expand Down
1 change: 1 addition & 0 deletions src/NoEmbed.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ const make = ({
Effect.logDebug(TreeFormatter.formatIssue(error.error)),
}),
Effect.catchAllCause(Effect.logError),
Effect.withSpan("NoEmbed.handleMessage"),
)

yield* _(
Expand Down
56 changes: 35 additions & 21 deletions src/OpenAI.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,20 @@ import {
Option,
pipe,
Predicate,
Metric,
} from "effect"
import * as Tokenizer from "gpt-tokenizer"
import * as OAI from "openai"
import type { APIError } from "openai/error.mjs"

const metrics = {
duration: Metric.timer("openai_duration"),
calls: Metric.counter("openai_calls", {
bigint: true,
incremental: true,
}),
} as const

export class OpenAIError extends Data.TaggedError("OpenAIError")<{
readonly error: APIError
}> {
Expand Down Expand Up @@ -47,36 +56,41 @@ const make = (params: {
Effect.tryPromise({
try: signal => f(client, signal),
catch: error => new OpenAIError({ error: error as APIError }),
})
}).pipe(
Metric.trackDuration(metrics.duration),
Metric.trackAll(metrics.calls, 1n),
Effect.withSpan("OpenAI.call"),
)

const generateTitle = (prompt: string) =>
Effect.flatMap(
call((_, signal) =>
_.chat.completions.create(
{
model: "gpt-4-turbo-preview",
messages: [
{
role: "user",
content: `Create a short title summarizing the following text:
call((_, signal) =>
_.chat.completions.create(
{
model: "gpt-4-turbo-preview",
messages: [
{
role: "user",
content: `Create a short title summarizing the following text:
${Str.truncateWords(prompt, 75)}`,
},
],
temperature: 0.25,
max_tokens: 64,
top_p: 1,
frequency_penalty: 0,
presence_penalty: 0,
},
{ signal },
),
},
],
temperature: 0.25,
max_tokens: 64,
top_p: 1,
frequency_penalty: 0,
presence_penalty: 0,
},
{ signal },
),
_ =>
).pipe(
Effect.flatMap(_ =>
pipe(
Option.fromNullable(_.choices[0]?.message?.content),
Option.map(cleanTitle),
),
),
Effect.withSpan("OpenAI.generateTitle"),
)

const generateReply = (title: string, messages: ReadonlyArray<Message>) =>
Expand Down
1 change: 1 addition & 0 deletions src/Reminders.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,7 @@ const make = Effect.gen(function* (_) {
),
Effect.asUnit,
Effect.retry(createThreadPolicy),
Effect.withSpan("Reminders.createThread", { attributes: { message } }),
)

yield* _(
Expand Down
68 changes: 46 additions & 22 deletions src/Summarizer.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ const make = Effect.gen(function* (_) {
Effect.flatMap(({ parentChannel, messages }) =>
summarize(parentChannel, channel, messages, small),
),
Effect.withSpan("Summarizer.summarizeThread"),
)

const summarizeWithMessages = (
Expand All @@ -53,6 +54,7 @@ const make = Effect.gen(function* (_) {
Effect.flatMap(parentChannel =>
summarize(parentChannel, channel, messages, small),
),
Effect.withSpan("Summarizer.summarizeWithMessages"),
)

const summarize = (
Expand All @@ -61,32 +63,34 @@ const make = Effect.gen(function* (_) {
messages: Chunk.Chunk<Discord.Message>,
small: boolean,
) =>
Effect.map(
Effect.forEach(
messages,
(message, index) => {
const reply = pipe(
Option.fromNullable(message.message_reference),
Option.flatMap(ref =>
Chunk.findFirstIndex(messages, _ => _.id === ref.message_id),
),
Option.map(
index => [Chunk.unsafeGet(messages, index), index + 1] as const,
),
)
return summarizeMessage(thread, index + 1, message, reply, small)
},
{ concurrency: "unbounded" },
),
messageContent =>
`# ${thread.name}
Effect.forEach(
messages,
(message, index) => {
const reply = pipe(
Option.fromNullable(message.message_reference),
Option.flatMap(ref =>
Chunk.findFirstIndex(messages, _ => _.id === ref.message_id),
),
Option.map(
index => [Chunk.unsafeGet(messages, index), index + 1] as const,
),
)
return summarizeMessage(thread, index + 1, message, reply, small)
},
{ concurrency: "unbounded" },
).pipe(
Effect.map(
messageContent =>
`# ${thread.name}
Thread started in: #${channel.name}<br />
Thread started at: ${new Date(
thread.thread_metadata!.create_timestamp!,
).toUTCString()}
thread.thread_metadata!.create_timestamp!,
).toUTCString()}
${messageContent.join("\n\n")}`,
),
Effect.withSpan("Summarizer.summarize"),
)

const summarizeMessage = (
Expand Down Expand Up @@ -123,7 +127,14 @@ ${messageContent.join("\n\n")}`,

return `${header}<br />
${message.content}${imagesContent}`
})
}).pipe(
Effect.withSpan("Summarizer.summarizeMessage", {
attributes: {
channelId: thread.id,
messageId: message.id,
},
}),
)

const followUpResponse = (
context: Discord.Interaction,
Expand Down Expand Up @@ -156,6 +167,12 @@ ${message.content}${imagesContent}`
"\n```",
}),
),
Effect.withSpan("Summarizer.followUpResponse", {
attributes: {
channelId: channel.id,
small,
},
}),
)

const command = Ix.global(
Expand Down Expand Up @@ -183,6 +200,12 @@ ${message.content}${imagesContent}`
Effect.bind("channel", ({ context }) =>
channels.get(context.guild_id!, context.channel_id!),
),
Effect.tap(({ channel, small }) =>
Effect.annotateCurrentSpan({
channelId: channel.id,
small,
}),
),
Effect.filterOrFail(
({ channel }) => channel.type === Discord.ChannelType.PUBLIC_THREAD,
() => new NotInThreadError(),
Expand All @@ -199,6 +222,7 @@ ${message.content}${imagesContent}`
},
}),
),
Effect.withSpan("Summarizer.command"),
),
)

Expand Down
Loading

0 comments on commit 54c1b8c

Please sign in to comment.