From 14396bda54ea68c352d6c3ee538f58eebd43d374 Mon Sep 17 00:00:00 2001 From: James Grugett Date: Sun, 1 Sep 2024 16:48:58 -0700 Subject: [PATCH 1/4] Remove tools --- backend/src/main-prompt.ts | 18 +++++------------- backend/src/request-files-prompt.ts | 7 ------- 2 files changed, 5 insertions(+), 20 deletions(-) diff --git a/backend/src/main-prompt.ts b/backend/src/main-prompt.ts index ff7800bea4..5f139e8e06 100644 --- a/backend/src/main-prompt.ts +++ b/backend/src/main-prompt.ts @@ -8,7 +8,6 @@ import { promptClaudeStream } from './claude' import { ProjectFileContext } from 'common/util/file' import { getSystemPrompt } from './system-prompt' import { STOP_MARKER } from 'common/constants' -import { getTools } from './tools' import { Message } from 'common/actions' import { ToolCall } from 'common/actions' import { debugLog } from './util/debug' @@ -34,7 +33,6 @@ export async function mainPrompt( ) let fullResponse = '' - const tools = getTools() let shouldCheckFiles = true if (Object.keys(fileContext.files).length === 0) { @@ -45,7 +43,7 @@ export async function mainPrompt( const responseChunk = await updateFileContext( ws, fileContext, - { messages, system, tools }, + { messages, system }, null, onResponseChunk, userId @@ -82,11 +80,10 @@ ${STOP_MARKER} ? [...messages, ...continuedMessages] : messages - savePromptLengthInfo(messagesWithContinuedMessage, system, tools) + savePromptLengthInfo(messagesWithContinuedMessage, system) const stream = promptClaudeStream(messagesWithContinuedMessage, { system, - tools, userId, }) const fileStream = processStreamWithFiles( @@ -133,7 +130,6 @@ ${STOP_MARKER} { messages, system, - tools, }, fileContext, toolCall.input['prompt'], @@ -193,18 +189,16 @@ async function updateFileContext( { messages, system, - tools, }: { messages: Message[] system: string | Array - tools: Tool[] }, prompt: string | null, onResponseChunk: (chunk: string) => void, userId: string ) { const relevantFiles = await requestRelevantFiles( - { messages, system, tools }, + { messages, system }, fileContext, prompt, userId @@ -256,8 +250,7 @@ export async function processFileBlock( const savePromptLengthInfo = ( messages: Message[], - system: string | Array, - tools: Tool[] + system: string | Array ) => { console.log('Prompting claude num messages:', messages.length) debugLog('Prompting claude num messages:', messages.length) @@ -270,8 +263,7 @@ const savePromptLengthInfo = ( typeof lastMessageContent === 'string' ? lastMessageContent : '[object]', messages: JSON.stringify(messages).length, system: system.length, - tools: JSON.stringify(tools).length, - timestamp: new Date().toISOString(), // Add a timestamp for each entry + timestamp: new Date().toISOString(), } debugLog(JSON.stringify(promptDebugInfo)) diff --git a/backend/src/request-files-prompt.ts b/backend/src/request-files-prompt.ts index e33dcc1e27..af4e42a864 100644 --- a/backend/src/request-files-prompt.ts +++ b/backend/src/request-files-prompt.ts @@ -12,11 +12,9 @@ export async function requestRelevantFiles( { messages, system, - tools, }: { messages: Message[] system: string | Array - tools: Tool[] }, fileContext: ProjectFileContext, assistantPrompt: string | null, @@ -50,7 +48,6 @@ export async function requestRelevantFiles( { messages: messagesExcludingLastIfByUser, system, - tools, }, nonObviousPrompt, models.sonnet, @@ -78,7 +75,6 @@ export async function requestRelevantFiles( { messages: messagesExcludingLastIfByUser, system, - tools, }, keyPrompt, models.sonnet, @@ -105,11 +101,9 @@ async function getRelevantFiles( { messages, system, - tools, }: { messages: Message[] system: string | Array - tools: Tool[] }, userPrompt: string, model: model_types, @@ -127,7 +121,6 @@ async function getRelevantFiles( const response = await promptClaude(messagesWithPrompt, { model, system, - tools, userId, }) const end = performance.now() From 5e201ab05db125cb0087e94a5341e81d3fe06366 Mon Sep 17 00:00:00 2001 From: James Grugett Date: Mon, 2 Sep 2024 08:22:05 -0700 Subject: [PATCH 2/4] Layers! --- backend/package.json | 2 +- backend/src/claude.ts | 6 +- backend/src/layer-1.ts | 51 ++++ backend/src/layer-2.ts | 100 ++++++ backend/src/layer-3.ts | 217 +++++++++++++ backend/src/layer-chooser.ts | 18 ++ backend/src/main-prompt.ts | 337 +++++---------------- backend/src/request-files-prompt.ts | 44 ++- backend/src/websockets/websocket-action.ts | 40 ++- bun.lockb | Bin 77912 -> 77912 bytes common/src/actions.ts | 3 + common/src/util/object.ts | 14 +- 12 files changed, 547 insertions(+), 285 deletions(-) create mode 100644 backend/src/layer-1.ts create mode 100644 backend/src/layer-2.ts create mode 100644 backend/src/layer-3.ts create mode 100644 backend/src/layer-chooser.ts diff --git a/backend/package.json b/backend/package.json index ca65de519c..6b318216e2 100644 --- a/backend/package.json +++ b/backend/package.json @@ -17,7 +17,7 @@ "test:watch": "bun test --watch" }, "dependencies": { - "@anthropic-ai/sdk": "^0.26.1", + "@anthropic-ai/sdk": "0.27.1", "diff": "5.2.0", "dotenv": "16.4.5", "express": "4.19.2", diff --git a/backend/src/claude.ts b/backend/src/claude.ts index f270008404..c1a6d029a2 100644 --- a/backend/src/claude.ts +++ b/backend/src/claude.ts @@ -13,12 +13,15 @@ export const models = { export type model_types = (typeof models)[keyof typeof models] +export type System = string | Array + export const promptClaudeStream = async function* ( messages: Message[], options: { - system?: string | Array + system?: System tools?: Tool[] model?: model_types + maxTokens?: number userId: string } ): AsyncGenerator { @@ -100,6 +103,7 @@ export const promptClaude = async ( system?: string | Array tools?: Tool[] model?: model_types + maxTokens?: number userId: string } ) => { diff --git a/backend/src/layer-1.ts b/backend/src/layer-1.ts new file mode 100644 index 0000000000..3e3f94a930 --- /dev/null +++ b/backend/src/layer-1.ts @@ -0,0 +1,51 @@ +import { WebSocket } from 'ws' + +import { Message } from 'common/actions' +import { getSystemPrompt } from './system-prompt' +import { ProjectFileContext } from 'common/util/file' +import { requestRelevantFiles } from './request-files-prompt' +import { assert } from 'common/util/object' + +export const layer1 = async ( + ws: WebSocket, + userId: string, + messages: Message[], + fileContext: ProjectFileContext +) => { + const lastMessage = messages[messages.length - 1] + assert(lastMessage.role === 'user', 'Last message must be from user') + + const system = getSystemPrompt(fileContext, { + checkFiles: false, + }) + + const files = await requestRelevantFiles( + ws, + { messages, system }, + fileContext, + null, + userId + ) + + const filePaths = Object.keys(files) + const responseChunk = getRelevantFileInfoMessage(filePaths) + + return { responseChunk, files } +} + +function getRelevantFileInfoMessage(filePaths: string[]) { + if (filePaths.length === 0) { + return '' + } + return `Reading the following files...${filePaths.join(', ')}\n\n` +} + +const skipToLayer3 = (ws: WebSocket, userId: string, messages: Message[], fileContext: ProjectFileContext) => { + const lastMessage = messages[messages.length - 1] + assert(lastMessage.role === 'user', 'Last message must be from user') + + const system = getSystemPrompt(fileContext, { + checkFiles: false, + }) + +} \ No newline at end of file diff --git a/backend/src/layer-2.ts b/backend/src/layer-2.ts new file mode 100644 index 0000000000..1850f4f04d --- /dev/null +++ b/backend/src/layer-2.ts @@ -0,0 +1,100 @@ +import { WebSocket } from 'ws' +import { Message } from 'common/actions' +import { ProjectFileContext } from 'common/util/file' +import { getSystemPrompt } from './system-prompt' +import { promptClaudeStream, System } from './claude' +import { assert } from 'common/util/object' +import { requestRelevantFiles } from './request-files-prompt' + +export const layer2 = async ( + ws: WebSocket, + userId: string, + messages: Message[], + fileContext: ProjectFileContext, + onResponseChunk: (chunk: string) => void +) => { + const lastMessage = messages[messages.length - 1] + assert( + lastMessage.role === 'user' && typeof lastMessage.content === 'string', + 'Last message must be from user and must be a string ' + + `(got ${lastMessage.role} with content type ${typeof lastMessage.content})` + ) + const userMessage = lastMessage.content + const previousMessages = messages.slice(0, -1) + + const system = getSystemPrompt(fileContext, { + checkFiles: true, + }) + + const [codeReviewResponse, brainstormResponse, files] = await Promise.all([ + codeReviewPrompt(userId, system, previousMessages, userMessage), + brainstormPrompt( + userId, + system, + previousMessages, + userMessage, + onResponseChunk + ), + requestRelevantFiles(ws, { messages, system }, fileContext, null, userId), + ]) + + return { + codeReviewResponse, + brainstormResponse, + files, + } +} + +const codeReviewPrompt = async ( + userId: string, + system: System, + previousMessages: Message[], + userMessage: string +) => { + const prompt = ` +${userMessage} + +Please review the files and provide a detailed analysis of the code, especially as it relates to the user's request. +`.trim() + + const stream = promptClaudeStream( + [...previousMessages, { role: 'user', content: prompt }], + { + system, + userId, + } + ) + let fullResponse = '' + for await (const chunk of stream) { + fullResponse += chunk + } + return fullResponse +} + +const brainstormPrompt = async ( + userId: string, + system: System, + previousMessages: Message[], + userMessage: string, + onResponseChunk: (chunk: string) => void +) => { + const prompt = ` +${userMessage} + +Please brainstorm ideas to solve the user's request. +`.trim() + + const stream = promptClaudeStream( + [...previousMessages, { role: 'user', content: prompt }], + { + system, + userId, + } + ) + let fullResponse = '' + for await (const chunk of stream) { + fullResponse += chunk + onResponseChunk(chunk as string) + } + return fullResponse +} diff --git a/backend/src/layer-3.ts b/backend/src/layer-3.ts new file mode 100644 index 0000000000..6c085a2299 --- /dev/null +++ b/backend/src/layer-3.ts @@ -0,0 +1,217 @@ +import fs from 'fs' +import path from 'path' +import { WebSocket } from 'ws' +import { createPatch } from 'diff' + +import { Message, ToolCall } from 'common/actions' +import { STOP_MARKER } from 'common/constants' +import { ProjectFileContext } from 'common/util/file' +import { promptClaudeStream, System } from './claude' +import { processStreamWithFiles } from './process-stream' +import { requestRelevantFilesPrompt } from './request-files-prompt' +import { getSystemPrompt } from './system-prompt' +import { debugLog } from './util/debug' +import { generatePatch } from './generate-patch' +import { requestFile } from './websockets/websocket-action' + +export const layer3 = async ( + ws: WebSocket, + userId: string, + messages: Message[], + fileContext: ProjectFileContext, + extraFiles: Record, + onResponseChunk: (chunk: string) => void +) => { + let fullResponse = '' + const fileProcessingPromises: Promise[] = [] + let toolCall: ToolCall | null = null + let continuedMessages: Message[] = [] + let isComplete = false + let iterationCount = 0 + const MAX_ITERATIONS = 10 + + const lastMessage = messages[messages.length - 1] + if (lastMessage.role === 'user' && typeof lastMessage.content === 'string') { + lastMessage.content = `${lastMessage.content} + + +Please preserve as much of the existing code, its comments, and its behavior as possible. Make minimal edits to accomplish only the core of what is requested. Then pause to get more instructions from the user. + + +Always end your response with the following marker: +${STOP_MARKER} +` + } + + while (!isComplete && iterationCount < MAX_ITERATIONS) { + const system = getSystemPrompt(fileContext, { + checkFiles: false, + }) + const messagesWithContinuedMessage = continuedMessages + ? [...messages, ...continuedMessages] + : messages + + savePromptLengthInfo(messagesWithContinuedMessage, system) + + const stream = promptClaudeStream(messagesWithContinuedMessage, { + system, + userId, + }) + const fileStream = processStreamWithFiles( + stream, + (_filePath) => { + onResponseChunk('Modifying...') + }, + (filePath, fileContent) => { + console.log('on file!', filePath) + fileProcessingPromises.push( + processFileBlock( + userId, + ws, + messages, + fullResponse, + filePath, + fileContent + ).catch((error) => { + console.error('Error processing file block', error) + return '' + }) + ) + } + ) + + for await (const chunk of fileStream) { + if (typeof chunk === 'object') { + toolCall = chunk + debugLog('Received tool call:', toolCall) + continue + } + + fullResponse += chunk + onResponseChunk(chunk) + } + + if (fullResponse.includes(STOP_MARKER)) { + isComplete = true + fullResponse = fullResponse.replace(STOP_MARKER, '') + debugLog('Reached STOP_MARKER') + } else if (toolCall) { + if (toolCall.name === 'update_file_context') { + const relevantFiles = await requestRelevantFilesPrompt( + { + messages, + system, + }, + fileContext, + toolCall.input['prompt'], + userId + ) + const responseChunk = '\n' + getRelevantFileInfoMessage(relevantFiles) + onResponseChunk(responseChunk) + fullResponse += responseChunk + } + isComplete = true + } else { + console.log('continuing to generate') + debugLog('continuing to generate') + const fullResponseMinusLastLine = + fullResponse.split('\n').slice(0, -1).join('\n') + '\n' + continuedMessages = [ + { + role: 'assistant', + content: fullResponseMinusLastLine, + }, + { + role: 'user', + content: `You got cut off, but please continue from the very next line of your response. Do not repeat anything you have just said. Just continue as if there were no interruption from the very last character of your last response. (Alternatively, just end your response with the following marker if you were done generating and want to allow the user to give further guidance: ${STOP_MARKER})`, + }, + ] + } + + iterationCount++ + } + + if (iterationCount >= MAX_ITERATIONS) { + console.log('Reached maximum number of iterations in mainPrompt') + debugLog('Reached maximum number of iterations in mainPrompt') + } + + const changes = (await Promise.all(fileProcessingPromises)).filter( + (change) => change !== '' + ) + + return { + changes, + response: fullResponse, + } +} + +export async function processFileBlock( + userId: string, + ws: WebSocket, + messageHistory: Message[], + fullResponse: string, + filePath: string, + newContent: string +) { + debugLog('Processing file block', filePath) + + const oldContent = await requestFile(ws, filePath) + + if (oldContent === null) { + console.log(`Created new file: ${filePath}`) + debugLog(`Created new file: ${filePath}`) + return createPatch(filePath, '', newContent) + } + + const patch = await generatePatch( + userId, + oldContent, + newContent, + filePath, + messageHistory, + fullResponse + ) + console.log(`Generated patch for file: ${filePath}`) + debugLog(`Generated patch for file: ${filePath}`) + return patch +} + +function getRelevantFileInfoMessage(filePaths: string[]) { + if (filePaths.length === 0) { + return '' + } + return `Reading the following files...${filePaths.join(', ')}\n\n` +} + +const savePromptLengthInfo = (messages: Message[], system: System) => { + console.log('Prompting claude num messages:', messages.length) + debugLog('Prompting claude num messages:', messages.length) + + const lastMessageContent = messages[messages.length - 1].content + + // Save prompt debug information to a JSON array + const promptDebugInfo = { + input: + typeof lastMessageContent === 'string' ? lastMessageContent : '[object]', + messages: JSON.stringify(messages).length, + system: system.length, + timestamp: new Date().toISOString(), + } + + debugLog(JSON.stringify(promptDebugInfo)) + + const debugFilePath = path.join(__dirname, 'prompt.debug.json') + + let debugArray = [] + try { + const existingData = fs.readFileSync(debugFilePath, 'utf8') + debugArray = JSON.parse(existingData) + } catch (error) { + // If file doesn't exist or is empty, start with an empty array + } + + debugArray.push(promptDebugInfo) + + fs.writeFileSync(debugFilePath, JSON.stringify(debugArray, null, 2)) +} diff --git a/backend/src/layer-chooser.ts b/backend/src/layer-chooser.ts new file mode 100644 index 0000000000..1d0d0992b6 --- /dev/null +++ b/backend/src/layer-chooser.ts @@ -0,0 +1,18 @@ +import { Message } from 'common/actions' +import { promptClaudeStream, System } from './claude' + +export const chooseLayer = async ( + userId: string, + system: System, + messages: Message[], +) => { + const stream = promptClaudeStream(messages, { + system, + userId, + }) + let fullResponse = '' + for await (const chunk of stream) { + fullResponse += chunk + } + return fullResponse +} \ No newline at end of file diff --git a/backend/src/main-prompt.ts b/backend/src/main-prompt.ts index 5f139e8e06..4d52d3946e 100644 --- a/backend/src/main-prompt.ts +++ b/backend/src/main-prompt.ts @@ -1,24 +1,14 @@ import { WebSocket } from 'ws' -import fs from 'fs' -import path from 'path' -import { TextBlockParam, Tool } from '@anthropic-ai/sdk/resources' -import { createPatch } from 'diff' -import { promptClaudeStream } from './claude' import { ProjectFileContext } from 'common/util/file' -import { getSystemPrompt } from './system-prompt' -import { STOP_MARKER } from 'common/constants' import { Message } from 'common/actions' -import { ToolCall } from 'common/actions' import { debugLog } from './util/debug' -import { requestFiles, requestFile } from './websockets/websocket-action' -import { generatePatch } from './generate-patch' -import { requestRelevantFiles } from './request-files-prompt' -import { processStreamWithFiles } from './process-stream' +import { layer1 } from './layer-1' +import { layer2 } from './layer-2' +import { layer3 } from './layer-3' +import { getSystemPrompt } from './system-prompt' +import { models, promptClaude } from './claude' -/** - * Prompt claude, handle tool calls, and generate file changes. - */ export async function mainPrompt( ws: WebSocket, messages: Message[], @@ -32,253 +22,84 @@ export async function mainPrompt( messages.length ) - let fullResponse = '' - - let shouldCheckFiles = true - if (Object.keys(fileContext.files).length === 0) { - const system = getSystemPrompt(fileContext, { - checkFiles: true, - }) - // If the fileContext.files is empty, use prompts to select files and add them to context. - const responseChunk = await updateFileContext( - ws, - fileContext, - { messages, system }, - null, - onResponseChunk, - userId - ) - fullResponse += responseChunk - shouldCheckFiles = false - } - - const lastMessage = messages[messages.length - 1] - const fileProcessingPromises: Promise[] = [] - let toolCall: ToolCall | null = null - let continuedMessages: Message[] = [] - let isComplete = false - let iterationCount = 0 - const MAX_ITERATIONS = 10 - - if (lastMessage.role === 'user' && typeof lastMessage.content === 'string') { - lastMessage.content = `${lastMessage.content} - - -Please preserve as much of the existing code, its comments, and its behavior as possible. Make minimal edits to accomplish only the core of what is requested. Then pause to get more instructions from the user. - - -Always end your response with the following marker: -${STOP_MARKER} -` - } - - while (!isComplete && iterationCount < MAX_ITERATIONS) { - const system = getSystemPrompt(fileContext, { - checkFiles: shouldCheckFiles, - }) - const messagesWithContinuedMessage = continuedMessages - ? [...messages, ...continuedMessages] - : messages - - savePromptLengthInfo(messagesWithContinuedMessage, system) - - const stream = promptClaudeStream(messagesWithContinuedMessage, { - system, - userId, - }) - const fileStream = processStreamWithFiles( - stream, - (_filePath) => { - onResponseChunk('Modifying...') - }, - (filePath, fileContent) => { - console.log('on file!', filePath) - fileProcessingPromises.push( - processFileBlock( - userId, - ws, - messages, - fullResponse, - filePath, - fileContent - ).catch((error) => { - console.error('Error processing file block', error) - return '' - }) - ) - } - ) - - for await (const chunk of fileStream) { - if (typeof chunk === 'object') { - toolCall = chunk - debugLog('Received tool call:', toolCall) - continue - } - - fullResponse += chunk - onResponseChunk(chunk) - } + let layer = Object.keys(fileContext.files).length === 0 ? 1 : 2 - if (fullResponse.includes(STOP_MARKER)) { - isComplete = true - fullResponse = fullResponse.replace(STOP_MARKER, '') - debugLog('Reached STOP_MARKER') - } else if (toolCall) { - if (toolCall.name === 'update_file_context') { - const relevantFiles = await requestRelevantFiles( - { - messages, - system, - }, - fileContext, - toolCall.input['prompt'], - userId - ) - const responseChunk = '\n' + getRelevantFileInfoMessage(relevantFiles) - onResponseChunk(responseChunk) - fullResponse += responseChunk - } - isComplete = true - } else { - console.log('continuing to generate') - debugLog('continuing to generate') - const fullResponseMinusLastLine = - fullResponse.split('\n').slice(0, -1).join('\n') + '\n' - continuedMessages = [ - { - role: 'assistant', - content: fullResponseMinusLastLine, - }, + let fullResponse = '' + const extraFiles: Record = {} + + while (true) { + if (layer === 1) { + const { files, responseChunk } = await layer1( + ws, + userId, + messages, + fileContext + ) + const lastMessage = messages[messages.length - 1] + const input = lastMessage.content as string + lastMessage.content = [ { - role: 'user', - content: `You got cut off, but please continue from the very next line of your response. Do not repeat anything you have just said. Just continue as if there were no interruption from the very last character of your last response. (Alternatively, just end your response with the following marker if you were done generating and want to allow the user to give further guidance: ${STOP_MARKER})`, + type: 'text', + text: input, + cache_control: { type: 'ephemeral' as const }, }, ] - } - - iterationCount++ - } - - if (iterationCount >= MAX_ITERATIONS) { - console.log('Reached maximum number of iterations in mainPrompt') - debugLog('Reached maximum number of iterations in mainPrompt') - } - - const changes = (await Promise.all(fileProcessingPromises)).filter( - (change) => change !== '' - ) - - return { - response: fullResponse, - changes, - toolCall, - } -} - -function getRelevantFileInfoMessage(filePaths: string[]) { - if (filePaths.length === 0) { - return '' - } - return `Reading the following files...${filePaths.join(', ')}\n\n` -} - -async function updateFileContext( - ws: WebSocket, - fileContext: ProjectFileContext, - { - messages, - system, - }: { - messages: Message[] - system: string | Array - }, - prompt: string | null, - onResponseChunk: (chunk: string) => void, - userId: string -) { - const relevantFiles = await requestRelevantFiles( - { messages, system }, - fileContext, - prompt, - userId - ) - - if (relevantFiles.length === 0) { - return '' - } - - const responseChunk = getRelevantFileInfoMessage(relevantFiles) - onResponseChunk(responseChunk) - - // Load relevant files into fileContext - fileContext.files = await requestFiles(ws, relevantFiles) - - return responseChunk -} - -export async function processFileBlock( - userId: string, - ws: WebSocket, - messageHistory: Message[], - fullResponse: string, - filePath: string, - newContent: string -) { - debugLog('Processing file block', filePath) - - const oldContent = await requestFile(ws, filePath) - if (oldContent === null) { - console.log(`Created new file: ${filePath}`) - debugLog(`Created new file: ${filePath}`) - return createPatch(filePath, '', newContent) - } - - const patch = await generatePatch( - userId, - oldContent, - newContent, - filePath, - messageHistory, - fullResponse - ) - console.log(`Generated patch for file: ${filePath}`) - debugLog(`Generated patch for file: ${filePath}`) - return patch -} - -const savePromptLengthInfo = ( - messages: Message[], - system: string | Array -) => { - console.log('Prompting claude num messages:', messages.length) - debugLog('Prompting claude num messages:', messages.length) - - const lastMessageContent = messages[messages.length - 1].content - - // Save prompt debug information to a JSON array - const promptDebugInfo = { - input: - typeof lastMessageContent === 'string' ? lastMessageContent : '[object]', - messages: JSON.stringify(messages).length, - system: system.length, - timestamp: new Date().toISOString(), - } - - debugLog(JSON.stringify(promptDebugInfo)) - - const debugFilePath = path.join(__dirname, 'prompt.debug.json') - - let debugArray = [] - try { - const existingData = fs.readFileSync(debugFilePath, 'utf8') - debugArray = JSON.parse(existingData) - } catch (error) { - // If file doesn't exist or is empty, start with an empty array + fileContext.files = files + fullResponse += responseChunk + layer = 2 + + const system = getSystemPrompt(fileContext) + await promptClaude(messages, { + model: models.sonnet, + system, + userId, + maxTokens: 0, + }) + } else if (layer === 2) { + // TODO: A way to loop back to layer 2. + const { files, codeReviewResponse, brainstormResponse } = await layer2( + ws, + userId, + messages, + fileContext, + onResponseChunk + ) + Object.assign(extraFiles, files) + fullResponse += brainstormResponse + + const prefilledResponse = ` + +${extraFiles} + + +${codeReviewResponse} + + + +${brainstormResponse} + + `.trim() + + messages.push({ + role: 'assistant', + content: prefilledResponse, + }) + + layer = 3 + } else if (layer === 3) { + const { changes, response } = await layer3( + ws, + userId, + messages, + fileContext, + extraFiles, + onResponseChunk + ) + return { + response: fullResponse, + changes, + } + } } - - debugArray.push(promptDebugInfo) - - fs.writeFileSync(debugFilePath, JSON.stringify(debugArray, null, 2)) } diff --git a/backend/src/request-files-prompt.ts b/backend/src/request-files-prompt.ts index af4e42a864..68a7e349c3 100644 --- a/backend/src/request-files-prompt.ts +++ b/backend/src/request-files-prompt.ts @@ -1,20 +1,44 @@ import { range, shuffle, uniq } from 'lodash' import { dirname } from 'path' +import { WebSocket } from 'ws' import { Message } from 'common/actions' import { ProjectFileContext } from 'common/util/file' -import { model_types, models, promptClaude } from './claude' +import { model_types, models, promptClaude, System } from './claude' import { debugLog } from './util/debug' -import { TextBlockParam, Tool } from '@anthropic-ai/sdk/resources' +import { TextBlockParam } from '@anthropic-ai/sdk/resources' import { getAllFilePaths } from 'common/project-file-tree' +import { requestFiles } from './websockets/websocket-action' export async function requestRelevantFiles( + ws: WebSocket, { messages, system, }: { messages: Message[] - system: string | Array + system: System + }, + fileContext: ProjectFileContext, + assistantPrompt: string | null, + userId: string +) { + const filePaths = await requestRelevantFilesPrompt( + { messages, system }, + fileContext, + assistantPrompt, + userId + ) + return await requestFiles(ws, filePaths) +} + +export async function requestRelevantFilesPrompt( + { + messages, + system, + }: { + messages: Message[] + system: System }, fileContext: ProjectFileContext, assistantPrompt: string | null, @@ -39,6 +63,7 @@ export async function requestRelevantFiles( userPrompt, assistantPrompt, fileContext, + previousFiles, countPerRequest, index * 2 - 1 ) @@ -65,6 +90,7 @@ export async function requestRelevantFiles( userPrompt, assistantPrompt, fileContext, + previousFiles, index * 2 - 1, countPerRequest ) @@ -171,6 +197,7 @@ function generateNonObviousRequestFilesPrompt( userPrompt: string | null, assistantPrompt: string | null, fileContext: ProjectFileContext, + previousFiles: string[], count: number, index: number ): string { @@ -200,6 +227,11 @@ Please follow these steps to determine which files to request: 5. Try to list exactly ${count} files. Do not include any files with 'knowledge.md' in the name, because these files will be included by default. +${ + previousFiles.length > 0 + ? `Do not include any of the following files that have already been requested in previous instructions:\n${previousFiles.join('\n')}` + : '' +} Please provide no commentary and list the file paths you think are useful but not obvious in addressing the user's request. @@ -224,6 +256,7 @@ function generateKeyRequestFilesPrompt( userPrompt: string | null, assistantPrompt: string | null, fileContext: ProjectFileContext, + previousFiles: string[], index: number, count: number ): string { @@ -254,6 +287,11 @@ Please follow these steps to determine which key files to request: 5. Order the files by most important first. Do not include any files with 'knowledge.md' in the name, because these files will be included by default. +${ + previousFiles.length > 0 + ? `Do not include any of the following files that have already been requested in previous instructions:\n${previousFiles.join('\n')}` + : '' +} Please provide no commentary and only list the file paths at index ${start} through ${end} of the most relevant files that you think are most crucial for addressing the user's request. diff --git a/backend/src/websockets/websocket-action.ts b/backend/src/websockets/websocket-action.ts index c13822f2e8..9cad53dc09 100644 --- a/backend/src/websockets/websocket-action.ts +++ b/backend/src/websockets/websocket-action.ts @@ -1,4 +1,6 @@ import { WebSocket } from 'ws' +import { Message as AnthropicMessage } from '@anthropic-ai/sdk/resources' + import { ClientMessage } from 'common/websockets/websocket-schema' import { mainPrompt } from '../main-prompt' import { ClientAction, ServerAction } from 'common/actions' @@ -32,7 +34,7 @@ const onUserInput = async ( console.log('Input:', lastMessage) try { - const { toolCall, response, changes } = await mainPrompt( + const { response, changes } = await mainPrompt( ws, messages, fileContext, @@ -46,24 +48,22 @@ const onUserInput = async ( ) const allChanges = [...previousChanges, ...changes] - if (toolCall) { - console.log('toolCall', toolCall.name, toolCall.input) - sendAction(ws, { - type: 'tool-call', - userInputId, - response, - data: toolCall, - changes: allChanges, - }) - } else { - console.log('response-complete') - sendAction(ws, { - type: 'response-complete', - userInputId, - response, - changes: allChanges, - }) - } + // if (toolCall) { + // console.log('toolCall', toolCall.name, toolCall.input) + // sendAction(ws, { + // type: 'tool-call', + // userInputId, + // response, + // data: toolCall, + // changes: allChanges, + // }) + console.log('response-complete') + sendAction(ws, { + type: 'response-complete', + userInputId, + response, + changes: allChanges, + }) } catch (e) { console.error('Error in mainPrompt', e) const response = @@ -117,7 +117,6 @@ const onWarmContextCache = async ( ws: WebSocket ) => { const startTime = Date.now() - const tools = getTools() const system = getSystemPrompt(fileContext, { onlyCachePrefix: true }) await promptClaude( [ @@ -129,7 +128,6 @@ const onWarmContextCache = async ( { model: models.sonnet, system, - tools, userId: fingerprintId, } ) diff --git a/bun.lockb b/bun.lockb index 88a4e94893e11de1b692b91e172825fe2a157f25..8355c34bb9a16936ea605ef37f6a9a2e0569fdcd 100755 GIT binary patch delta 174 zcmccdfaS&mmI-j>i~G(%E(~t$w4QngOHq zWJ6Ky$qBq1n^zd9`ZzEwjQ!)!Zk@M}=c#|^ESHJT%k6UKSMbF?Vi&&=&Kb8qzrs>H zXYt-kQs1~Ym-*~_X47mi+s|a`)u$WU0v^ZBmD|i)yETd3K+nir&yZpB$@}fLjON<| aWf|QC8I2|@@~TfipvPzdVQyE@2QmTXAVqip delta 174 zcmV;f08#(g-~`y<1duKu@fzg<=2+y9Cv2C*f2BKwa4?!*0Opqphc?AX5eDe$6kWD%I`zQn4#-Eo* zs78l4`zn#F-~KHiU5GYbQt`!_-|DphYP71)vkiT(obj: T, predicate: (value: any, k } } return result -} \ No newline at end of file +} + +/** + * Asserts that a condition is true. If the condition is false, it throws an error with the provided message. + * @param condition The condition to check + * @param message The error message to display if the condition is false + * @throws {Error} If the condition is false + */ +export function assert(condition: boolean, message: string): asserts condition { + if (!condition) { + throw new Error(`Assertion failed: ${message}`); + } +} From 70c1867de020324f51c49c2bd43342517a41630c Mon Sep 17 00:00:00 2001 From: James Grugett Date: Mon, 2 Sep 2024 16:55:37 -0700 Subject: [PATCH 3/4] Working 3 layers! --- backend/src/claude.ts | 4 +- backend/src/layer-1.ts | 22 ++-- backend/src/layer-2.ts | 129 +++++++++++++++---- backend/src/layer-3.ts | 8 +- backend/src/main-prompt.ts | 129 ++++++++++++------- backend/src/request-files-prompt.ts | 88 +++++++++++++ backend/src/system-prompt.ts | 138 ++++++++++----------- backend/src/websockets/websocket-action.ts | 3 +- 8 files changed, 363 insertions(+), 158 deletions(-) diff --git a/backend/src/claude.ts b/backend/src/claude.ts index c1a6d029a2..ca410f68e4 100644 --- a/backend/src/claude.ts +++ b/backend/src/claude.ts @@ -25,7 +25,7 @@ export const promptClaudeStream = async function* ( userId: string } ): AsyncGenerator { - const { model = models.sonnet, system, tools, userId } = options + const { model = models.sonnet, system, tools, userId, maxTokens } = options const apiKey = process.env.ANTHROPIC_API_KEY @@ -48,7 +48,7 @@ export const promptClaudeStream = async function* ( const stream = anthropic.messages.stream( removeUndefinedProps({ model, - max_tokens: 4096, + max_tokens: maxTokens ?? 4096, temperature: 0, messages, system, diff --git a/backend/src/layer-1.ts b/backend/src/layer-1.ts index 3e3f94a930..87143190cb 100644 --- a/backend/src/layer-1.ts +++ b/backend/src/layer-1.ts @@ -27,25 +27,19 @@ export const layer1 = async ( userId ) - const filePaths = Object.keys(files) - const responseChunk = getRelevantFileInfoMessage(filePaths) - - return { responseChunk, files } -} - -function getRelevantFileInfoMessage(filePaths: string[]) { - if (filePaths.length === 0) { - return '' - } - return `Reading the following files...${filePaths.join(', ')}\n\n` + return { files } } -const skipToLayer3 = (ws: WebSocket, userId: string, messages: Message[], fileContext: ProjectFileContext) => { +const skipToLayer3 = ( + ws: WebSocket, + userId: string, + messages: Message[], + fileContext: ProjectFileContext +) => { const lastMessage = messages[messages.length - 1] assert(lastMessage.role === 'user', 'Last message must be from user') const system = getSystemPrompt(fileContext, { checkFiles: false, }) - -} \ No newline at end of file +} diff --git a/backend/src/layer-2.ts b/backend/src/layer-2.ts index 1850f4f04d..a5a6c14f8b 100644 --- a/backend/src/layer-2.ts +++ b/backend/src/layer-2.ts @@ -4,7 +4,7 @@ import { ProjectFileContext } from 'common/util/file' import { getSystemPrompt } from './system-prompt' import { promptClaudeStream, System } from './claude' import { assert } from 'common/util/object' -import { requestRelevantFiles } from './request-files-prompt' +import { requestAdditionalFiles } from './request-files-prompt' export const layer2 = async ( ws: WebSocket, @@ -26,22 +26,38 @@ export const layer2 = async ( checkFiles: true, }) - const [codeReviewResponse, brainstormResponse, files] = await Promise.all([ - codeReviewPrompt(userId, system, previousMessages, userMessage), - brainstormPrompt( - userId, - system, - previousMessages, - userMessage, - onResponseChunk - ), - requestRelevantFiles(ws, { messages, system }, fileContext, null, userId), - ]) + const [files, codeReviewResponse, brainstormResponse, choosePlanInfo] = + await Promise.all([ + requestAdditionalFiles(ws, { messages, system }, fileContext, userId), + codeReviewPrompt(userId, system, previousMessages, userMessage).catch( + (error) => { + console.error('Error in code review prompt:', error) + return '' + } + ), + brainstormPrompt(userId, system, previousMessages, userMessage).catch( + (error) => { + console.error('Error in brainstorm prompt:', error) + return '' + } + ), + choosePlanPrompt( + userId, + system, + previousMessages, + userMessage, + onResponseChunk + ).catch((error) => { + console.error('Error in choose plan prompt:', error) + return { fullResponse: '', uncertaintyScore: 0, chosenPlan: 'PAUSE' } + }), + ]) return { + files, codeReviewResponse, brainstormResponse, - files, + choosePlanInfo, } } @@ -54,16 +70,19 @@ const codeReviewPrompt = async ( const prompt = ` ${userMessage} -Please review the files and provide a detailed analysis of the code, especially as it relates to the user's request. +Please review the files and provide a detailed analysis of the code within blocks, especially as it relates to the user's request. Then stop. `.trim() - const stream = promptClaudeStream( - [...previousMessages, { role: 'user', content: prompt }], - { - system, - userId, - } - ) + const messages = [ + ...previousMessages, + { role: 'user' as const, content: prompt }, + { role: 'assistant' as const, content: '' }, + ] + + const stream = promptClaudeStream(messages, { + system, + userId, + }) let fullResponse = '' for await (const chunk of stream) { fullResponse += chunk @@ -72,6 +91,38 @@ Please review the files and provide a detailed analysis of the code, especially } const brainstormPrompt = async ( + userId: string, + system: System, + previousMessages: Message[], + userMessage: string +) => { + const prompt = ` +${userMessage} + +Please brainstorm ideas to solve the user's request in a block. Try to list several independent ideas. Then stop. +`.trim() + + const messages = [ + ...previousMessages, + { role: 'user' as const, content: prompt }, + { role: 'assistant' as const, content: '' }, + ] + + const stream = promptClaudeStream(messages, { + system, + userId, + }) + let fullResponse = '' + for await (const chunk of stream) { + fullResponse += chunk + } + return fullResponse +} + +const possiblePlans = ['PROCEED', 'PAUSE', 'GATHER_MORE_INFO'] as const +type PossiblePlan = (typeof possiblePlans)[number] + +const choosePlanPrompt = async ( userId: string, system: System, previousMessages: Message[], @@ -79,9 +130,19 @@ const brainstormPrompt = async ( onResponseChunk: (chunk: string) => void ) => { const prompt = ` -${userMessage} +${userMessage} -Please brainstorm ideas to solve the user's request. +Please discuss how much uncertainty or ambiguity there is in fulfilling the user's request and knowing what plan they would like most. + +Then, write out an block that contains an uncertainty score between 0 (no ambiguity) and 100 (high ambiguity) that you know what the user wants and can implement the plan they would like most. + +Finally, we need to choose a plan to address the level of uncertainty. We can either: + +- PROCEED with a solution +- PAUSE to ask the user for more information, or +- GATHER_MORE_INFO by reading files or running commands + +Please write out a block that contains "PROCEED", "PAUSE", or "GATHER_MORE_INFO". `.trim() const stream = promptClaudeStream( @@ -96,5 +157,25 @@ Please brainstorm ideas to solve the user's request. fullResponse += chunk onResponseChunk(chunk as string) } - return fullResponse + const { uncertaintyScore, chosenPlan } = parseChoosePlanPrompt(fullResponse) + return { fullResponse, uncertaintyScore, chosenPlan } +} + +const parseChoosePlanPrompt = (response: string) => { + const uncertaintyScoreRegex = /(.*?)<\/uncertainty_score>/ + const chosenPlanRegex = /(.*?)<\/chosen_plan>/ + + const uncertaintyScoreMatch = response.match(uncertaintyScoreRegex) + const chosenPlanMatch = response.match(chosenPlanRegex) + + if (!uncertaintyScoreMatch || !chosenPlanMatch) { + throw new Error('Could not parse choose plan prompt') + } + const uncertaintyScore = parseInt(uncertaintyScoreMatch[1], 10) + const chosenPlanStr = chosenPlanMatch[1] as PossiblePlan + const chosenPlan = possiblePlans.includes(chosenPlanStr) + ? chosenPlanStr + : ('PAUSE' as const) + + return { uncertaintyScore, chosenPlan } } diff --git a/backend/src/layer-3.ts b/backend/src/layer-3.ts index 6c085a2299..9d01f0b970 100644 --- a/backend/src/layer-3.ts +++ b/backend/src/layer-3.ts @@ -19,13 +19,15 @@ export const layer3 = async ( userId: string, messages: Message[], fileContext: ProjectFileContext, - extraFiles: Record, onResponseChunk: (chunk: string) => void ) => { - let fullResponse = '' + // Prefill the response so that we talk directly to the user. + let fullResponse = `Based on the above discussion, I'll proceed to answer the user's request:` const fileProcessingPromises: Promise[] = [] let toolCall: ToolCall | null = null - let continuedMessages: Message[] = [] + let continuedMessages: Message[] = [ + { role: 'assistant', content: fullResponse }, + ] let isComplete = false let iterationCount = 0 const MAX_ITERATIONS = 10 diff --git a/backend/src/main-prompt.ts b/backend/src/main-prompt.ts index 4d52d3946e..8c1befeb0b 100644 --- a/backend/src/main-prompt.ts +++ b/backend/src/main-prompt.ts @@ -24,78 +24,94 @@ export async function mainPrompt( let layer = Object.keys(fileContext.files).length === 0 ? 1 : 2 + let printedResponse = '' let fullResponse = '' - const extraFiles: Record = {} + const continuedMessages: Message[] = [] while (true) { + console.log('layer', layer) + const messagesWithContinuedMessages = [...messages, ...continuedMessages] if (layer === 1) { - const { files, responseChunk } = await layer1( + const { files } = await layer1( ws, userId, - messages, + messagesWithContinuedMessages, fileContext ) - const lastMessage = messages[messages.length - 1] - const input = lastMessage.content as string - lastMessage.content = [ - { - type: 'text', - text: input, - cache_control: { type: 'ephemeral' as const }, - }, - ] + + const filesInfoMessage = getRelevantFileInfoMessage(files) + onResponseChunk(filesInfoMessage) fileContext.files = files - fullResponse += responseChunk + printedResponse += filesInfoMessage + fullResponse += filesInfoMessage layer = 2 - const system = getSystemPrompt(fileContext) - await promptClaude(messages, { - model: models.sonnet, - system, - userId, - maxTokens: 0, - }) + await warmCache(fileContext, messagesWithContinuedMessages, userId) } else if (layer === 2) { - // TODO: A way to loop back to layer 2. - const { files, codeReviewResponse, brainstormResponse } = await layer2( - ws, - userId, - messages, - fileContext, - onResponseChunk - ) - Object.assign(extraFiles, files) - fullResponse += brainstormResponse + const { files, codeReviewResponse, brainstormResponse, choosePlanInfo } = + await layer2( + ws, + userId, + messagesWithContinuedMessages, + fileContext, + onResponseChunk + ) + + const filesInfoMessage = getRelevantFileInfoMessage(files) - const prefilledResponse = ` - -${extraFiles} - + fileContext.files = files + + const assistantResponse = ` +${filesInfoMessage} ${codeReviewResponse} - ${brainstormResponse} + +${choosePlanInfo.fullResponse} + `.trim() + console.log('', assistantResponse, '') + fullResponse += assistantResponse - messages.push({ - role: 'assistant', - content: prefilledResponse, - }) + continuedMessages.push( + { + role: 'assistant', + content: assistantResponse, + }, + { + role: 'user', + content: 'Continue', + } + ) - layer = 3 + const { chosenPlan } = choosePlanInfo + if (chosenPlan === 'PAUSE') { + onResponseChunk(choosePlanInfo.fullResponse) + printedResponse += choosePlanInfo.fullResponse + return { + response: fullResponse, + changes: [], + } + } else if (chosenPlan === 'GATHER_MORE_INFO') { + layer = 2 + await warmCache(fileContext, messagesWithContinuedMessages, userId) + } else if (chosenPlan === 'PROCEED') layer = 3 } else if (layer === 3) { const { changes, response } = await layer3( ws, userId, - messages, + messagesWithContinuedMessages, fileContext, - extraFiles, onResponseChunk ) + + printedResponse += response + fullResponse += response + return { response: fullResponse, changes, @@ -103,3 +119,32 @@ ${brainstormResponse} } } } + +const warmCache = async ( + fileContext: ProjectFileContext, + messages: Message[], + userId: string +) => { + console.log('Starting to warm cache') + const startTime = Date.now() + const system = getSystemPrompt(fileContext) + await promptClaude(messages, { + model: models.sonnet, + system, + userId, + maxTokens: 1, + }) + const endTime = Date.now() + const duration = endTime - startTime + console.log(`Warmed cache in ${duration}ms`) +} + +function getRelevantFileInfoMessage(files: { + [filePath: string]: string | null +}) { + const filePaths = Object.keys(files) + if (filePaths.length === 0) { + return '' + } + return `Reading the following files...${filePaths.join(', ')}\n\n` +} diff --git a/backend/src/request-files-prompt.ts b/backend/src/request-files-prompt.ts index 68a7e349c3..dd372c3a0d 100644 --- a/backend/src/request-files-prompt.ts +++ b/backend/src/request-files-prompt.ts @@ -32,6 +32,49 @@ export async function requestRelevantFiles( return await requestFiles(ws, filePaths) } +export async function requestAdditionalFiles( + ws: WebSocket, + { + messages, + system, + }: { + messages: Message[] + system: System + }, + fileContext: ProjectFileContext, + userId: string +) { + const lastMessage = messages[messages.length - 1] + const messagesExcludingLastIfByUser = + lastMessage.role === 'user' ? messages.slice(0, -1) : messages + const userPrompt = + lastMessage.role === 'user' + ? typeof lastMessage.content === 'string' + ? lastMessage.content + : JSON.stringify(lastMessage.content) + : null + + const prompt = generateRequestAdditionalFilesPrompt( + userPrompt, + null, + fileContext + ) + const { files } = await getRelevantFiles( + { messages: messagesExcludingLastIfByUser, system }, + prompt, + models.sonnet, + 'Additional files', + userId + ).catch((error) => { + console.error('Error requesting additional files:', error) + return { files: [], duration: 0 } + }) + console.log('Additional files:', files) + const previousFiles = Object.keys(fileContext.files) + const uniqueFiles = uniq([...files, ...previousFiles]) + return await requestFiles(ws, uniqueFiles) +} + export async function requestRelevantFilesPrompt( { messages, @@ -311,3 +354,48 @@ Example response: ${getExampleFileList(fileContext, count).join('\n')} `.trim() } + +function generateRequestAdditionalFilesPrompt( + userPrompt: string | null, + assistantPrompt: string | null, + fileContext: ProjectFileContext +): string { + const previousFiles = Object.keys(fileContext.files) + return ` +${ + userPrompt + ? `${userPrompt}` + : `${assistantPrompt}` +} + +The following files have already been requested: +${previousFiles.join('\n')} + +What are the most important new files that the user needs to understand to complete their request? + +Consider the following steps: +1. Analyze the conversation history to understand the user's last request and identify the core components or tasks. +2. Focus on the most critical areas of the codebase that are directly related to the request, such as: + - Main functionality files + - Key configuration files + - Central utility functions + - Primary test files (if testing is involved) + - Documentation files +3. Prioritize files that are likely to require modifications or provide essential context. +4. Consider files one step removed from the core files you would first request. +5. Order the files by most important first. + +Please provide no commentary and only list the file paths in the following format: +${range(3) + .map((i) => `path/to/file${i + 1}.ts`) + .join('\n')} +... + +Remember to focus on the most important files. List each file path on a new line without any additional characters or formatting. + +Be sure to include the full path from the project root directory for each file. Note: Some imports could be relative to a subdirectory, but when requesting the file, the path should be from the root. You should correct any requested file paths to include the full path from the project root. + +Example response: +${getExampleFileList(fileContext, 10).join('\n')} +`.trim() +} diff --git a/backend/src/system-prompt.ts b/backend/src/system-prompt.ts index 24ebc4c566..f77a9bdd07 100644 --- a/backend/src/system-prompt.ts +++ b/backend/src/system-prompt.ts @@ -15,13 +15,11 @@ export function getSystemPrompt( onlyCachePrefix?: boolean } = {} ) { - const { checkFiles, onlyCachePrefix } = options + const { onlyCachePrefix } = options const truncatedFiles = getTruncatedFilesBasedOnTokenBudget( fileContext, 100_000 ) - const files = Object.keys(truncatedFiles) - return buildArray( { type: 'text' as const, @@ -33,19 +31,25 @@ ${editingFilesPrompt} ${knowledgeFilesPrompt} -${toolsPrompt} +${getResponseFormatPrompt()} ${getProjectFileTreePrompt(fileContext)} -${getRelevantFilesPromptPart1(fileContext)} +${getKnowledgeFilesPrompt(fileContext)} `.trim(), }, + { + type: 'text' as const, + cache_control: { type: 'ephemeral' as const }, + text: ` +${getFilesPrompt(fileContext, truncatedFiles)} +`.trimEnd(), + }, !onlyCachePrefix && { type: 'text' as const, text: ` -${getRelevantFilesPromptPart2(fileContext, truncatedFiles)} -${getResponseFormatPrompt(checkFiles ?? false, files)} +${getGitStatusPrompt(fileContext)} `.trimEnd(), } ) @@ -207,6 +211,44 @@ When using this tool, keep the following guidelines in mind: Scrape any url that could help address the user's request. `.trim() +const getResponseFormatPrompt = () => { + let bulletNumber = 1 + return ` +# Response format + +The goal is to make as few changes as possible to the codebase to address the user's request. Only do what the user has asked for and no more. + +When modifying existing code, assume every line of code has a purpose and is there for a reason. Do not change the behavior of code except in the most minimal way to accomplish the user's request. + +Special cases: + +A. If a user writes what looks like a terminal command, you should execute it. The result of the command is already printed to the user. Do not write any further text after running the command, unless the user has asked explicitly for more information. + +B. If the user provided a url, please use the scrape_web_page tool on it to better answer their question, and then proceed to the general case below. + +General case: + +${bulletNumber++}. If the plan is somewhat complex, you should then explain the reasoning behind the plan step-by-step. +If you discover an error, you should correct it and then explain the reasoning behind the corrected plan. +If you need to read more files, use the update_file_context tool and go back to step 1 to review the files. + +${bulletNumber++}. You may edit files to address the user's request (but make as few changes as possible!) and run commands in the terminal (but try not to run too many commands!). However, you should stop after a second attempt at any task has failed and ask the user how to proceed (e.g. you ran tests and they have failed twice, or you tried to compile and it failed twice.) It is a bad experience for the user when you keep trying for a third time in a row without checking to see how they user wants to proceed. + +If the user corrected you or gave feedback and it helped you understand something better, you must edit a knowledge file with a short note that condenses what you learned and what to do next time you so you don't make the same mistake again. Pure documentation of code doesn't need to be added to knowlege. But if the user says use yarn instead of npm, or to use one function instead of another, or to use a certain style, or that you should always write tests, then this is good information to add to a knoweldge file (create the file if it doesn't exist!). + + +Confine your edits to only what is directly necessary. Preserve the behavior of all existing code. Change only what you must to accomplish the user's request or add to a knowledge file. + + +Always end your response with the following marker: +${STOP_MARKER} + +This marker helps ensure that your entire response has been received and processed correctly. +If you don't end with this marker, you will automatically be prompted to continue. However, it is good to stop your response with this token so the user can give further guidence. +`.trim() +} + + const getProjectFileTreePrompt = (fileContext: ProjectFileContext) => { const { fileTree, currentWorkingDirectory } = fileContext return ` @@ -229,7 +271,7 @@ Note: the project file tree is cached from the start of this conversation. `.trim() } -const getRelevantFilesPromptPart1 = (fileContext: ProjectFileContext) => { +const getKnowledgeFilesPrompt = (fileContext: ProjectFileContext) => { const { knowledgeFiles } = fileContext return ` @@ -245,7 +287,7 @@ Note: the knowledge files are cached from the start of this conversation. `.trim() } -const getRelevantFilesPromptPart2 = ( +const getFilesPrompt = ( fileContext: ProjectFileContext, truncatedFiles: Record ) => { @@ -263,10 +305,25 @@ const getRelevantFilesPromptPart2 = ( ) .join('\n') + return ` + +Here are some files that were selected to aid in the user request, ordered by most important first: +${fileBlocks} + +Use the tool update_file_context to change the set of files listed here. You should not use this tool to read a file that is already included. + + +As you can see, some files that you might find useful are already provided. If the included set of files is not sufficient to address the user's request, you should use the update_file_context tool to update the set of files and their contents. +`.trim() +} + +const getGitStatusPrompt = (projectFileContext: ProjectFileContext) => { + const { gitChanges } = projectFileContext + return ` ${ gitChanges - ? `Current Git Changes: + ? `Current git changes on the user's project: ${gitChanges.status} @@ -285,70 +342,9 @@ ${gitChanges.lastCommitMessages} ` : '' } - -Here are some files that were selected to aid in the user request, ordered by most important first: -${fileBlocks} - -Use the tool update_file_context to change the set of files listed here. You should not use this tool to read a file that is already included. - - -As you can see, some files that you might find useful are already provided. If the included set of files is not sufficient to address the user's request, you should use the update_file_context tool to update the set of files and their contents. `.trim() } -const getResponseFormatPrompt = (checkFiles: boolean, files: string[]) => { - let bulletNumber = 1 - return ` -# Response format - -The goal is to make as few changes as possible to the codebase to address the user's request. Only do what the user has asked for and no more. - -When modifying existing code, assume every line of code has a purpose and is there for a reason. Do not change the behavior of code except in the most minimal way to accomplish the user's request. - -Special cases: - -A. If a user writes what looks like a terminal command, you should execute it. The result of the command is already printed to the user. Do not write any further text after running the command, unless the user has asked explicitly for more information. - -B. If the user provided a url, please use the scrape_web_page tool on it to better answer their question, and then proceed to the general case below. - -General case: - -${bulletNumber++}. Create a block and describe what is happening in the key files included in the user message. - -${ - checkFiles - ? `${bulletNumber++}. Request files. You are reading the following files: ${files.join(', ')}. Carefully consider if there are any files not listed here that you need to read or intend to modify before continuing in order to address the last user request. If you think you have all the files you need, please double check. Use the update_file_context tool to request any files you need. Otherwise, write "I have all the files I need". Remember, any files that are not listed in the block should not be requested since they don't exist.\n` - : '' -} - -${bulletNumber++}. After understanding the user request and the code, you should create a block. In it, you should: -I. List all the possible plans to solve the user's problem. -II. Discuss how much uncertainty or ambiguity there is in fulfilling the user's request and knowing what plan they would like most. -Assign an uncertainty score between 0 (no ambiguity) and 100 (high ambiguity) that you know what the user wants and can implement the plan they would like most. -If your uncertainty score is greater than 5, you should pause, not modify any files, and ask the user to clarify their request or ask them if your plan is good. -If your uncertainty score is 5 or lower, you should proceed to the next step. -III. Decide on a plan to address the user's request. What is the core of what the user wants done? Only implement that, and leave the rest as a follow up. - -${bulletNumber++}. If the plan is somewhat complex, you should then explain the reasoning behind the plan step-by-step. -If you discover an error, you should correct it and then explain the reasoning behind the corrected plan. -If you need to read more files, use the update_file_context tool and go back to step 1 to review the files. - -${bulletNumber++}. You may edit files to address the user's request (but make as few changes as possible!) and run commands in the terminal (but try not to run too many commands!). However, you should stop after a second attempt at any task has failed and ask the user how to proceed (e.g. you ran tests and they have failed twice, or you tried to compile and it failed twice.) It is a bad experience for the user when you keep trying for a third time in a row without checking to see how they user wants to proceed. - -If the user corrected you or gave feedback and it helped you understand something better, you must edit a knowledge file with a short note that condenses what you learned and what to do next time you so you don't make the same mistake again. Pure documentation of code doesn't need to be added to knowlege. But if the user says use yarn instead of npm, or to use one function instead of another, or to use a certain style, or that you should always write tests, then this is good information to add to a knoweldge file (create the file if it doesn't exist!). - - -Confine your edits to only what is directly necessary. Preserve the behavior of all existing code. Change only what you must to accomplish the user's request or add to a knowledge file. - - -Always end your response with the following marker: -${STOP_MARKER} - -This marker helps ensure that your entire response has been received and processed correctly. -If you don't end with this marker, you will automatically be prompted to continue. However, it is good to stop your response with this token so the user can give further guidence. -`.trim() -} - const getTruncatedFilesBasedOnTokenBudget = ( fileContext: ProjectFileContext, tokenBudget: number diff --git a/backend/src/websockets/websocket-action.ts b/backend/src/websockets/websocket-action.ts index 9cad53dc09..7eaf9f16c7 100644 --- a/backend/src/websockets/websocket-action.ts +++ b/backend/src/websockets/websocket-action.ts @@ -1,5 +1,4 @@ import { WebSocket } from 'ws' -import { Message as AnthropicMessage } from '@anthropic-ai/sdk/resources' import { ClientMessage } from 'common/websockets/websocket-schema' import { mainPrompt } from '../main-prompt' @@ -8,7 +7,6 @@ import { sendMessage } from './server' import { isEqual } from 'lodash' import fs from 'fs' import path from 'path' -import { getTools } from '../tools' import { getSystemPrompt } from '../system-prompt' import { promptClaude, models } from '../claude' @@ -129,6 +127,7 @@ const onWarmContextCache = async ( model: models.sonnet, system, userId: fingerprintId, + maxTokens: 1, } ) sendAction(ws, { From 774fbfac7514c0be93a004517a2bc00cb16e6762 Mon Sep 17 00:00:00 2001 From: James Grugett Date: Tue, 3 Sep 2024 11:24:06 -0700 Subject: [PATCH 4/4] Fix plan parsing, add log latency logs to main prompt --- backend/src/layer-2.ts | 28 +++++++++++++++++++++------- backend/src/main-prompt.ts | 8 ++++++++ 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/backend/src/layer-2.ts b/backend/src/layer-2.ts index a5a6c14f8b..6b3cad832e 100644 --- a/backend/src/layer-2.ts +++ b/backend/src/layer-2.ts @@ -162,18 +162,32 @@ Please write out a block that contains "PROCEED", "PAUSE", or "GAT } const parseChoosePlanPrompt = (response: string) => { - const uncertaintyScoreRegex = /(.*?)<\/uncertainty_score>/ - const chosenPlanRegex = /(.*?)<\/chosen_plan>/ + const uncertaintyScoreRegex = + /[\s\S]*?<\/uncertainty_score>/s + const chosenPlanRegex = /[\s\S]*?<\/chosen_plan>/s const uncertaintyScoreMatch = response.match(uncertaintyScoreRegex) const chosenPlanMatch = response.match(chosenPlanRegex) - if (!uncertaintyScoreMatch || !chosenPlanMatch) { - throw new Error('Could not parse choose plan prompt') + let uncertaintyScore = 0 + let chosenPlanStr = 'PAUSE' + if (!uncertaintyScoreMatch) { + console.error('Could not parse uncertainty score', response) + } else { + uncertaintyScore = parseInt( + uncertaintyScoreMatch[0].replace(/<\/?uncertainty_score>/g, '').trim(), + 10 + ) } - const uncertaintyScore = parseInt(uncertaintyScoreMatch[1], 10) - const chosenPlanStr = chosenPlanMatch[1] as PossiblePlan - const chosenPlan = possiblePlans.includes(chosenPlanStr) + if (!chosenPlanMatch) { + console.error('Could not parse chosen plan', response) + } else { + chosenPlanStr = chosenPlanMatch[0] + .replace(/<\/?chosen_plan>/g, '') + .trim() as PossiblePlan + } + + const chosenPlan = possiblePlans.includes(chosenPlanStr as PossiblePlan) ? chosenPlanStr : ('PAUSE' as const) diff --git a/backend/src/main-prompt.ts b/backend/src/main-prompt.ts index 8c1befeb0b..25adc43440 100644 --- a/backend/src/main-prompt.ts +++ b/backend/src/main-prompt.ts @@ -30,6 +30,8 @@ export async function mainPrompt( while (true) { console.log('layer', layer) + const startTime = Date.now() + const messagesWithContinuedMessages = [...messages, ...continuedMessages] if (layer === 1) { const { files } = await layer1( @@ -47,6 +49,8 @@ export async function mainPrompt( fullResponse += filesInfoMessage layer = 2 + console.log(`Layer 1 took ${Date.now() - startTime}ms`) + await warmCache(fileContext, messagesWithContinuedMessages, userId) } else if (layer === 2) { const { files, codeReviewResponse, brainstormResponse, choosePlanInfo } = @@ -88,6 +92,8 @@ ${choosePlanInfo.fullResponse} } ) + console.log(`Layer 2 took ${Date.now() - startTime}ms`) + const { chosenPlan } = choosePlanInfo if (chosenPlan === 'PAUSE') { onResponseChunk(choosePlanInfo.fullResponse) @@ -112,6 +118,8 @@ ${choosePlanInfo.fullResponse} printedResponse += response fullResponse += response + console.log(`Layer 3 took ${Date.now() - startTime}ms`) + return { response: fullResponse, changes,