diff --git a/apps/vscode-e2e/src/suite/tools/read-file.test.ts b/apps/vscode-e2e/src/suite/tools/read-file.test.ts index 00aca7f58ab..6f3e28f60fc 100644 --- a/apps/vscode-e2e/src/suite/tools/read-file.test.ts +++ b/apps/vscode-e2e/src/suite/tools/read-file.test.ts @@ -376,7 +376,7 @@ suite.skip("Roo Code read_file Tool", function () { } }) - test("Should read file with line range", async function () { + test("Should read file with slice offset/limit", async function () { const api = globalThis.api const messages: ClineMessage[] = [] let taskCompleted = false @@ -446,7 +446,7 @@ suite.skip("Roo Code read_file Tool", function () { alwaysAllowReadOnly: true, alwaysAllowReadOnlyOutsideWorkspace: true, }, - text: `Use the read_file tool to read the file "${fileName}" and show me what's on lines 2, 3, and 4. The file contains lines like "Line 1", "Line 2", etc. Assume the file exists and you can read it directly.`, + text: `Use the read_file tool to read the file "${fileName}" using slice mode with offset=2 and limit=3 (1-based offset). The file contains lines like "Line 1", "Line 2", etc. After reading, show me the three lines you read.`, }) // Wait for task completion @@ -455,9 +455,8 @@ suite.skip("Roo Code read_file Tool", function () { // Verify tool was executed assert.ok(toolExecuted, "The read_file tool should have been executed") - // Verify the tool returned the correct lines (when line range is used) + // Verify the tool returned the correct lines (offset=2, limit=3 -> lines 2-4) if (toolResult && (toolResult as string).includes(" | ")) { - // The result includes line numbers assert.ok( (toolResult as string).includes("2 | Line 2"), "Tool result should include line 2 with line number", diff --git a/packages/cloud/src/__tests__/CloudSettingsService.parsing.test.ts b/packages/cloud/src/__tests__/CloudSettingsService.parsing.test.ts index 22191ec90a7..8d69303c380 100644 --- a/packages/cloud/src/__tests__/CloudSettingsService.parsing.test.ts +++ b/packages/cloud/src/__tests__/CloudSettingsService.parsing.test.ts @@ -81,7 +81,6 @@ describe("CloudSettingsService - Response Parsing", () => { version: 2, defaultSettings: { maxOpenTabsContext: 10, - maxReadFileLine: 1000, }, allowList: { allowAll: false, diff --git a/packages/types/src/cloud.ts b/packages/types/src/cloud.ts index f14f14370b9..206a5647b3e 100644 --- a/packages/types/src/cloud.ts +++ b/packages/types/src/cloud.ts @@ -95,7 +95,6 @@ export const organizationDefaultSettingsSchema = globalSettingsSchema .pick({ enableCheckpoints: true, maxOpenTabsContext: true, - maxReadFileLine: true, maxWorkspaceFiles: true, showRooIgnoredFiles: true, terminalCommandDelay: true, @@ -107,7 +106,6 @@ export const organizationDefaultSettingsSchema = globalSettingsSchema .merge( z.object({ maxOpenTabsContext: z.number().int().nonnegative().optional(), - maxReadFileLine: z.number().int().gte(-1).optional(), maxWorkspaceFiles: z.number().int().nonnegative().optional(), terminalCommandDelay: z.number().int().nonnegative().optional(), terminalShellIntegrationTimeout: z.number().int().nonnegative().optional(), diff --git a/packages/types/src/global-settings.ts b/packages/types/src/global-settings.ts index 72c8b8256f8..11b9fe148d1 100644 --- a/packages/types/src/global-settings.ts +++ b/packages/types/src/global-settings.ts @@ -118,7 +118,6 @@ export const globalSettingsSchema = z.object({ allowedMaxCost: z.number().nullish(), autoCondenseContext: z.boolean().optional(), autoCondenseContextPercent: z.number().optional(), - maxConcurrentFileReads: z.number().optional(), /** * Whether to include current time in the environment details @@ -172,7 +171,6 @@ export const globalSettingsSchema = z.object({ maxWorkspaceFiles: z.number().optional(), showRooIgnoredFiles: z.boolean().optional(), enableSubfolderRules: z.boolean().optional(), - maxReadFileLine: z.number().optional(), maxImageFileSize: z.number().optional(), maxTotalImageSize: z.number().optional(), @@ -382,7 +380,6 @@ export const EVALS_SETTINGS: RooCodeSettings = { maxWorkspaceFiles: 200, maxGitStatusFiles: 20, showRooIgnoredFiles: true, - maxReadFileLine: -1, // -1 to enable full file reading. includeDiagnosticMessages: true, maxDiagnosticMessages: 50, diff --git a/packages/types/src/telemetry.ts b/packages/types/src/telemetry.ts index f8127e69888..68ed38fe326 100644 --- a/packages/types/src/telemetry.ts +++ b/packages/types/src/telemetry.ts @@ -73,6 +73,7 @@ export enum TelemetryEventName { CODE_INDEX_ERROR = "Code Index Error", TELEMETRY_SETTINGS_CHANGED = "Telemetry Settings Changed", MODEL_CACHE_EMPTY_RESPONSE = "Model Cache Empty Response", + READ_FILE_LEGACY_FORMAT_USED = "Read File Legacy Format Used", } /** @@ -203,6 +204,7 @@ export const rooCodeTelemetryEventSchema = z.discriminatedUnion("type", [ TelemetryEventName.TAB_SHOWN, TelemetryEventName.MODE_SETTINGS_CHANGED, TelemetryEventName.CUSTOM_MODE_CREATED, + TelemetryEventName.READ_FILE_LEGACY_FORMAT_USED, ]), properties: telemetryPropertiesSchema, }), diff --git a/packages/types/src/tool-params.ts b/packages/types/src/tool-params.ts index f8708b0c2b4..75be318d8c0 100644 --- a/packages/types/src/tool-params.ts +++ b/packages/types/src/tool-params.ts @@ -2,16 +2,96 @@ * Tool parameter type definitions for native protocol */ +/** + * Read mode for the read_file tool. + * - "slice": Simple offset/limit reading (default) + * - "indentation": Semantic block extraction based on code structure + */ +export type ReadFileMode = "slice" | "indentation" + +/** + * Indentation-mode configuration for the read_file tool. + */ +export interface IndentationParams { + /** 1-based line number to anchor indentation extraction (defaults to offset) */ + anchor_line?: number + /** Maximum indentation levels to include above anchor (0 = unlimited) */ + max_levels?: number + /** Include sibling blocks at the same indentation level */ + include_siblings?: boolean + /** Include file header (imports, comments at top) */ + include_header?: boolean + /** Hard cap on lines returned for indentation mode */ + max_lines?: number +} + +/** + * Parameters for the read_file tool (new format). + * + * NOTE: This is the canonical, single-file-per-call shape. + */ +export interface ReadFileParams { + /** Path to the file, relative to workspace */ + path: string + /** Reading mode: "slice" (default) or "indentation" */ + mode?: ReadFileMode + /** 1-based line number to start reading from (slice mode, default: 1) */ + offset?: number + /** Maximum number of lines to read (default: 2000) */ + limit?: number + /** Indentation-mode configuration (only used when mode === "indentation") */ + indentation?: IndentationParams +} + +// ─── Legacy Format Types (Backward Compatibility) ───────────────────────────── + +/** + * Line range specification for legacy read_file format. + * Represents a contiguous range of lines [start, end] (1-based, inclusive). + */ export interface LineRange { start: number end: number } +/** + * File entry for legacy read_file format. + * Supports reading multiple disjoint line ranges from a single file. + */ export interface FileEntry { + /** Path to the file, relative to workspace */ path: string + /** Optional list of line ranges to read (if omitted, reads entire file) */ lineRanges?: LineRange[] } +/** + * Legacy parameters for the read_file tool (pre-refactor format). + * Supports reading multiple files in a single call with optional line ranges. + * + * @deprecated Use ReadFileParams instead. This format is maintained for + * backward compatibility with existing chat histories. + */ +export interface LegacyReadFileParams { + /** Array of file entries to read */ + files: FileEntry[] + /** Discriminant flag for type narrowing */ + _legacyFormat: true +} + +/** + * Union type for read_file tool parameters. + * Supports both new single-file format and legacy multi-file format. + */ +export type ReadFileToolParams = ReadFileParams | LegacyReadFileParams + +/** + * Type guard to check if params are in legacy format. + */ +export function isLegacyReadFileParams(params: ReadFileToolParams): params is LegacyReadFileParams { + return "_legacyFormat" in params && params._legacyFormat === true +} + export interface Coordinate { x: number y: number diff --git a/packages/types/src/vscode-extension-host.ts b/packages/types/src/vscode-extension-host.ts index 5670fa1ade9..b9eb318fa9f 100644 --- a/packages/types/src/vscode-extension-host.ts +++ b/packages/types/src/vscode-extension-host.ts @@ -64,7 +64,6 @@ export interface ExtensionMessage { | "remoteBrowserEnabled" | "ttsStart" | "ttsStop" - | "maxReadFileLine" | "fileSearchResults" | "toggleApiConfigPin" | "acceptInput" @@ -301,7 +300,6 @@ export type ExtensionState = Pick< | "ttsSpeed" | "soundEnabled" | "soundVolume" - | "maxConcurrentFileReads" | "terminalOutputPreviewSize" | "terminalShellIntegrationTimeout" | "terminalShellIntegrationDisabled" @@ -352,7 +350,6 @@ export type ExtensionState = Pick< maxWorkspaceFiles: number // Maximum number of files to include in current working directory details (0-500) showRooIgnoredFiles: boolean // Whether to show .rooignore'd files in listings enableSubfolderRules: boolean // Whether to load rules from subdirectories - maxReadFileLine: number // Maximum number of lines to read from a file before truncating maxImageFileSize: number // Maximum size of image files to process in MB maxTotalImageSize: number // Maximum total size for all images in a single read operation in MB @@ -807,6 +804,7 @@ export interface ClineSayTool { isProtected?: boolean additionalFileCount?: number // Number of additional files in the same read_file request lineNumber?: number + startLine?: number // Starting line for read_file operations (for navigation on click) query?: string batchFiles?: Array<{ path: string diff --git a/src/__tests__/command-mentions.spec.ts b/src/__tests__/command-mentions.spec.ts index 1b3ccc01aab..7b69d245d81 100644 --- a/src/__tests__/command-mentions.spec.ts +++ b/src/__tests__/command-mentions.spec.ts @@ -36,7 +36,6 @@ describe("Command Mentions", () => { false, // showRooIgnoredFiles true, // includeDiagnosticMessages 50, // maxDiagnosticMessages - undefined, // maxReadFileLine ) } diff --git a/src/api/providers/__tests__/bedrock-native-tools.spec.ts b/src/api/providers/__tests__/bedrock-native-tools.spec.ts index d3f54d65b83..e95b2c34b69 100644 --- a/src/api/providers/__tests__/bedrock-native-tools.spec.ts +++ b/src/api/providers/__tests__/bedrock-native-tools.spec.ts @@ -135,23 +135,18 @@ describe("AwsBedrockHandler Native Tool Calling", () => { parameters: { type: "object", properties: { - files: { - type: "array", - items: { - type: "object", - properties: { - path: { type: "string" }, - line_ranges: { - type: ["array", "null"], - items: { type: "integer" }, - description: "Optional line ranges", - }, + path: { type: "string" }, + indentation: { + type: ["object", "null"], + properties: { + anchor_line: { + type: ["integer", "null"], + description: "Optional anchor line", }, - required: ["path", "line_ranges"], }, }, }, - required: ["files"], + required: ["path"], }, }, }, @@ -167,15 +162,14 @@ describe("AwsBedrockHandler Native Tool Calling", () => { expect(executeCommandSchema.properties.cwd.type).toBeUndefined() expect(executeCommandSchema.properties.cwd.description).toBe("Working directory (optional)") - // Second tool: line_ranges should be transformed from type: ["array", "null"] to anyOf - // with items moved inside the array variant (required by GPT-5-mini strict schema validation) + // Second tool: nested nullable object should be transformed from type: ["object", "null"] to anyOf const readFileSchema = bedrockTools[1].toolSpec.inputSchema.json as any - const lineRanges = readFileSchema.properties.files.items.properties.line_ranges - expect(lineRanges.anyOf).toEqual([{ type: "array", items: { type: "integer" } }, { type: "null" }]) - expect(lineRanges.type).toBeUndefined() - // items should now be inside the array variant, not at root - expect(lineRanges.items).toBeUndefined() - expect(lineRanges.description).toBe("Optional line ranges") + const indentation = readFileSchema.properties.indentation + expect(indentation.anyOf).toBeDefined() + expect(indentation.type).toBeUndefined() + // Object-level schema properties are preserved at the root, not inside the anyOf object variant + expect(indentation.additionalProperties).toBe(false) + expect(indentation.properties.anchor_line.anyOf).toEqual([{ type: "integer" }, { type: "null" }]) }) it("should filter non-function tools", () => { diff --git a/src/core/assistant-message/NativeToolCallParser.ts b/src/core/assistant-message/NativeToolCallParser.ts index ecf649e2734..72c34f94a07 100644 --- a/src/core/assistant-message/NativeToolCallParser.ts +++ b/src/core/assistant-message/NativeToolCallParser.ts @@ -313,9 +313,22 @@ export class NativeToolCallParser { return finalToolUse } + private static coerceOptionalNumber(value: unknown): number | undefined { + if (typeof value === "number" && Number.isFinite(value)) { + return value + } + if (typeof value === "string") { + const n = Number(value) + if (Number.isFinite(n)) { + return n + } + } + return undefined + } + /** * Convert raw file entries from API (with line_ranges) to FileEntry objects - * (with lineRanges). Handles multiple formats for compatibility: + * (with lineRanges). Handles multiple formats for backward compatibility: * * New tuple format: { path: string, line_ranges: [[1, 50], [100, 150]] } * Object format: { path: string, line_ranges: [{ start: 1, end: 50 }] } @@ -323,19 +336,21 @@ export class NativeToolCallParser { * * Returns: { path: string, lineRanges: [{ start: 1, end: 50 }] } */ - private static convertFileEntries(files: any[]): FileEntry[] { - return files.map((file: any) => { - const entry: FileEntry = { path: file.path } - if (file.line_ranges && Array.isArray(file.line_ranges)) { - entry.lineRanges = file.line_ranges - .map((range: any) => { + private static convertFileEntries(files: unknown[]): FileEntry[] { + return files.map((file: unknown) => { + const f = file as Record + const entry: FileEntry = { path: f.path as string } + if (f.line_ranges && Array.isArray(f.line_ranges)) { + entry.lineRanges = (f.line_ranges as unknown[]) + .map((range: unknown) => { // Handle tuple format: [start, end] if (Array.isArray(range) && range.length >= 2) { return { start: Number(range[0]), end: Number(range[1]) } } // Handle object format: { start: number, end: number } if (typeof range === "object" && range !== null && "start" in range && "end" in range) { - return { start: Number(range.start), end: Number(range.end) } + const r = range as { start: unknown; end: unknown } + return { start: Number(r.start), end: Number(r.end) } } // Handle legacy string format: "1-50" if (typeof range === "string") { @@ -346,7 +361,7 @@ export class NativeToolCallParser { } return null }) - .filter(Boolean) + .filter((r): r is { start: number; end: number } => r !== null) } return entry }) @@ -378,10 +393,60 @@ export class NativeToolCallParser { // Build partial nativeArgs based on what we have so far let nativeArgs: any = undefined + // Track if legacy format was used (for telemetry) + let usedLegacyFormat = false + switch (name) { case "read_file": - if (partialArgs.files && Array.isArray(partialArgs.files)) { - nativeArgs = { files: this.convertFileEntries(partialArgs.files) } + // Check for legacy format first: { files: [...] } + // Handle both array and stringified array (some models double-stringify) + if (partialArgs.files !== undefined) { + let filesArray: unknown[] | null = null + + if (Array.isArray(partialArgs.files)) { + filesArray = partialArgs.files + } else if (typeof partialArgs.files === "string") { + // Handle double-stringified case: files is a string containing JSON array + try { + const parsed = JSON.parse(partialArgs.files) + if (Array.isArray(parsed)) { + filesArray = parsed + } + } catch { + // Not valid JSON, ignore + } + } + + if (filesArray && filesArray.length > 0) { + usedLegacyFormat = true + nativeArgs = { + files: this.convertFileEntries(filesArray), + _legacyFormat: true as const, + } + } + } + // New format: { path: "...", mode: "..." } + if (!nativeArgs && partialArgs.path !== undefined) { + nativeArgs = { + path: partialArgs.path, + mode: partialArgs.mode, + offset: this.coerceOptionalNumber(partialArgs.offset), + limit: this.coerceOptionalNumber(partialArgs.limit), + indentation: + partialArgs.indentation && typeof partialArgs.indentation === "object" + ? { + anchor_line: this.coerceOptionalNumber(partialArgs.indentation.anchor_line), + max_levels: this.coerceOptionalNumber(partialArgs.indentation.max_levels), + max_lines: this.coerceOptionalNumber(partialArgs.indentation.max_lines), + include_siblings: this.coerceOptionalBoolean( + partialArgs.indentation.include_siblings, + ), + include_header: this.coerceOptionalBoolean( + partialArgs.indentation.include_header, + ), + } + : undefined, + } } break @@ -596,6 +661,11 @@ export class NativeToolCallParser { result.originalName = originalName } + // Track legacy format usage for telemetry + if (usedLegacyFormat) { + result.usedLegacyFormat = true + } + return result } @@ -642,13 +712,6 @@ export class NativeToolCallParser { const params: Partial> = {} for (const [key, value] of Object.entries(args)) { - // Skip complex parameters that have been migrated to nativeArgs. - // For read_file, the 'files' parameter is a FileEntry[] array that can't be - // meaningfully stringified. The properly typed data is in nativeArgs instead. - if (resolvedName === "read_file" && key === "files") { - continue - } - // Validate parameter name if (!toolParamNames.includes(key as ToolParamName) && !customToolRegistry.has(resolvedName)) { console.warn(`Unknown parameter '${key}' for tool '${resolvedName}'`) @@ -666,10 +729,58 @@ export class NativeToolCallParser { // nativeArgs object. If validation fails, we treat the tool call as invalid and fail fast. let nativeArgs: NativeArgsFor | undefined = undefined + // Track if legacy format was used (for telemetry) + let usedLegacyFormat = false + switch (resolvedName) { case "read_file": - if (args.files && Array.isArray(args.files)) { - nativeArgs = { files: this.convertFileEntries(args.files) } as NativeArgsFor + // Check for legacy format first: { files: [...] } + // Handle both array and stringified array (some models double-stringify) + if (args.files !== undefined) { + let filesArray: unknown[] | null = null + + if (Array.isArray(args.files)) { + filesArray = args.files + } else if (typeof args.files === "string") { + // Handle double-stringified case: files is a string containing JSON array + try { + const parsed = JSON.parse(args.files) + if (Array.isArray(parsed)) { + filesArray = parsed + } + } catch { + // Not valid JSON, ignore + } + } + + if (filesArray && filesArray.length > 0) { + usedLegacyFormat = true + nativeArgs = { + files: this.convertFileEntries(filesArray), + _legacyFormat: true as const, + } as NativeArgsFor + } + } + // New format: { path: "...", mode: "..." } + if (!nativeArgs && args.path !== undefined) { + nativeArgs = { + path: args.path, + mode: args.mode, + offset: this.coerceOptionalNumber(args.offset), + limit: this.coerceOptionalNumber(args.limit), + indentation: + args.indentation && typeof args.indentation === "object" + ? { + anchor_line: this.coerceOptionalNumber(args.indentation.anchor_line), + max_levels: this.coerceOptionalNumber(args.indentation.max_levels), + max_lines: this.coerceOptionalNumber(args.indentation.max_lines), + include_siblings: this.coerceOptionalBoolean( + args.indentation.include_siblings, + ), + include_header: this.coerceOptionalBoolean(args.indentation.include_header), + } + : undefined, + } as NativeArgsFor } break @@ -918,6 +1029,11 @@ export class NativeToolCallParser { result.originalName = toolCall.name } + // Track legacy format usage for telemetry + if (usedLegacyFormat) { + result.usedLegacyFormat = true + } + return result } catch (error) { console.error( diff --git a/src/core/assistant-message/__tests__/NativeToolCallParser.spec.ts b/src/core/assistant-message/__tests__/NativeToolCallParser.spec.ts index 0e81671cc15..db0dc00de41 100644 --- a/src/core/assistant-message/__tests__/NativeToolCallParser.spec.ts +++ b/src/core/assistant-message/__tests__/NativeToolCallParser.spec.ts @@ -8,20 +8,12 @@ describe("NativeToolCallParser", () => { describe("parseToolCall", () => { describe("read_file tool", () => { - it("should handle line_ranges as tuples (new format)", () => { + it("should parse minimal single-file read_file args", () => { const toolCall = { id: "toolu_123", name: "read_file" as const, arguments: JSON.stringify({ - files: [ - { - path: "src/core/task/Task.ts", - line_ranges: [ - [1920, 1990], - [2060, 2120], - ], - }, - ], + path: "src/core/task/Task.ts", }), } @@ -31,29 +23,20 @@ describe("NativeToolCallParser", () => { expect(result?.type).toBe("tool_use") if (result?.type === "tool_use") { expect(result.nativeArgs).toBeDefined() - const nativeArgs = result.nativeArgs as { - files: Array<{ path: string; lineRanges?: Array<{ start: number; end: number }> }> - } - expect(nativeArgs.files).toHaveLength(1) - expect(nativeArgs.files[0].path).toBe("src/core/task/Task.ts") - expect(nativeArgs.files[0].lineRanges).toEqual([ - { start: 1920, end: 1990 }, - { start: 2060, end: 2120 }, - ]) + const nativeArgs = result.nativeArgs as { path: string } + expect(nativeArgs.path).toBe("src/core/task/Task.ts") } }) - it("should handle line_ranges as strings (legacy format)", () => { + it("should parse slice-mode params", () => { const toolCall = { id: "toolu_123", name: "read_file" as const, arguments: JSON.stringify({ - files: [ - { - path: "src/core/task/Task.ts", - line_ranges: ["1920-1990", "2060-2120"], - }, - ], + path: "src/core/task/Task.ts", + mode: "slice", + offset: 10, + limit: 20, }), } @@ -62,29 +45,32 @@ describe("NativeToolCallParser", () => { expect(result).not.toBeNull() expect(result?.type).toBe("tool_use") if (result?.type === "tool_use") { - expect(result.nativeArgs).toBeDefined() const nativeArgs = result.nativeArgs as { - files: Array<{ path: string; lineRanges?: Array<{ start: number; end: number }> }> - } - expect(nativeArgs.files).toHaveLength(1) - expect(nativeArgs.files[0].path).toBe("src/core/task/Task.ts") - expect(nativeArgs.files[0].lineRanges).toEqual([ - { start: 1920, end: 1990 }, - { start: 2060, end: 2120 }, - ]) + path: string + mode?: string + offset?: number + limit?: number + } + expect(nativeArgs.path).toBe("src/core/task/Task.ts") + expect(nativeArgs.mode).toBe("slice") + expect(nativeArgs.offset).toBe(10) + expect(nativeArgs.limit).toBe(20) } }) - it("should handle files without line_ranges", () => { + it("should parse indentation-mode params", () => { const toolCall = { id: "toolu_123", name: "read_file" as const, arguments: JSON.stringify({ - files: [ - { - path: "src/utils.ts", - }, - ], + path: "src/utils.ts", + mode: "indentation", + indentation: { + anchor_line: 123, + max_levels: 2, + include_siblings: true, + include_header: false, + }, }), } @@ -94,120 +80,242 @@ describe("NativeToolCallParser", () => { expect(result?.type).toBe("tool_use") if (result?.type === "tool_use") { const nativeArgs = result.nativeArgs as { - files: Array<{ path: string; lineRanges?: Array<{ start: number; end: number }> }> + path: string + mode?: string + indentation?: { + anchor_line?: number + max_levels?: number + include_siblings?: boolean + include_header?: boolean + } } - expect(nativeArgs.files).toHaveLength(1) - expect(nativeArgs.files[0].path).toBe("src/utils.ts") - expect(nativeArgs.files[0].lineRanges).toBeUndefined() + expect(nativeArgs.path).toBe("src/utils.ts") + expect(nativeArgs.mode).toBe("indentation") + expect(nativeArgs.indentation?.anchor_line).toBe(123) + expect(nativeArgs.indentation?.include_siblings).toBe(true) + expect(nativeArgs.indentation?.include_header).toBe(false) } }) - it("should handle multiple files with different line_ranges", () => { - const toolCall = { - id: "toolu_123", - name: "read_file" as const, - arguments: JSON.stringify({ - files: [ - { - path: "file1.ts", - line_ranges: ["1-50"], - }, - { - path: "file2.ts", - line_ranges: ["100-150", "200-250"], - }, - { - path: "file3.ts", - }, - ], - }), - } + // Legacy format backward compatibility tests + describe("legacy format backward compatibility", () => { + it("should parse legacy files array format with single file", () => { + const toolCall = { + id: "toolu_legacy_1", + name: "read_file" as const, + arguments: JSON.stringify({ + files: [{ path: "src/legacy/file.ts" }], + }), + } - const result = NativeToolCallParser.parseToolCall(toolCall) + const result = NativeToolCallParser.parseToolCall(toolCall) - expect(result).not.toBeNull() - expect(result?.type).toBe("tool_use") - if (result?.type === "tool_use") { - const nativeArgs = result.nativeArgs as { - files: Array<{ path: string; lineRanges?: Array<{ start: number; end: number }> }> - } - expect(nativeArgs.files).toHaveLength(3) - expect(nativeArgs.files[0].lineRanges).toEqual([{ start: 1, end: 50 }]) - expect(nativeArgs.files[1].lineRanges).toEqual([ - { start: 100, end: 150 }, - { start: 200, end: 250 }, - ]) - expect(nativeArgs.files[2].lineRanges).toBeUndefined() - } - }) + expect(result).not.toBeNull() + expect(result?.type).toBe("tool_use") + if (result?.type === "tool_use") { + expect(result.usedLegacyFormat).toBe(true) + const nativeArgs = result.nativeArgs as { files: Array<{ path: string }>; _legacyFormat: true } + expect(nativeArgs._legacyFormat).toBe(true) + expect(nativeArgs.files).toHaveLength(1) + expect(nativeArgs.files[0].path).toBe("src/legacy/file.ts") + } + }) - it("should filter out invalid line_range strings", () => { - const toolCall = { - id: "toolu_123", - name: "read_file" as const, - arguments: JSON.stringify({ - files: [ - { - path: "file.ts", - line_ranges: ["1-50", "invalid", "100-200", "abc-def"], - }, - ], - }), - } + it("should parse legacy files array format with multiple files", () => { + const toolCall = { + id: "toolu_legacy_2", + name: "read_file" as const, + arguments: JSON.stringify({ + files: [{ path: "src/file1.ts" }, { path: "src/file2.ts" }, { path: "src/file3.ts" }], + }), + } - const result = NativeToolCallParser.parseToolCall(toolCall) + const result = NativeToolCallParser.parseToolCall(toolCall) - expect(result).not.toBeNull() - expect(result?.type).toBe("tool_use") - if (result?.type === "tool_use") { - const nativeArgs = result.nativeArgs as { - files: Array<{ path: string; lineRanges?: Array<{ start: number; end: number }> }> + expect(result).not.toBeNull() + expect(result?.type).toBe("tool_use") + if (result?.type === "tool_use") { + expect(result.usedLegacyFormat).toBe(true) + const nativeArgs = result.nativeArgs as { files: Array<{ path: string }>; _legacyFormat: true } + expect(nativeArgs.files).toHaveLength(3) + expect(nativeArgs.files[0].path).toBe("src/file1.ts") + expect(nativeArgs.files[1].path).toBe("src/file2.ts") + expect(nativeArgs.files[2].path).toBe("src/file3.ts") } - expect(nativeArgs.files[0].lineRanges).toEqual([ - { start: 1, end: 50 }, - { start: 100, end: 200 }, - ]) - } + }) + + it("should parse legacy line_ranges as tuples", () => { + const toolCall = { + id: "toolu_legacy_3", + name: "read_file" as const, + arguments: JSON.stringify({ + files: [ + { + path: "src/task.ts", + line_ranges: [ + [1, 50], + [100, 150], + ], + }, + ], + }), + } + + const result = NativeToolCallParser.parseToolCall(toolCall) + + expect(result).not.toBeNull() + expect(result?.type).toBe("tool_use") + if (result?.type === "tool_use") { + expect(result.usedLegacyFormat).toBe(true) + const nativeArgs = result.nativeArgs as { + files: Array<{ path: string; lineRanges?: Array<{ start: number; end: number }> }> + _legacyFormat: true + } + expect(nativeArgs.files[0].lineRanges).toHaveLength(2) + expect(nativeArgs.files[0].lineRanges?.[0]).toEqual({ start: 1, end: 50 }) + expect(nativeArgs.files[0].lineRanges?.[1]).toEqual({ start: 100, end: 150 }) + } + }) + + it("should parse legacy line_ranges as objects", () => { + const toolCall = { + id: "toolu_legacy_4", + name: "read_file" as const, + arguments: JSON.stringify({ + files: [ + { + path: "src/task.ts", + line_ranges: [ + { start: 10, end: 20 }, + { start: 30, end: 40 }, + ], + }, + ], + }), + } + + const result = NativeToolCallParser.parseToolCall(toolCall) + + expect(result).not.toBeNull() + expect(result?.type).toBe("tool_use") + if (result?.type === "tool_use") { + expect(result.usedLegacyFormat).toBe(true) + const nativeArgs = result.nativeArgs as { + files: Array<{ path: string; lineRanges?: Array<{ start: number; end: number }> }> + } + expect(nativeArgs.files[0].lineRanges).toHaveLength(2) + expect(nativeArgs.files[0].lineRanges?.[0]).toEqual({ start: 10, end: 20 }) + expect(nativeArgs.files[0].lineRanges?.[1]).toEqual({ start: 30, end: 40 }) + } + }) + + it("should parse legacy line_ranges as strings", () => { + const toolCall = { + id: "toolu_legacy_5", + name: "read_file" as const, + arguments: JSON.stringify({ + files: [ + { + path: "src/task.ts", + line_ranges: ["1-50", "100-150"], + }, + ], + }), + } + + const result = NativeToolCallParser.parseToolCall(toolCall) + + expect(result).not.toBeNull() + expect(result?.type).toBe("tool_use") + if (result?.type === "tool_use") { + expect(result.usedLegacyFormat).toBe(true) + const nativeArgs = result.nativeArgs as { + files: Array<{ path: string; lineRanges?: Array<{ start: number; end: number }> }> + } + expect(nativeArgs.files[0].lineRanges).toHaveLength(2) + expect(nativeArgs.files[0].lineRanges?.[0]).toEqual({ start: 1, end: 50 }) + expect(nativeArgs.files[0].lineRanges?.[1]).toEqual({ start: 100, end: 150 }) + } + }) + + it("should parse double-stringified files array (model quirk)", () => { + // This tests the real-world case where some models double-stringify the files array + // e.g., { files: "[{\"path\": \"...\"}]" } instead of { files: [{path: "..."}] } + const toolCall = { + id: "toolu_double_stringify", + name: "read_file" as const, + arguments: JSON.stringify({ + files: JSON.stringify([ + { path: "src/services/browser/browserDiscovery.ts" }, + { path: "src/services/mcp/McpServerManager.ts" }, + ]), + }), + } + + const result = NativeToolCallParser.parseToolCall(toolCall) + + expect(result).not.toBeNull() + expect(result?.type).toBe("tool_use") + if (result?.type === "tool_use") { + expect(result.usedLegacyFormat).toBe(true) + const nativeArgs = result.nativeArgs as { + files: Array<{ path: string }> + _legacyFormat: true + } + expect(nativeArgs._legacyFormat).toBe(true) + expect(nativeArgs.files).toHaveLength(2) + expect(nativeArgs.files[0].path).toBe("src/services/browser/browserDiscovery.ts") + expect(nativeArgs.files[1].path).toBe("src/services/mcp/McpServerManager.ts") + } + }) + + it("should NOT set usedLegacyFormat for new format", () => { + const toolCall = { + id: "toolu_new", + name: "read_file" as const, + arguments: JSON.stringify({ + path: "src/new/format.ts", + mode: "slice", + offset: 1, + limit: 100, + }), + } + + const result = NativeToolCallParser.parseToolCall(toolCall) + + expect(result).not.toBeNull() + expect(result?.type).toBe("tool_use") + if (result?.type === "tool_use") { + expect(result.usedLegacyFormat).toBeUndefined() + } + }) }) }) }) describe("processStreamingChunk", () => { describe("read_file tool", () => { - it("should convert line_ranges strings to lineRanges objects during streaming", () => { + it("should emit a partial ToolUse with nativeArgs.path during streaming", () => { const id = "toolu_streaming_123" NativeToolCallParser.startStreamingToolCall(id, "read_file") // Simulate streaming chunks - const fullArgs = JSON.stringify({ - files: [ - { - path: "src/test.ts", - line_ranges: ["10-20", "30-40"], - }, - ], - }) + const fullArgs = JSON.stringify({ path: "src/test.ts" }) // Process the complete args as a single chunk for simplicity const result = NativeToolCallParser.processStreamingChunk(id, fullArgs) expect(result).not.toBeNull() expect(result?.nativeArgs).toBeDefined() - const nativeArgs = result?.nativeArgs as { - files: Array<{ path: string; lineRanges?: Array<{ start: number; end: number }> }> - } - expect(nativeArgs.files).toHaveLength(1) - expect(nativeArgs.files[0].lineRanges).toEqual([ - { start: 10, end: 20 }, - { start: 30, end: 40 }, - ]) + const nativeArgs = result?.nativeArgs as { path: string } + expect(nativeArgs.path).toBe("src/test.ts") }) }) }) describe("finalizeStreamingToolCall", () => { describe("read_file tool", () => { - it("should convert line_ranges strings to lineRanges objects on finalize", () => { + it("should parse read_file args on finalize", () => { const id = "toolu_finalize_123" NativeToolCallParser.startStreamingToolCall(id, "read_file") @@ -215,12 +323,10 @@ describe("NativeToolCallParser", () => { NativeToolCallParser.processStreamingChunk( id, JSON.stringify({ - files: [ - { - path: "finalized.ts", - line_ranges: ["500-600"], - }, - ], + path: "finalized.ts", + mode: "slice", + offset: 1, + limit: 10, }), ) @@ -229,11 +335,10 @@ describe("NativeToolCallParser", () => { expect(result).not.toBeNull() expect(result?.type).toBe("tool_use") if (result?.type === "tool_use") { - const nativeArgs = result.nativeArgs as { - files: Array<{ path: string; lineRanges?: Array<{ start: number; end: number }> }> - } - expect(nativeArgs.files[0].path).toBe("finalized.ts") - expect(nativeArgs.files[0].lineRanges).toEqual([{ start: 500, end: 600 }]) + const nativeArgs = result.nativeArgs as { path: string; offset?: number; limit?: number } + expect(nativeArgs.path).toBe("finalized.ts") + expect(nativeArgs.offset).toBe(1) + expect(nativeArgs.limit).toBe(10) } }) }) diff --git a/src/core/assistant-message/presentAssistantMessage.ts b/src/core/assistant-message/presentAssistantMessage.ts index 1d69f39cc79..5b9e1c4840e 100644 --- a/src/core/assistant-message/presentAssistantMessage.ts +++ b/src/core/assistant-message/presentAssistantMessage.ts @@ -2,7 +2,7 @@ import { serializeError } from "serialize-error" import { Anthropic } from "@anthropic-ai/sdk" import type { ToolName, ClineAsk, ToolProgressStatus } from "@roo-code/types" -import { ConsecutiveMistakeError } from "@roo-code/types" +import { ConsecutiveMistakeError, TelemetryEventName } from "@roo-code/types" import { TelemetryService } from "@roo-code/telemetry" import { customToolRegistry } from "@roo-code/core" @@ -600,6 +600,15 @@ export async function presentAssistantMessage(cline: Task) { const recordName = isCustomTool ? "custom_tool" : block.name cline.recordToolUsage(recordName) TelemetryService.instance.captureToolUsage(cline.taskId, recordName) + + // Track legacy format usage for read_file tool (for migration monitoring) + if (block.name === "read_file" && block.usedLegacyFormat) { + const modelInfo = cline.api.getModel() + TelemetryService.instance.captureEvent(TelemetryEventName.READ_FILE_LEGACY_FORMAT_USED, { + taskId: cline.taskId, + model: modelInfo?.id, + }) + } } // Validate tool use before execution - ONLY for complete (non-partial) blocks. diff --git a/src/core/mentions/__tests__/processUserContentMentions.spec.ts b/src/core/mentions/__tests__/processUserContentMentions.spec.ts index 4f45e404cc1..7732cf279b4 100644 --- a/src/core/mentions/__tests__/processUserContentMentions.spec.ts +++ b/src/core/mentions/__tests__/processUserContentMentions.spec.ts @@ -26,100 +26,10 @@ describe("processUserContentMentions", () => { vi.mocked(parseMentions).mockImplementation(async (text) => ({ text: `parsed: ${text}`, mode: undefined, + contentBlocks: [], })) }) - describe("maxReadFileLine parameter", () => { - it("should pass maxReadFileLine to parseMentions when provided", async () => { - const userContent = [ - { - type: "text" as const, - text: "Read file with limit", - }, - ] - - await processUserContentMentions({ - userContent, - cwd: "/test", - urlContentFetcher: mockUrlContentFetcher, - fileContextTracker: mockFileContextTracker, - rooIgnoreController: mockRooIgnoreController, - maxReadFileLine: 100, - }) - - expect(parseMentions).toHaveBeenCalledWith( - "Read file with limit", - "/test", - mockUrlContentFetcher, - mockFileContextTracker, - mockRooIgnoreController, - false, - true, // includeDiagnosticMessages - 50, // maxDiagnosticMessages - 100, - ) - }) - - it("should pass undefined maxReadFileLine when not provided", async () => { - const userContent = [ - { - type: "text" as const, - text: "Read file without limit", - }, - ] - - await processUserContentMentions({ - userContent, - cwd: "/test", - urlContentFetcher: mockUrlContentFetcher, - fileContextTracker: mockFileContextTracker, - rooIgnoreController: mockRooIgnoreController, - }) - - expect(parseMentions).toHaveBeenCalledWith( - "Read file without limit", - "/test", - mockUrlContentFetcher, - mockFileContextTracker, - mockRooIgnoreController, - false, - true, // includeDiagnosticMessages - 50, // maxDiagnosticMessages - undefined, - ) - }) - - it("should handle UNLIMITED_LINES constant correctly", async () => { - const userContent = [ - { - type: "text" as const, - text: "Read unlimited lines", - }, - ] - - await processUserContentMentions({ - userContent, - cwd: "/test", - urlContentFetcher: mockUrlContentFetcher, - fileContextTracker: mockFileContextTracker, - rooIgnoreController: mockRooIgnoreController, - maxReadFileLine: -1, - }) - - expect(parseMentions).toHaveBeenCalledWith( - "Read unlimited lines", - "/test", - mockUrlContentFetcher, - mockFileContextTracker, - mockRooIgnoreController, - false, - true, // includeDiagnosticMessages - 50, // maxDiagnosticMessages - -1, - ) - }) - }) - describe("content processing", () => { it("should process text blocks with tags", async () => { const userContent = [ @@ -181,10 +91,16 @@ describe("processUserContentMentions", () => { }) expect(parseMentions).toHaveBeenCalled() + // String content is now converted to array format to support content blocks expect(result.content[0]).toEqual({ type: "tool_result", tool_use_id: "123", - content: "parsed: Tool feedback", + content: [ + { + type: "text", + text: "parsed: Tool feedback", + }, + ], }) expect(result.mode).toBeUndefined() }) @@ -258,7 +174,6 @@ describe("processUserContentMentions", () => { cwd: "/test", urlContentFetcher: mockUrlContentFetcher, fileContextTracker: mockFileContextTracker, - maxReadFileLine: 50, }) expect(parseMentions).toHaveBeenCalledTimes(2) @@ -268,10 +183,16 @@ describe("processUserContentMentions", () => { text: "parsed: First task", }) expect(result.content[1]).toEqual(userContent[1]) // Image block unchanged + // String content is now converted to array format to support content blocks expect(result.content[2]).toEqual({ type: "tool_result", tool_use_id: "456", - content: "parsed: Feedback", + content: [ + { + type: "text", + text: "parsed: Feedback", + }, + ], }) expect(result.mode).toBeUndefined() }) @@ -302,7 +223,6 @@ describe("processUserContentMentions", () => { false, // showRooIgnoredFiles should default to false true, // includeDiagnosticMessages 50, // maxDiagnosticMessages - undefined, ) }) @@ -331,7 +251,6 @@ describe("processUserContentMentions", () => { false, true, // includeDiagnosticMessages 50, // maxDiagnosticMessages - undefined, ) }) }) @@ -342,6 +261,7 @@ describe("processUserContentMentions", () => { text: "parsed text", slashCommandHelp: "command help", mode: undefined, + contentBlocks: [], }) const userContent = [ @@ -374,6 +294,7 @@ describe("processUserContentMentions", () => { text: "parsed tool output", slashCommandHelp: "command help", mode: undefined, + contentBlocks: [], }) const userContent = [ @@ -413,6 +334,7 @@ describe("processUserContentMentions", () => { text: "parsed array item", slashCommandHelp: "command help", mode: undefined, + contentBlocks: [], }) const userContent = [ diff --git a/src/core/mentions/index.ts b/src/core/mentions/index.ts index ebff1bcd8c7..faa7236e67c 100644 --- a/src/core/mentions/index.ts +++ b/src/core/mentions/index.ts @@ -9,8 +9,9 @@ import { mentionRegexGlobal, commandRegexGlobal, unescapeSpaces } from "../../sh import { getCommitInfo, getWorkingState } from "../../utils/git" import { openFile } from "../../integrations/misc/open-file" -import { extractTextFromFile } from "../../integrations/misc/extract-text" +import { extractTextFromFileWithMetadata, type ExtractTextResult } from "../../integrations/misc/extract-text" import { diagnosticsToProblemsString } from "../../integrations/diagnostics" +import { DEFAULT_LINE_LIMIT } from "../prompts/tools/native-tools/read_file" import { UrlContentFetcher } from "../../services/browser/UrlContentFetcher" @@ -71,12 +72,59 @@ export async function openMention(cwd: string, mention?: string): Promise } } +/** + * Represents a content block generated from an @ mention. + * These are returned separately from the user's text to enable + * proper formatting as distinct message blocks. + */ +export interface MentionContentBlock { + type: "file" | "folder" | "url" | "diagnostics" | "git_changes" | "git_commit" | "terminal" | "command" + /** Path for file/folder mentions */ + path?: string + /** The content to display */ + content: string + /** Metadata about truncation (for files) */ + metadata?: { + totalLines: number + returnedLines: number + wasTruncated: boolean + linesShown?: [number, number] + } +} + export interface ParseMentionsResult { + /** User's text with @ mentions replaced by clean path references */ text: string + /** Separate content blocks for each mention (file content, URLs, etc.) */ + contentBlocks: MentionContentBlock[] slashCommandHelp?: string mode?: string // Mode from the first slash command that has one } +/** + * Formats file content to look like a read_file tool result. + * Includes Gemini-style truncation warning when content is truncated. + */ +function formatFileReadResult(filePath: string, result: ExtractTextResult): string { + const header = `[read_file for '${filePath}']` + + if (result.wasTruncated && result.linesShown) { + const [start, end] = result.linesShown + const nextOffset = end + 1 + return `${header} +IMPORTANT: File content truncated. +Status: Showing lines ${start}-${end} of ${result.totalLines} total lines. +To read more: Use the read_file tool with offset=${nextOffset} and limit=${DEFAULT_LINE_LIMIT}. + +File: ${filePath} +${result.content}` + } + + return `${header} +File: ${filePath} +${result.content}` +} + export async function parseMentions( text: string, cwd: string, @@ -86,10 +134,10 @@ export async function parseMentions( showRooIgnoredFiles: boolean = false, includeDiagnosticMessages: boolean = true, maxDiagnosticMessages: number = 50, - maxReadFileLine?: number, ): Promise { const mentions: Set = new Set() const validCommands: Map = new Map() + const contentBlocks: MentionContentBlock[] = [] let commandMode: string | undefined // Track mode from the first slash command that has one // First pass: check which command mentions exist and cache the results @@ -119,7 +167,7 @@ export async function parseMentions( } } - // Only replace text for commands that actually exist + // Only replace text for commands that actually exist (keep "see below" for commands) let parsedText = text for (const [match, commandName] of commandMatches) { if (validCommands.has(commandName)) { @@ -127,16 +175,17 @@ export async function parseMentions( } } - // Second pass: handle regular mentions + // Second pass: handle regular mentions - replace with clean references + // Content will be provided as separate blocks that look like read_file results parsedText = parsedText.replace(mentionRegexGlobal, (match, mention) => { mentions.add(mention) if (mention.startsWith("http")) { + // Keep old style for URLs (still XML-based) return `'${mention}' (see below for site content)` } else if (mention.startsWith("/")) { + // Clean path reference - no "see below" since we format like tool results const mentionPath = mention.slice(1) - return mentionPath.endsWith("/") - ? `'${mentionPath}' (see below for folder content)` - : `'${mentionPath}' (see below for file content)` + return mentionPath.endsWith("/") ? `'${mentionPath}'` : `'${mentionPath}'` } else if (mention === "problems") { return `Workspace Problems (see below for diagnostics)` } else if (mention === "git-changes") { @@ -189,31 +238,26 @@ export async function parseMentions( result = `Error fetching content: ${rawErrorMessage}` } } + // URLs still use XML format (appended to text for backwards compat) parsedText += `\n\n\n${result}\n` } else if (mention.startsWith("/")) { const mentionPath = mention.slice(1) try { - const content = await getFileOrFolderContent( + const fileResult = await getFileOrFolderContentWithMetadata( mentionPath, cwd, rooIgnoreController, showRooIgnoredFiles, - maxReadFileLine, + fileContextTracker, ) - if (mention.endsWith("/")) { - parsedText += `\n\n\n${content}\n` - } else { - parsedText += `\n\n\n${content}\n` - if (fileContextTracker) { - await fileContextTracker.trackFileContext(mentionPath, "file_mentioned") - } - } + contentBlocks.push(fileResult) } catch (error) { - if (mention.endsWith("/")) { - parsedText += `\n\n\nError fetching content: ${error.message}\n` - } else { - parsedText += `\n\n\nError fetching content: ${error.message}\n` - } + const errorMsg = error instanceof Error ? error.message : String(error) + contentBlocks.push({ + type: mention.endsWith("/") ? "folder" : "file", + path: mentionPath, + content: `[read_file for '${mentionPath}']\nError: ${errorMsg}`, + }) } } else if (mention === "problems") { try { @@ -269,18 +313,28 @@ export async function parseMentions( } } - return { text: parsedText, mode: commandMode, slashCommandHelp: slashCommandHelp.trim() || undefined } + return { + text: parsedText, + contentBlocks, + mode: commandMode, + slashCommandHelp: slashCommandHelp.trim() || undefined, + } } -async function getFileOrFolderContent( +/** + * Gets file or folder content and returns it as a MentionContentBlock + * formatted to look like a read_file tool result. + */ +async function getFileOrFolderContentWithMetadata( mentionPath: string, cwd: string, rooIgnoreController?: any, showRooIgnoredFiles: boolean = false, - maxReadFileLine?: number, -): Promise { + fileContextTracker?: FileContextTracker, +): Promise { const unescapedPath = unescapeSpaces(mentionPath) const absPath = path.resolve(cwd, unescapedPath) + const isFolder = mentionPath.endsWith("/") try { const stats = await fs.stat(absPath) @@ -290,21 +344,50 @@ async function getFileOrFolderContent( // Image mentions are handled separately via image attachment flow. const isBinary = await isBinaryFile(absPath).catch(() => false) if (isBinary) { - return `(Binary file ${mentionPath} omitted)` + return { + type: "file", + path: mentionPath, + content: `[read_file for '${mentionPath}']\nNote: Binary file omitted from context.`, + } } if (rooIgnoreController && !rooIgnoreController.validateAccess(unescapedPath)) { - return `(File ${mentionPath} is ignored by .rooignore)` + return { + type: "file", + path: mentionPath, + content: `[read_file for '${mentionPath}']\nNote: File is ignored by .rooignore.`, + } } try { - const content = await extractTextFromFile(absPath, maxReadFileLine) - return content + const result = await extractTextFromFileWithMetadata(absPath) + + // Track file context + if (fileContextTracker) { + await fileContextTracker.trackFileContext(mentionPath, "file_mentioned") + } + + return { + type: "file", + path: mentionPath, + content: formatFileReadResult(mentionPath, result), + metadata: { + totalLines: result.totalLines, + returnedLines: result.returnedLines, + wasTruncated: result.wasTruncated, + linesShown: result.linesShown, + }, + } } catch (error) { - return `(Failed to read contents of ${mentionPath}): ${error.message}` + const errorMsg = error instanceof Error ? error.message : String(error) + return { + type: "file", + path: mentionPath, + content: `[read_file for '${mentionPath}']\nError: ${errorMsg}`, + } } } else if (stats.isDirectory()) { const entries = await fs.readdir(absPath, { withFileTypes: true }) - let folderContent = "" - const fileContentPromises: Promise[] = [] + let folderListing = "" + const fileReadResults: string[] = [] const LOCK_SYMBOL = "🔒" for (let index = 0; index < entries.length; index++) { @@ -325,38 +408,48 @@ async function getFileOrFolderContent( const displayName = isIgnored ? `${LOCK_SYMBOL} ${entry.name}` : entry.name if (entry.isFile()) { - folderContent += `${linePrefix}${displayName}\n` + folderListing += `${linePrefix}${displayName}\n` if (!isIgnored) { const filePath = path.join(mentionPath, entry.name) const absoluteFilePath = path.resolve(absPath, entry.name) - fileContentPromises.push( - (async () => { - try { - const isBinary = await isBinaryFile(absoluteFilePath).catch(() => false) - if (isBinary) { - return undefined - } - const content = await extractTextFromFile(absoluteFilePath, maxReadFileLine) - return `\n${content}\n` - } catch (error) { - return undefined - } - })(), - ) + try { + const isBinary = await isBinaryFile(absoluteFilePath).catch(() => false) + if (!isBinary) { + const result = await extractTextFromFileWithMetadata(absoluteFilePath) + fileReadResults.push(formatFileReadResult(filePath.toPosix(), result)) + } + } catch (error) { + // Skip files that can't be read + } } } else if (entry.isDirectory()) { - folderContent += `${linePrefix}${displayName}/\n` + folderListing += `${linePrefix}${displayName}/\n` } else { - folderContent += `${linePrefix}${displayName}\n` + folderListing += `${linePrefix}${displayName}\n` } } - const fileContents = (await Promise.all(fileContentPromises)).filter((content) => content) - return `${folderContent}\n${fileContents.join("\n\n")}`.trim() + + // Format folder content similar to read_file output + let content = `[read_file for folder '${mentionPath}']\nFolder listing:\n${folderListing}` + if (fileReadResults.length > 0) { + content += `\n\n--- File Contents ---\n\n${fileReadResults.join("\n\n")}` + } + + return { + type: "folder", + path: mentionPath, + content, + } } else { - return `(Failed to read contents of ${mentionPath})` + return { + type: isFolder ? "folder" : "file", + path: mentionPath, + content: `[read_file for '${mentionPath}']\nError: Unable to read (not a file or directory)`, + } } } catch (error) { - throw new Error(`Failed to access path "${mentionPath}": ${error.message}`) + const errorMsg = error instanceof Error ? error.message : String(error) + throw new Error(`Failed to access path "${mentionPath}": ${errorMsg}`) } } diff --git a/src/core/mentions/processUserContentMentions.ts b/src/core/mentions/processUserContentMentions.ts index 79911adcb91..d27f2cae66a 100644 --- a/src/core/mentions/processUserContentMentions.ts +++ b/src/core/mentions/processUserContentMentions.ts @@ -1,5 +1,5 @@ import { Anthropic } from "@anthropic-ai/sdk" -import { parseMentions, ParseMentionsResult } from "./index" +import { parseMentions, ParseMentionsResult, MentionContentBlock } from "./index" import { UrlContentFetcher } from "../../services/browser/UrlContentFetcher" import { FileContextTracker } from "../context-tracking/FileContextTracker" @@ -9,7 +9,23 @@ export interface ProcessUserContentMentionsResult { } /** - * Process mentions in user content, specifically within task and feedback tags + * Converts MentionContentBlocks to Anthropic text blocks. + * Each file/folder mention becomes a separate text block formatted + * to look like a read_file tool result. + */ +function contentBlocksToAnthropicBlocks(contentBlocks: MentionContentBlock[]): Anthropic.Messages.TextBlockParam[] { + return contentBlocks.map((block) => ({ + type: "text" as const, + text: block.content, + })) +} + +/** + * Process mentions in user content, specifically within task and feedback tags. + * + * File/folder @ mentions are now returned as separate text blocks that + * look like read_file tool results, making it clear to the model that + * the file has already been read. */ export async function processUserContentMentions({ userContent, @@ -20,7 +36,6 @@ export async function processUserContentMentions({ showRooIgnoredFiles = false, includeDiagnosticMessages = true, maxDiagnosticMessages = 50, - maxReadFileLine, }: { userContent: Anthropic.Messages.ContentBlockParam[] cwd: string @@ -30,7 +45,6 @@ export async function processUserContentMentions({ showRooIgnoredFiles?: boolean includeDiagnosticMessages?: boolean maxDiagnosticMessages?: number - maxReadFileLine?: number }): Promise { // Track the first mode found from slash commands let commandMode: string | undefined @@ -58,18 +72,28 @@ export async function processUserContentMentions({ showRooIgnoredFiles, includeDiagnosticMessages, maxDiagnosticMessages, - maxReadFileLine, ) // Capture the first mode found if (!commandMode && result.mode) { commandMode = result.mode } + + // Build the blocks array: + // 1. User's text (with @ mentions replaced by clean paths) + // 2. File/folder content blocks (formatted like read_file results) + // 3. Slash command help (if any) const blocks: Anthropic.Messages.ContentBlockParam[] = [ { ...block, text: result.text, }, ] + + // Add file/folder content as separate blocks + if (result.contentBlocks.length > 0) { + blocks.push(...contentBlocksToAnthropicBlocks(result.contentBlocks)) + } + if (result.slashCommandHelp) { blocks.push({ type: "text" as const, @@ -92,30 +116,38 @@ export async function processUserContentMentions({ showRooIgnoredFiles, includeDiagnosticMessages, maxDiagnosticMessages, - maxReadFileLine, ) // Capture the first mode found if (!commandMode && result.mode) { commandMode = result.mode } + + // Build content array with file blocks included + const contentParts: Array<{ type: "text"; text: string }> = [ + { + type: "text" as const, + text: result.text, + }, + ] + + // Add file/folder content blocks + for (const contentBlock of result.contentBlocks) { + contentParts.push({ + type: "text" as const, + text: contentBlock.content, + }) + } + if (result.slashCommandHelp) { - return { - ...block, - content: [ - { - type: "text" as const, - text: result.text, - }, - { - type: "text" as const, - text: result.slashCommandHelp, - }, - ], - } + contentParts.push({ + type: "text" as const, + text: result.slashCommandHelp, + }) } + return { ...block, - content: result.text, + content: contentParts, } } @@ -134,18 +166,28 @@ export async function processUserContentMentions({ showRooIgnoredFiles, includeDiagnosticMessages, maxDiagnosticMessages, - maxReadFileLine, ) // Capture the first mode found if (!commandMode && result.mode) { commandMode = result.mode } - const blocks = [ + + // Build blocks array with file content + const blocks: Array<{ type: "text"; text: string }> = [ { ...contentBlock, text: result.text, }, ] + + // Add file/folder content blocks + for (const cb of result.contentBlocks) { + blocks.push({ + type: "text" as const, + text: cb.content, + }) + } + if (result.slashCommandHelp) { blocks.push({ type: "text" as const, diff --git a/src/core/prompts/__tests__/__snapshots__/add-custom-instructions/partial-reads-enabled.snap b/src/core/prompts/__tests__/__snapshots__/add-custom-instructions/partial-reads-enabled.snap deleted file mode 100644 index 5bed6df09d1..00000000000 --- a/src/core/prompts/__tests__/__snapshots__/add-custom-instructions/partial-reads-enabled.snap +++ /dev/null @@ -1,127 +0,0 @@ -You are Roo, an experienced technical leader who is inquisitive and an excellent planner. Your goal is to gather information and get context to create a detailed plan for accomplishing the user's task, which the user will review and approve before they switch into another mode to implement the solution. - -==== - -MARKDOWN RULES - -ALL responses MUST show ANY `language construct` OR filename reference as clickable, exactly as [`filename OR language.declaration()`](relative/file/path.ext:line); line is required for `syntax` and optional for filename links. This applies to ALL markdown responses and ALSO those in attempt_completion - -==== - -TOOL USE - -You have access to a set of tools that are executed upon the user's approval. Use the provider-native tool-calling mechanism. Do not include XML markup or examples. You must call at least one tool per assistant response. Prefer calling as many tools as are reasonably needed in a single response to reduce back-and-forth and complete tasks faster. - - # Tool Use Guidelines - -1. Assess what information you already have and what information you need to proceed with the task. -2. Choose the most appropriate tool based on the task and the tool descriptions provided. Assess if you need additional information to proceed, and which of the available tools would be most effective for gathering this information. For example using the list_files tool is more effective than running a command like `ls` in the terminal. It's critical that you think about each available tool and use the one that best fits the current step in the task. -3. If multiple actions are needed, you may use multiple tools in a single message when appropriate, or use tools iteratively across messages. Each tool use should be informed by the results of previous tool uses. Do not assume the outcome of any tool use. Each step must be informed by the previous step's result. - -By carefully considering the user's response after tool executions, you can react accordingly and make informed decisions about how to proceed with the task. This iterative process helps ensure the overall success and accuracy of your work. - -==== - -CAPABILITIES - -- You have access to tools that let you execute CLI commands on the user's computer, list files, view source code definitions, regex search, read and write files, and ask follow-up questions. These tools help you effectively accomplish a wide range of tasks, such as writing code, making edits or improvements to existing files, understanding the current state of a project, performing system operations, and much more. -- When the user initially gives you a task, a recursive list of all filepaths in the current workspace directory ('/test/path') will be included in environment_details. This provides an overview of the project's file structure, offering key insights into the project from directory/file names (how developers conceptualize and organize their code) and file extensions (the language used). This can also guide decision-making on which files to explore further. If you need to further explore directories such as outside the current workspace directory, you can use the list_files tool. If you pass 'true' for the recursive parameter, it will list files recursively. Otherwise, it will list files at the top level, which is better suited for generic directories where you don't necessarily need the nested structure, like the Desktop. -- You can use the execute_command tool to run commands on the user's computer whenever you feel it can help accomplish the user's task. When you need to execute a CLI command, you must provide a clear explanation of what the command does. Prefer to execute complex CLI commands over creating executable scripts, since they are more flexible and easier to run. Interactive and long-running commands are allowed, since the commands are run in the user's VSCode terminal. The user may keep commands running in the background and you will be kept updated on their status along the way. Each command you execute is run in a new terminal instance. - -==== - -MODES - -- Test modes section - -==== - -RULES - -- The project base directory is: /test/path -- All file paths must be relative to this directory. However, commands may change directories in terminals, so respect working directory specified by the response to execute_command. -- You cannot `cd` into a different directory to complete a task. You are stuck operating from '/test/path', so be sure to pass in the correct 'path' parameter when using tools that require a path. -- Do not use the ~ character or $HOME to refer to the home directory. -- Before using the execute_command tool, you must first think about the SYSTEM INFORMATION context provided to understand the user's environment and tailor your commands to ensure they are compatible with their system. You must also consider if the command you need to run should be executed in a specific directory outside of the current working directory '/test/path', and if so prepend with `cd`'ing into that directory && then executing the command (as one command since you are stuck operating from '/test/path'). For example, if you needed to run `npm install` in a project outside of '/test/path', you would need to prepend with a `cd` i.e. pseudocode for this would be `cd (path to project) && (command, in this case npm install)`. -- Some modes have restrictions on which files they can edit. If you attempt to edit a restricted file, the operation will be rejected with a FileRestrictionError that will specify which file patterns are allowed for the current mode. -- Be sure to consider the type of project (e.g. Python, JavaScript, web application) when determining the appropriate structure and files to include. Also consider what files may be most relevant to accomplishing the task, for example looking at a project's manifest file would help you understand the project's dependencies, which you could incorporate into any code you write. - * For example, in architect mode trying to edit app.js would be rejected because architect mode can only edit files matching "\.md$" -- When making changes to code, always consider the context in which the code is being used. Ensure that your changes are compatible with the existing codebase and that they follow the project's coding standards and best practices. -- Do not ask for more information than necessary. Use the tools provided to accomplish the user's request efficiently and effectively. When you've completed your task, you must use the attempt_completion tool to present the result to the user. The user may provide feedback, which you can use to make improvements and try again. -- You are only allowed to ask the user questions using the ask_followup_question tool. Use this tool only when you need additional details to complete a task, and be sure to use a clear and concise question that will help you move forward with the task. When you ask a question, provide the user with 2-4 suggested answers based on your question so they don't need to do so much typing. The suggestions should be specific, actionable, and directly related to the completed task. They should be ordered by priority or logical sequence. However if you can use the available tools to avoid having to ask the user questions, you should do so. For example, if the user mentions a file that may be in an outside directory like the Desktop, you should use the list_files tool to list the files in the Desktop and check if the file they are talking about is there, rather than asking the user to provide the file path themselves. -- When executing commands, if you don't see the expected output, assume the terminal executed the command successfully and proceed with the task. The user's terminal may be unable to stream the output back properly. If you absolutely need to see the actual terminal output, use the ask_followup_question tool to request the user to copy and paste it back to you. -- The user may provide a file's contents directly in their message, in which case you shouldn't use the read_file tool to get the file contents again since you already have it. -- Your goal is to try to accomplish the user's task, NOT engage in a back and forth conversation. -- NEVER end attempt_completion result with a question or request to engage in further conversation! Formulate the end of your result in a way that is final and does not require further input from the user. -- You are STRICTLY FORBIDDEN from starting your messages with "Great", "Certainly", "Okay", "Sure". You should NOT be conversational in your responses, but rather direct and to the point. For example you should NOT say "Great, I've updated the CSS" but instead something like "I've updated the CSS". It is important you be clear and technical in your messages. -- When presented with images, utilize your vision capabilities to thoroughly examine them and extract meaningful information. Incorporate these insights into your thought process as you accomplish the user's task. -- At the end of each user message, you will automatically receive environment_details. This information is not written by the user themselves, but is auto-generated to provide potentially relevant context about the project structure and environment. While this information can be valuable for understanding the project context, do not treat it as a direct part of the user's request or response. Use it to inform your actions and decisions, but don't assume the user is explicitly asking about or referring to this information unless they clearly do so in their message. When using environment_details, explain your actions clearly to ensure the user understands, as they may not be aware of these details. -- Before executing commands, check the "Actively Running Terminals" section in environment_details. If present, consider how these active processes might impact your task. For example, if a local development server is already running, you wouldn't need to start it again. If no active terminals are listed, proceed with command execution as normal. -- MCP operations should be used one at a time, similar to other tool usage. Wait for confirmation of success before proceeding with additional operations. -- It is critical you wait for the user's response after each tool use, in order to confirm the success of the tool use. For example, if asked to make a todo app, you would create a file, wait for the user's response it was created successfully, then create another file if needed, wait for the user's response it was created successfully, etc. - -==== - -SYSTEM INFORMATION - -Operating System: Linux -Default Shell: /bin/zsh -Home Directory: /home/user -Current Workspace Directory: /test/path - -The Current Workspace Directory is the active VS Code project directory, and is therefore the default directory for all tool operations. New terminals will be created in the current workspace directory, however if you change directories in a terminal it will then have a different working directory; changing directories in a terminal does not modify the workspace directory, because you do not have access to change the workspace directory. When the user initially gives you a task, a recursive list of all filepaths in the current workspace directory ('/test/path') will be included in environment_details. This provides an overview of the project's file structure, offering key insights into the project from directory/file names (how developers conceptualize and organize their code) and file extensions (the language used). This can also guide decision-making on which files to explore further. If you need to further explore directories such as outside the current workspace directory, you can use the list_files tool. If you pass 'true' for the recursive parameter, it will list files recursively. Otherwise, it will list files at the top level, which is better suited for generic directories where you don't necessarily need the nested structure, like the Desktop. - -==== - -OBJECTIVE - -You accomplish a given task iteratively, breaking it down into clear steps and working through them methodically. - -1. Analyze the user's task and set clear, achievable goals to accomplish it. Prioritize these goals in a logical order. -2. Work through these goals sequentially, utilizing available tools one at a time as necessary. Each goal should correspond to a distinct step in your problem-solving process. You will be informed on the work completed and what's remaining as you go. -3. Remember, you have extensive capabilities with access to a wide range of tools that can be used in powerful and clever ways as necessary to accomplish each goal. Before calling a tool, do some analysis. First, analyze the file structure provided in environment_details to gain context and insights for proceeding effectively. Next, think about which of the provided tools is the most relevant tool to accomplish the user's task. Go through each of the required parameters of the relevant tool and determine if the user has directly provided or given enough information to infer a value. When deciding if the parameter can be inferred, carefully consider all the context to see if it supports a specific value. If all of the required parameters are present or can be reasonably inferred, proceed with the tool use. BUT, if one of the values for a required parameter is missing, DO NOT invoke the tool (not even with fillers for the missing params) and instead, ask the user to provide the missing parameters using the ask_followup_question tool. DO NOT ask for more information on optional parameters if it is not provided. -4. Once you've completed the user's task, you must use the attempt_completion tool to present the result of the task to the user. -5. The user may provide feedback, which you can use to make improvements and try again. But DO NOT continue in pointless back and forth conversations, i.e. don't end your responses with questions or offers for further assistance. - - -==== - -USER'S CUSTOM INSTRUCTIONS - -The following additional instructions are provided by the user, and should be followed to the best of your ability without interfering with the TOOL USE guidelines. - -Language Preference: -You should always speak and think in the "en" language. - -Mode-specific Instructions: -1. Do some information gathering (using provided tools) to get more context about the task. - -2. You should also ask the user clarifying questions to get a better understanding of the task. - -3. Once you've gained more context about the user's request, break down the task into clear, actionable steps and create a todo list using the `update_todo_list` tool. Each todo item should be: - - Specific and actionable - - Listed in logical execution order - - Focused on a single, well-defined outcome - - Clear enough that another mode could execute it independently - - **Note:** If the `update_todo_list` tool is not available, write the plan to a markdown file (e.g., `plan.md` or `todo.md`) instead. - -4. As you gather more information or discover new requirements, update the todo list to reflect the current understanding of what needs to be accomplished. - -5. Ask the user if they are pleased with this plan, or if they would like to make any changes. Think of this as a brainstorming session where you can discuss the task and refine the todo list. - -6. Include Mermaid diagrams if they help clarify complex workflows or system architecture. Please avoid using double quotes ("") and parentheses () inside square brackets ([]) in Mermaid diagrams, as this can cause parsing errors. - -7. Use the switch_mode tool to request that the user switch to another mode to implement the solution. - -**IMPORTANT: Focus on creating clear, actionable todo lists rather than lengthy markdown documents. Use the todo list as your primary planning tool to track and organize the work that needs to be done.** - -**CRITICAL: Never provide level of effort time estimates (e.g., hours, days, weeks) for tasks. Focus solely on breaking down the work into clear, actionable steps without estimating how long they will take.** - -Unless told otherwise, if you want to save a plan file, put it in the /plans directory - -Rules: -# Rules from .clinerules-architect: -Mock mode-specific rules -# Rules from .clinerules: -Mock generic rules \ No newline at end of file diff --git a/src/core/prompts/__tests__/add-custom-instructions.spec.ts b/src/core/prompts/__tests__/add-custom-instructions.spec.ts index b7813d0f5b8..f10a8bade56 100644 --- a/src/core/prompts/__tests__/add-custom-instructions.spec.ts +++ b/src/core/prompts/__tests__/add-custom-instructions.spec.ts @@ -264,27 +264,6 @@ describe("addCustomInstructions", () => { expect(prompt).toMatchFileSnapshot("./__snapshots__/add-custom-instructions/mcp-server-creation-disabled.snap") }) - it("should include partial read instructions when partialReadsEnabled is true", async () => { - const prompt = await SYSTEM_PROMPT( - mockContext, - "/test/path", - false, // supportsImages - undefined, // mcpHub - undefined, // diffStrategy - undefined, // browserViewportSize - defaultModeSlug, // mode - undefined, // customModePrompts - undefined, // customModes, - undefined, // globalCustomInstructions - undefined, // experiments - undefined, // language - undefined, // rooIgnoreInstructions - true, // partialReadsEnabled - ) - - expect(prompt).toMatchFileSnapshot("./__snapshots__/add-custom-instructions/partial-reads-enabled.snap") - }) - it("should prioritize mode-specific rules for code mode", async () => { const instructions = await addCustomInstructions("", "", "/test/path", defaultModeSlug) expect(instructions).toMatchFileSnapshot("./__snapshots__/add-custom-instructions/code-mode-rules.snap") diff --git a/src/core/prompts/__tests__/sections.spec.ts b/src/core/prompts/__tests__/sections.spec.ts index 011b279698e..dbfa7cf137e 100644 --- a/src/core/prompts/__tests__/sections.spec.ts +++ b/src/core/prompts/__tests__/sections.spec.ts @@ -70,7 +70,6 @@ describe("getRulesSection", () => { it("includes vendor confidentiality section when isStealthModel is true", () => { const settings = { - maxConcurrentFileReads: 5, todoListEnabled: true, useAgentRules: true, newTaskRequireTodos: false, @@ -88,7 +87,6 @@ describe("getRulesSection", () => { it("excludes vendor confidentiality section when isStealthModel is false", () => { const settings = { - maxConcurrentFileReads: 5, todoListEnabled: true, useAgentRules: true, newTaskRequireTodos: false, @@ -103,7 +101,6 @@ describe("getRulesSection", () => { it("excludes vendor confidentiality section when isStealthModel is undefined", () => { const settings = { - maxConcurrentFileReads: 5, todoListEnabled: true, useAgentRules: true, newTaskRequireTodos: false, diff --git a/src/core/prompts/__tests__/system-prompt.spec.ts b/src/core/prompts/__tests__/system-prompt.spec.ts index 91fb9350b4c..612783b3db3 100644 --- a/src/core/prompts/__tests__/system-prompt.spec.ts +++ b/src/core/prompts/__tests__/system-prompt.spec.ts @@ -228,7 +228,6 @@ describe("SYSTEM_PROMPT", () => { experiments, undefined, // language undefined, // rooIgnoreInstructions - undefined, // partialReadsEnabled ) expect(prompt).toMatchFileSnapshot("./__snapshots__/system-prompt/consistent-system-prompt.snap") @@ -249,7 +248,6 @@ describe("SYSTEM_PROMPT", () => { experiments, undefined, // language undefined, // rooIgnoreInstructions - undefined, // partialReadsEnabled ) expect(prompt).toMatchFileSnapshot("./__snapshots__/system-prompt/with-computer-use-support.snap") @@ -272,7 +270,6 @@ describe("SYSTEM_PROMPT", () => { experiments, undefined, // language undefined, // rooIgnoreInstructions - undefined, // partialReadsEnabled ) expect(prompt).toMatchFileSnapshot("./__snapshots__/system-prompt/with-mcp-hub-provided.snap") @@ -293,7 +290,6 @@ describe("SYSTEM_PROMPT", () => { experiments, undefined, // language undefined, // rooIgnoreInstructions - undefined, // partialReadsEnabled ) expect(prompt).toMatchFileSnapshot("./__snapshots__/system-prompt/with-undefined-mcp-hub.snap") @@ -314,7 +310,6 @@ describe("SYSTEM_PROMPT", () => { experiments, undefined, // language undefined, // rooIgnoreInstructions - undefined, // partialReadsEnabled ) expect(prompt).toMatchFileSnapshot("./__snapshots__/system-prompt/with-different-viewport-size.snap") @@ -362,7 +357,6 @@ describe("SYSTEM_PROMPT", () => { undefined, // experiments undefined, // language undefined, // rooIgnoreInstructions - undefined, // partialReadsEnabled ) expect(prompt).toContain("Language Preference:") @@ -421,7 +415,6 @@ describe("SYSTEM_PROMPT", () => { experiments, undefined, // language undefined, // rooIgnoreInstructions - undefined, // partialReadsEnabled ) // Role definition should be at the top @@ -457,7 +450,6 @@ describe("SYSTEM_PROMPT", () => { undefined, // experiments undefined, // language undefined, // rooIgnoreInstructions - undefined, // partialReadsEnabled ) // Role definition from promptComponent should be at the top @@ -488,7 +480,6 @@ describe("SYSTEM_PROMPT", () => { undefined, // experiments undefined, // language undefined, // rooIgnoreInstructions - undefined, // partialReadsEnabled ) // Should use the default mode's role definition @@ -497,7 +488,6 @@ describe("SYSTEM_PROMPT", () => { it("should exclude update_todo_list tool when todoListEnabled is false", async () => { const settings = { - maxConcurrentFileReads: 5, todoListEnabled: false, useAgentRules: true, newTaskRequireTodos: false, @@ -517,7 +507,6 @@ describe("SYSTEM_PROMPT", () => { experiments, undefined, // language undefined, // rooIgnoreInstructions - undefined, // partialReadsEnabled settings, // settings ) @@ -528,7 +517,6 @@ describe("SYSTEM_PROMPT", () => { it("should include update_todo_list tool when todoListEnabled is true", async () => { const settings = { - maxConcurrentFileReads: 5, todoListEnabled: true, useAgentRules: true, newTaskRequireTodos: false, @@ -548,7 +536,6 @@ describe("SYSTEM_PROMPT", () => { experiments, undefined, // language undefined, // rooIgnoreInstructions - undefined, // partialReadsEnabled settings, // settings ) @@ -559,7 +546,6 @@ describe("SYSTEM_PROMPT", () => { it("should include update_todo_list tool when todoListEnabled is undefined", async () => { const settings = { - maxConcurrentFileReads: 5, todoListEnabled: true, useAgentRules: true, newTaskRequireTodos: false, @@ -579,7 +565,6 @@ describe("SYSTEM_PROMPT", () => { experiments, undefined, // language undefined, // rooIgnoreInstructions - undefined, // partialReadsEnabled settings, // settings ) @@ -590,7 +575,6 @@ describe("SYSTEM_PROMPT", () => { it("should include native tool instructions", async () => { const settings = { - maxConcurrentFileReads: 5, todoListEnabled: true, useAgentRules: true, newTaskRequireTodos: false, @@ -610,7 +594,6 @@ describe("SYSTEM_PROMPT", () => { experiments, undefined, // language undefined, // rooIgnoreInstructions - undefined, // partialReadsEnabled settings, // settings ) diff --git a/src/core/prompts/sections/__tests__/custom-instructions.spec.ts b/src/core/prompts/sections/__tests__/custom-instructions.spec.ts index 260cb221033..ea3b31ee938 100644 --- a/src/core/prompts/sections/__tests__/custom-instructions.spec.ts +++ b/src/core/prompts/sections/__tests__/custom-instructions.spec.ts @@ -543,7 +543,6 @@ describe("addCustomInstructions", () => { "test-mode", { settings: { - maxConcurrentFileReads: 5, todoListEnabled: true, useAgentRules: true, newTaskRequireTodos: false, @@ -575,7 +574,6 @@ describe("addCustomInstructions", () => { "test-mode", { settings: { - maxConcurrentFileReads: 5, todoListEnabled: true, useAgentRules: false, newTaskRequireTodos: false, @@ -636,7 +634,6 @@ describe("addCustomInstructions", () => { "test-mode", { settings: { - maxConcurrentFileReads: 5, todoListEnabled: true, useAgentRules: true, newTaskRequireTodos: false, @@ -682,7 +679,6 @@ describe("addCustomInstructions", () => { "test-mode", { settings: { - maxConcurrentFileReads: 5, todoListEnabled: true, useAgentRules: true, newTaskRequireTodos: false, @@ -750,7 +746,6 @@ describe("addCustomInstructions", () => { "test-mode", { settings: { - maxConcurrentFileReads: 5, todoListEnabled: true, useAgentRules: true, newTaskRequireTodos: false, @@ -802,7 +797,6 @@ describe("addCustomInstructions", () => { "test-mode", { settings: { - maxConcurrentFileReads: 5, todoListEnabled: true, useAgentRules: true, newTaskRequireTodos: false, @@ -856,7 +850,6 @@ describe("addCustomInstructions", () => { "test-mode", { settings: { - maxConcurrentFileReads: 5, todoListEnabled: true, useAgentRules: true, newTaskRequireTodos: false, @@ -902,7 +895,6 @@ describe("addCustomInstructions", () => { "test-mode", { settings: { - maxConcurrentFileReads: 5, todoListEnabled: true, useAgentRules: true, newTaskRequireTodos: false, diff --git a/src/core/prompts/system.ts b/src/core/prompts/system.ts index 4b66b36be35..1731952a4eb 100644 --- a/src/core/prompts/system.ts +++ b/src/core/prompts/system.ts @@ -55,7 +55,6 @@ async function generatePrompt( experiments?: Record, language?: string, rooIgnoreInstructions?: string, - partialReadsEnabled?: boolean, settings?: SystemPromptSettings, todoList?: TodoItem[], modelId?: string, @@ -128,7 +127,6 @@ export const SYSTEM_PROMPT = async ( experiments?: Record, language?: string, rooIgnoreInstructions?: string, - partialReadsEnabled?: boolean, settings?: SystemPromptSettings, todoList?: TodoItem[], modelId?: string, @@ -196,7 +194,6 @@ ${customInstructions}` experiments, language, rooIgnoreInstructions, - partialReadsEnabled, settings, todoList, modelId, diff --git a/src/core/prompts/tools/native-tools/__tests__/converters.spec.ts b/src/core/prompts/tools/native-tools/__tests__/converters.spec.ts index 02032346b2a..dfef164659b 100644 --- a/src/core/prompts/tools/native-tools/__tests__/converters.spec.ts +++ b/src/core/prompts/tools/native-tools/__tests__/converters.spec.ts @@ -80,27 +80,27 @@ describe("converters", () => { const openAITool: OpenAI.Chat.ChatCompletionTool = { type: "function", function: { - name: "read_file", - description: "Read files", + name: "process_data", + description: "Process data with filters", parameters: { type: "object", properties: { - files: { + items: { type: "array", items: { type: "object", properties: { - path: { type: "string" }, - line_ranges: { + name: { type: "string" }, + tags: { type: ["array", "null"], - items: { type: "string", pattern: "^[0-9]+-[0-9]+$" }, + items: { type: "string" }, }, }, - required: ["path", "line_ranges"], + required: ["name"], }, }, }, - required: ["files"], + required: ["items"], additionalProperties: false, }, }, diff --git a/src/core/prompts/tools/native-tools/__tests__/read_file.spec.ts b/src/core/prompts/tools/native-tools/__tests__/read_file.spec.ts index 9561fe417d0..dded7fba50b 100644 --- a/src/core/prompts/tools/native-tools/__tests__/read_file.spec.ts +++ b/src/core/prompts/tools/native-tools/__tests__/read_file.spec.ts @@ -1,5 +1,5 @@ import type OpenAI from "openai" -import { createReadFileTool, type ReadFileToolOptions } from "../read_file" +import { createReadFileTool } from "../read_file" // Helper type to access function tools type FunctionTool = OpenAI.Chat.ChatCompletionTool & { type: "function" } @@ -8,91 +8,46 @@ type FunctionTool = OpenAI.Chat.ChatCompletionTool & { type: "function" } const getFunctionDef = (tool: OpenAI.Chat.ChatCompletionTool) => (tool as FunctionTool).function describe("createReadFileTool", () => { - describe("maxConcurrentFileReads documentation", () => { - it("should include default maxConcurrentFileReads limit (5) in description", () => { + describe("single-file-per-call documentation", () => { + it("should indicate single-file-per-call and suggest parallel tool calls", () => { const tool = createReadFileTool() const description = getFunctionDef(tool).description - expect(description).toContain("maximum of 5 files") - expect(description).toContain("If you need to read more files, use multiple sequential read_file requests") - }) - - it("should include custom maxConcurrentFileReads limit in description", () => { - const tool = createReadFileTool({ maxConcurrentFileReads: 3 }) - const description = getFunctionDef(tool).description - - expect(description).toContain("maximum of 3 files") - expect(description).toContain("within 3-file limit") - }) - - it("should indicate single file reads only when maxConcurrentFileReads is 1", () => { - const tool = createReadFileTool({ maxConcurrentFileReads: 1 }) - const description = getFunctionDef(tool).description - - expect(description).toContain("Multiple file reads are currently disabled") - expect(description).toContain("only read one file at a time") - expect(description).not.toContain("Example multiple files") - }) - - it("should use singular 'Read a file' in base description when maxConcurrentFileReads is 1", () => { - const tool = createReadFileTool({ maxConcurrentFileReads: 1 }) - const description = getFunctionDef(tool).description - - expect(description).toMatch(/^Read a file/) - expect(description).not.toContain("Read one or more files") - }) - - it("should use plural 'Read one or more files' in base description when maxConcurrentFileReads is > 1", () => { - const tool = createReadFileTool({ maxConcurrentFileReads: 5 }) - const description = getFunctionDef(tool).description - - expect(description).toMatch(/^Read one or more files/) - }) - - it("should not show multiple files example when maxConcurrentFileReads is 1", () => { - const tool = createReadFileTool({ maxConcurrentFileReads: 1, partialReadsEnabled: true }) - const description = getFunctionDef(tool).description - - expect(description).not.toContain("Example multiple files") - }) - - it("should show multiple files example when maxConcurrentFileReads is > 1", () => { - const tool = createReadFileTool({ maxConcurrentFileReads: 5, partialReadsEnabled: true }) - const description = getFunctionDef(tool).description - - expect(description).toContain("Example multiple files") + expect(description).toContain("exactly one file per call") + expect(description).toContain("multiple parallel read_file calls") }) }) - describe("partialReadsEnabled option", () => { - it("should include line_ranges in description when partialReadsEnabled is true", () => { - const tool = createReadFileTool({ partialReadsEnabled: true }) + describe("indentation mode", () => { + it("should always include indentation mode in description", () => { + const tool = createReadFileTool() const description = getFunctionDef(tool).description - expect(description).toContain("line_ranges") - expect(description).toContain("Example with line ranges") + expect(description).toContain("indentation") }) - it("should not include line_ranges in description when partialReadsEnabled is false", () => { - const tool = createReadFileTool({ partialReadsEnabled: false }) - const description = getFunctionDef(tool).description + it("should always include indentation parameter in schema", () => { + const tool = createReadFileTool() + const schema = getFunctionDef(tool).parameters as any - expect(description).not.toContain("line_ranges") - expect(description).not.toContain("Example with line ranges") + expect(schema.properties).toHaveProperty("indentation") }) - it("should include line_ranges parameter in schema when partialReadsEnabled is true", () => { - const tool = createReadFileTool({ partialReadsEnabled: true }) + it("should include mode parameter in schema", () => { + const tool = createReadFileTool() const schema = getFunctionDef(tool).parameters as any - expect(schema.properties.files.items.properties).toHaveProperty("line_ranges") + expect(schema.properties).toHaveProperty("mode") + expect(schema.properties.mode.enum).toContain("slice") + expect(schema.properties.mode.enum).toContain("indentation") }) - it("should not include line_ranges parameter in schema when partialReadsEnabled is false", () => { - const tool = createReadFileTool({ partialReadsEnabled: false }) + it("should include offset and limit parameters in schema", () => { + const tool = createReadFileTool() const schema = getFunctionDef(tool).parameters as any - expect(schema.properties.files.items.properties).not.toHaveProperty("line_ranges") + expect(schema.properties).toHaveProperty("offset") + expect(schema.properties).toHaveProperty("limit") }) }) @@ -138,75 +93,6 @@ describe("createReadFileTool", () => { }) }) - describe("combined options", () => { - it("should correctly combine low maxConcurrentFileReads with partialReadsEnabled", () => { - const tool = createReadFileTool({ - maxConcurrentFileReads: 2, - partialReadsEnabled: true, - }) - const description = getFunctionDef(tool).description - - expect(description).toContain("maximum of 2 files") - expect(description).toContain("line_ranges") - expect(description).toContain("within 2-file limit") - }) - - it("should correctly handle maxConcurrentFileReads of 1 with partialReadsEnabled false", () => { - const tool = createReadFileTool({ - maxConcurrentFileReads: 1, - partialReadsEnabled: false, - }) - const description = getFunctionDef(tool).description - - expect(description).toContain("only read one file at a time") - expect(description).not.toContain("line_ranges") - expect(description).not.toContain("Example multiple files") - }) - - it("should correctly combine partialReadsEnabled and supportsImages", () => { - const tool = createReadFileTool({ - partialReadsEnabled: true, - supportsImages: true, - }) - const description = getFunctionDef(tool).description - - // Should have both line_ranges and image support - expect(description).toContain("line_ranges") - expect(description).toContain( - "Automatically processes and returns image files (PNG, JPG, JPEG, GIF, BMP, SVG, WEBP, ICO, AVIF) for visual analysis", - ) - }) - - it("should work with partialReadsEnabled=false and supportsImages=true", () => { - const tool = createReadFileTool({ - partialReadsEnabled: false, - supportsImages: true, - }) - const description = getFunctionDef(tool).description - - // Should have image support but no line_ranges - expect(description).not.toContain("line_ranges") - expect(description).toContain( - "Automatically processes and returns image files (PNG, JPG, JPEG, GIF, BMP, SVG, WEBP, ICO, AVIF) for visual analysis", - ) - }) - - it("should correctly combine all three options", () => { - const tool = createReadFileTool({ - maxConcurrentFileReads: 3, - partialReadsEnabled: true, - supportsImages: true, - }) - const description = getFunctionDef(tool).description - - expect(description).toContain("maximum of 3 files") - expect(description).toContain("line_ranges") - expect(description).toContain( - "Automatically processes and returns image files (PNG, JPG, JPEG, GIF, BMP, SVG, WEBP, ICO, AVIF) for visual analysis", - ) - }) - }) - describe("tool structure", () => { it("should have correct tool name", () => { const tool = createReadFileTool() @@ -226,18 +112,11 @@ describe("createReadFileTool", () => { expect(getFunctionDef(tool).strict).toBe(true) }) - it("should require files parameter", () => { + it("should require path parameter", () => { const tool = createReadFileTool() const schema = getFunctionDef(tool).parameters as any - expect(schema.required).toContain("files") - }) - - it("should require path in file objects", () => { - const tool = createReadFileTool({ partialReadsEnabled: false }) - const schema = getFunctionDef(tool).parameters as any - - expect(schema.properties.files.items.required).toContain("path") + expect(schema.required).toContain("path") }) }) }) diff --git a/src/core/prompts/tools/native-tools/index.ts b/src/core/prompts/tools/native-tools/index.ts index f23a7b2f28f..1c0825233e0 100644 --- a/src/core/prompts/tools/native-tools/index.ts +++ b/src/core/prompts/tools/native-tools/index.ts @@ -30,10 +30,6 @@ export type { ReadFileToolOptions } from "./read_file" * Options for customizing the native tools array. */ export interface NativeToolsOptions { - /** Whether to include line_ranges support in read_file tool (default: true) */ - partialReadsEnabled?: boolean - /** Maximum number of files that can be read in a single read_file request (default: 5) */ - maxConcurrentFileReads?: number /** Whether the model supports image processing (default: false) */ supportsImages?: boolean } @@ -45,11 +41,9 @@ export interface NativeToolsOptions { * @returns Array of native tool definitions */ export function getNativeTools(options: NativeToolsOptions = {}): OpenAI.Chat.ChatCompletionTool[] { - const { partialReadsEnabled = true, maxConcurrentFileReads = 5, supportsImages = false } = options + const { supportsImages = false } = options const readFileOptions: ReadFileToolOptions = { - partialReadsEnabled, - maxConcurrentFileReads, supportsImages, } diff --git a/src/core/prompts/tools/native-tools/read_file.ts b/src/core/prompts/tools/native-tools/read_file.ts index 7171be0f1d6..08d7e252967 100644 --- a/src/core/prompts/tools/native-tools/read_file.ts +++ b/src/core/prompts/tools/native-tools/read_file.ts @@ -1,5 +1,18 @@ import type OpenAI from "openai" +// ─── Constants ──────────────────────────────────────────────────────────────── + +/** Default maximum lines to return per file (Codex-inspired predictable limit) */ +export const DEFAULT_LINE_LIMIT = 2000 + +/** Maximum characters per line before truncation */ +export const MAX_LINE_LENGTH = 500 + +/** Default indentation levels to include above anchor (0 = unlimited) */ +export const DEFAULT_MAX_LEVELS = 0 + +// ─── Helper Functions ───────────────────────────────────────────────────────── + /** * Generates the file support note, optionally including image format support. * @@ -13,86 +26,115 @@ function getReadFileSupportsNote(supportsImages: boolean): string { return `Supports text extraction from PDF and DOCX files, but may not handle other binary files properly.` } +// ─── Types ──────────────────────────────────────────────────────────────────── + /** * Options for creating the read_file tool definition. */ export interface ReadFileToolOptions { - /** Whether to include line_ranges parameter (default: true) */ - partialReadsEnabled?: boolean - /** Maximum number of files that can be read in a single request (default: 5) */ - maxConcurrentFileReads?: number /** Whether the model supports image processing (default: false) */ supportsImages?: boolean } +// ─── Schema Builder ─────────────────────────────────────────────────────────── + /** - * Creates the read_file tool definition, optionally including line_ranges support - * based on whether partial reads are enabled. + * Creates the read_file tool definition with Codex-inspired modes. + * + * Two reading modes are supported: + * + * 1. **Slice Mode** (default): Simple offset/limit reading + * - Reads contiguous lines starting from `offset` (1-based, default: 1) + * - Limited to `limit` lines (default: 2000) + * - Predictable and efficient for agent planning + * + * 2. **Indentation Mode**: Semantic code block extraction + * - Anchored on a specific line number (1-based) + * - Extracts the block containing that line plus context + * - Respects code structure based on indentation hierarchy + * - Useful for extracting functions, classes, or logical blocks * * @param options - Configuration options for the tool * @returns Native tool definition for read_file */ export function createReadFileTool(options: ReadFileToolOptions = {}): OpenAI.Chat.ChatCompletionTool { - const { partialReadsEnabled = true, maxConcurrentFileReads = 5, supportsImages = false } = options - const isMultipleReadsEnabled = maxConcurrentFileReads > 1 + const { supportsImages = false } = options - // Build description intro with concurrent reads limit message - const descriptionIntro = isMultipleReadsEnabled - ? `Read one or more files and return their contents with line numbers for diffing or discussion. IMPORTANT: You can read a maximum of ${maxConcurrentFileReads} files in a single request. If you need to read more files, use multiple sequential read_file requests. ` - : "Read a file and return its contents with line numbers for diffing or discussion. IMPORTANT: Multiple file reads are currently disabled. You can only read one file at a time. " + // Build description based on capabilities + const descriptionIntro = + "Read a file and return its contents with line numbers for diffing or discussion. IMPORTANT: This tool reads exactly one file per call. If you need multiple files, issue multiple parallel read_file calls." - const baseDescription = - descriptionIntro + - "Structure: { files: [{ path: 'relative/path.ts'" + - (partialReadsEnabled ? ", line_ranges: [[1, 50], [100, 150]]" : "") + - " }] }. " + - "The 'path' is required and relative to workspace. " - - const optionalRangesDescription = partialReadsEnabled - ? "The 'line_ranges' is optional for reading specific sections. Each range is a [start, end] tuple (1-based inclusive). " - : "" - - const examples = partialReadsEnabled - ? "Example single file: { files: [{ path: 'src/app.ts' }] }. " + - "Example with line ranges: { files: [{ path: 'src/app.ts', line_ranges: [[1, 50], [100, 150]] }] }. " + - (isMultipleReadsEnabled - ? `Example multiple files (within ${maxConcurrentFileReads}-file limit): { files: [{ path: 'file1.ts', line_ranges: [[1, 50]] }, { path: 'file2.ts' }] }` - : "") - : "Example single file: { files: [{ path: 'src/app.ts' }] }. " + - (isMultipleReadsEnabled - ? `Example multiple files (within ${maxConcurrentFileReads}-file limit): { files: [{ path: 'file1.ts' }, { path: 'file2.ts' }] }` - : "") + const modeDescription = + ` Supports two modes: 'slice' (default) reads lines sequentially with offset/limit; 'indentation' extracts semantic code blocks around an anchor line based on indentation hierarchy.` + + ` Use slice mode when exploring a file from the beginning, reading configuration files, or when you don't have a specific line number to target.` + + ` Use indentation mode when you have a specific line number from search results, error messages, or definition lookups and want the full containing function/class without truncation.` + + const limitNote = ` By default, returns up to ${DEFAULT_LINE_LIMIT} lines per file. Lines longer than ${MAX_LINE_LENGTH} characters are truncated.` const description = - baseDescription + optionalRangesDescription + getReadFileSupportsNote(supportsImages) + " " + examples + descriptionIntro + + modeDescription + + limitNote + + " " + + getReadFileSupportsNote(supportsImages) + + ` Example: { path: 'src/app.ts' }` + + ` Example (indentation mode): { path: 'src/app.ts', mode: 'indentation', indentation: { anchor_line: 42 } }` - // Build the properties object conditionally - const fileProperties: Record = { + const indentationProperties: Record = { + anchor_line: { + type: "integer", + description: + "1-based line number to anchor the extraction (required for indentation mode). The complete containing function, method, or class will be extracted with proper context. Typically obtained from search results, error stack traces, or definition lookups. If you don't have a specific line number, use slice mode instead.", + }, + max_levels: { + type: "integer", + description: `Maximum indentation levels to include above the anchor (indentation mode, 0 = unlimited (default)). Higher values include more parent context.`, + }, + include_siblings: { + type: "boolean", + description: + "Include sibling blocks at the same indentation level as the anchor block (indentation mode, default: false). Useful for seeing related methods in a class.", + }, + include_header: { + type: "boolean", + description: + "Include file header content (imports, module-level comments) at the top of output (indentation mode, default: true).", + }, + max_lines: { + type: "integer", + description: + "Hard cap on lines returned for indentation mode. Acts as a separate limit from the top-level 'limit' parameter.", + }, + } + + const properties: Record = { path: { type: "string", description: "Path to the file to read, relative to the workspace", }, - } - - // Only include line_ranges if partial reads are enabled - if (partialReadsEnabled) { - fileProperties.line_ranges = { - type: ["array", "null"], + mode: { + type: "string", + enum: ["slice", "indentation"], description: - "Optional line ranges to read. Each range is a [start, end] tuple with 1-based inclusive line numbers. Use multiple ranges for non-contiguous sections.", - items: { - type: "array", - items: { type: "integer" }, - minItems: 2, - maxItems: 2, - }, - } + "Reading mode. 'slice' (default): read lines sequentially with offset/limit - use for general file exploration or when you don't have a target line number. 'indentation': extract complete semantic code blocks containing anchor_line - use when you have a line number and want the full function/class without truncation.", + }, + offset: { + type: "integer", + description: "1-based line offset to start reading from (slice mode, default: 1)", + }, + limit: { + type: "integer", + description: `Maximum number of lines to return (slice mode, default: ${DEFAULT_LINE_LIMIT})`, + }, + indentation: { + type: "object", + description: "Indentation mode options. Only used when mode='indentation'.", + properties: indentationProperties, + required: [], + additionalProperties: false, + }, } - // When using strict mode, ALL properties must be in the required array - // Optional properties are handled by having type: ["...", "null"] - const fileRequiredProperties = partialReadsEnabled ? ["path", "line_ranges"] : ["path"] - return { type: "function", function: { @@ -101,24 +143,15 @@ export function createReadFileTool(options: ReadFileToolOptions = {}): OpenAI.Ch strict: true, parameters: { type: "object", - properties: { - files: { - type: "array", - description: "List of files to read; request related files together when allowed", - items: { - type: "object", - properties: fileProperties, - required: fileRequiredProperties, - additionalProperties: false, - }, - minItems: 1, - }, - }, - required: ["files"], + properties, + required: ["path"], additionalProperties: false, }, }, } satisfies OpenAI.Chat.ChatCompletionTool } -export const read_file = createReadFileTool({ partialReadsEnabled: false }) +/** + * Default read_file tool with all parameters + */ +export const read_file = createReadFileTool() diff --git a/src/core/prompts/types.ts b/src/core/prompts/types.ts index d438735f279..ca10dc12772 100644 --- a/src/core/prompts/types.ts +++ b/src/core/prompts/types.ts @@ -2,7 +2,6 @@ * Settings passed to system prompt generation functions */ export interface SystemPromptSettings { - maxConcurrentFileReads: number todoListEnabled: boolean browserToolEnabled?: boolean useAgentRules: boolean diff --git a/src/core/task/Task.ts b/src/core/task/Task.ts index 6bc2ef4ea71..0905f6d5e8f 100644 --- a/src/core/task/Task.ts +++ b/src/core/task/Task.ts @@ -1654,8 +1654,6 @@ export class Task extends EventEmitter implements TaskLike { customModes: state?.customModes, experiments: state?.experiments, apiConfiguration, - maxReadFileLine: state?.maxReadFileLine ?? -1, - maxConcurrentFileReads: state?.maxConcurrentFileReads ?? 5, browserToolEnabled: state?.browserToolEnabled ?? true, modelInfo, includeAllToolsWithRestrictions: false, @@ -2588,7 +2586,6 @@ export class Task extends EventEmitter implements TaskLike { showRooIgnoredFiles = false, includeDiagnosticMessages = true, maxDiagnosticMessages = 50, - maxReadFileLine = -1, } = (await this.providerRef.deref()?.getState()) ?? {} const { content: parsedUserContent, mode: slashCommandMode } = await processUserContentMentions({ @@ -2600,7 +2597,6 @@ export class Task extends EventEmitter implements TaskLike { showRooIgnoredFiles, includeDiagnosticMessages, maxDiagnosticMessages, - maxReadFileLine, }) // Switch mode if specified in a slash command's frontmatter @@ -3760,8 +3756,6 @@ export class Task extends EventEmitter implements TaskLike { experiments, browserToolEnabled, language, - maxConcurrentFileReads, - maxReadFileLine, apiConfiguration, enableSubfolderRules, } = state ?? {} @@ -3798,9 +3792,7 @@ export class Task extends EventEmitter implements TaskLike { experiments, language, rooIgnoreInstructions, - maxReadFileLine !== -1, { - maxConcurrentFileReads: maxConcurrentFileReads ?? 5, todoListEnabled: apiConfiguration?.todoListEnabled ?? true, browserToolEnabled: browserToolEnabled ?? true, useAgentRules: @@ -3863,8 +3855,6 @@ export class Task extends EventEmitter implements TaskLike { customModes: state?.customModes, experiments: state?.experiments, apiConfiguration, - maxReadFileLine: state?.maxReadFileLine ?? -1, - maxConcurrentFileReads: state?.maxConcurrentFileReads ?? 5, browserToolEnabled: state?.browserToolEnabled ?? true, modelInfo, includeAllToolsWithRestrictions: false, @@ -4079,8 +4069,6 @@ export class Task extends EventEmitter implements TaskLike { customModes: state?.customModes, experiments: state?.experiments, apiConfiguration, - maxReadFileLine: state?.maxReadFileLine ?? -1, - maxConcurrentFileReads: state?.maxConcurrentFileReads ?? 5, browserToolEnabled: state?.browserToolEnabled ?? true, modelInfo, includeAllToolsWithRestrictions: false, @@ -4245,8 +4233,6 @@ export class Task extends EventEmitter implements TaskLike { customModes: state?.customModes, experiments: state?.experiments, apiConfiguration, - maxReadFileLine: state?.maxReadFileLine ?? -1, - maxConcurrentFileReads: state?.maxConcurrentFileReads ?? 5, browserToolEnabled: state?.browserToolEnabled ?? true, modelInfo, includeAllToolsWithRestrictions: supportsAllowedFunctionNames, diff --git a/src/core/task/__tests__/Task.spec.ts b/src/core/task/__tests__/Task.spec.ts index 3f0df9d24e5..2a2c87151b1 100644 --- a/src/core/task/__tests__/Task.spec.ts +++ b/src/core/task/__tests__/Task.spec.ts @@ -138,7 +138,7 @@ vi.mock("vscode", () => { vi.mock("../../mentions", () => ({ parseMentions: vi.fn().mockImplementation((text) => { - return Promise.resolve({ text: `processed: ${text}`, mode: undefined }) + return Promise.resolve({ text: `processed: ${text}`, mode: undefined, contentBlocks: [] }) }), openMention: vi.fn(), getLatestTerminalOutput: vi.fn(), diff --git a/src/core/task/__tests__/flushPendingToolResultsToHistory.spec.ts b/src/core/task/__tests__/flushPendingToolResultsToHistory.spec.ts index ca68347cbde..f4d78802d29 100644 --- a/src/core/task/__tests__/flushPendingToolResultsToHistory.spec.ts +++ b/src/core/task/__tests__/flushPendingToolResultsToHistory.spec.ts @@ -106,7 +106,7 @@ vi.mock("vscode", () => { vi.mock("../../mentions", () => ({ parseMentions: vi.fn().mockImplementation((text) => { - return Promise.resolve(`processed: ${text}`) + return Promise.resolve({ text: `processed: ${text}`, mode: undefined, contentBlocks: [] }) }), openMention: vi.fn(), getLatestTerminalOutput: vi.fn(), diff --git a/src/core/task/__tests__/grace-retry-errors.spec.ts b/src/core/task/__tests__/grace-retry-errors.spec.ts index 3c3e40b98c3..283b402f69b 100644 --- a/src/core/task/__tests__/grace-retry-errors.spec.ts +++ b/src/core/task/__tests__/grace-retry-errors.spec.ts @@ -111,7 +111,7 @@ vi.mock("vscode", () => { vi.mock("../../mentions", () => ({ parseMentions: vi.fn().mockImplementation((text) => { - return Promise.resolve(`processed: ${text}`) + return Promise.resolve({ text: `processed: ${text}`, mode: undefined, contentBlocks: [] }) }), openMention: vi.fn(), getLatestTerminalOutput: vi.fn(), diff --git a/src/core/task/__tests__/grounding-sources.test.ts b/src/core/task/__tests__/grounding-sources.test.ts index dc1212ead51..764e1ea37fb 100644 --- a/src/core/task/__tests__/grounding-sources.test.ts +++ b/src/core/task/__tests__/grounding-sources.test.ts @@ -112,7 +112,7 @@ vi.mock("fs/promises", () => ({ // Mock mentions vi.mock("../../mentions", () => ({ - parseMentions: vi.fn().mockImplementation((text) => Promise.resolve(text)), + parseMentions: vi.fn().mockImplementation((text) => Promise.resolve({ text, mode: undefined, contentBlocks: [] })), openMention: vi.fn(), getLatestTerminalOutput: vi.fn(), })) diff --git a/src/core/task/__tests__/reasoning-preservation.test.ts b/src/core/task/__tests__/reasoning-preservation.test.ts index 45fb602f665..3bf2dec2986 100644 --- a/src/core/task/__tests__/reasoning-preservation.test.ts +++ b/src/core/task/__tests__/reasoning-preservation.test.ts @@ -112,7 +112,7 @@ vi.mock("fs/promises", () => ({ // Mock mentions vi.mock("../../mentions", () => ({ - parseMentions: vi.fn().mockImplementation((text) => Promise.resolve(text)), + parseMentions: vi.fn().mockImplementation((text) => Promise.resolve({ text, mode: undefined, contentBlocks: [] })), openMention: vi.fn(), getLatestTerminalOutput: vi.fn(), })) diff --git a/src/core/task/build-tools.ts b/src/core/task/build-tools.ts index 46896d050b6..0206df71c44 100644 --- a/src/core/task/build-tools.ts +++ b/src/core/task/build-tools.ts @@ -22,8 +22,6 @@ interface BuildToolsOptions { customModes: ModeConfig[] | undefined experiments: Record | undefined apiConfiguration: ProviderSettings | undefined - maxReadFileLine: number - maxConcurrentFileReads: number browserToolEnabled: boolean modelInfo?: ModelInfo /** @@ -89,8 +87,6 @@ export async function buildNativeToolsArrayWithRestrictions(options: BuildToolsO customModes, experiments, apiConfiguration, - maxReadFileLine, - maxConcurrentFileReads, browserToolEnabled, modelInfo, includeAllToolsWithRestrictions, @@ -109,16 +105,11 @@ export async function buildNativeToolsArrayWithRestrictions(options: BuildToolsO modelInfo, } - // Determine if partial reads are enabled based on maxReadFileLine setting. - const partialReadsEnabled = maxReadFileLine !== -1 - // Check if the model supports images for read_file tool description. const supportsImages = modelInfo?.supportsImages ?? false // Build native tools with dynamic read_file tool based on settings. const nativeTools = getNativeTools({ - partialReadsEnabled, - maxConcurrentFileReads, supportsImages, }) diff --git a/src/core/tools/ReadFileTool.ts b/src/core/tools/ReadFileTool.ts index 1e20ac5cb32..557d4b68dd4 100644 --- a/src/core/tools/ReadFileTool.ts +++ b/src/core/tools/ReadFileTool.ts @@ -1,22 +1,29 @@ +/** + * ReadFileTool - Codex-inspired file reading with indentation mode support. + * + * Supports two modes: + * 1. Slice mode (default): Read contiguous lines with offset/limit + * 2. Indentation mode: Extract semantic code blocks based on indentation hierarchy + * + * Also supports legacy format for backward compatibility: + * - Legacy format: { files: [{ path: string, lineRanges?: [...] }] } + */ import path from "path" import * as fs from "fs/promises" import { isBinaryFile } from "isbinaryfile" -import type { FileEntry, LineRange } from "@roo-code/types" -import { type ClineSayTool, ANTHROPIC_DEFAULT_MAX_TOKENS } from "@roo-code/types" +import type { ReadFileParams, ReadFileMode, ReadFileToolParams, FileEntry, LineRange } from "@roo-code/types" +import { isLegacyReadFileParams, type ClineSayTool } from "@roo-code/types" import { Task } from "../task/Task" import { formatResponse } from "../prompts/responses" -import { getModelMaxOutputTokens } from "../../shared/api" -import { t } from "../../i18n" import { RecordSource } from "../context-tracking/FileContextTrackerTypes" import { isPathOutsideWorkspace } from "../../utils/pathUtils" import { getReadablePath } from "../../utils/path" -import { countFileLines } from "../../integrations/misc/line-counter" -import { readLines } from "../../integrations/misc/read-lines" import { extractTextFromFile, addLineNumbers, getSupportedBinaryFormats } from "../../integrations/misc/extract-text" -import { parseSourceCodeDefinitionsForFile } from "../../services/tree-sitter" -import type { ToolUse } from "../../shared/tools" +import { readWithIndentation, readWithSlice } from "../../integrations/misc/indentation-reader" +import { DEFAULT_LINE_LIMIT } from "../prompts/tools/native-tools/read_file" +import type { ToolUse, PushToolResult } from "../../shared/tools" import { DEFAULT_MAX_IMAGE_FILE_SIZE_MB, @@ -26,60 +33,92 @@ import { processImageFile, ImageMemoryTracker, } from "./helpers/imageHelpers" -import { FILE_READ_BUDGET_PERCENT, readFileWithTokenBudget } from "./helpers/fileTokenBudget" -import { truncateDefinitionsToLineLimit } from "./helpers/truncateDefinitions" import { BaseTool, ToolCallbacks } from "./BaseTool" +// ─── Types ──────────────────────────────────────────────────────────────────── + +/** + * Internal entry structure for tracking file read parameters. + */ +interface InternalFileEntry { + path: string + mode?: ReadFileMode + offset?: number + limit?: number + anchor_line?: number + max_levels?: number + include_siblings?: boolean + include_header?: boolean + max_lines?: number +} + interface FileResult { path: string status: "approved" | "denied" | "blocked" | "error" | "pending" content?: string error?: string notice?: string - lineRanges?: LineRange[] nativeContent?: string imageDataUrl?: string feedbackText?: string - feedbackImages?: any[] + feedbackImages?: string[] + // Store the original entry for mode processing + entry?: InternalFileEntry } +// ─── Tool Implementation ────────────────────────────────────────────────────── + export class ReadFileTool extends BaseTool<"read_file"> { readonly name = "read_file" as const - async execute(params: { files: FileEntry[] }, task: Task, callbacks: ToolCallbacks): Promise { - const { handleError, pushToolResult } = callbacks - const fileEntries = params.files - const modelInfo = task.api.getModel().info - const useNative = true - - if (!fileEntries || fileEntries.length === 0) { - task.consecutiveMistakeCount++ - task.recordToolError("read_file") - const errorMsg = await task.sayAndCreateMissingParamError("read_file", "files") - const errorResult = `Error: ${errorMsg}` - pushToolResult(errorResult) - return + async execute(params: ReadFileToolParams, task: Task, callbacks: ToolCallbacks): Promise { + // Dispatch to legacy or new execution path based on format + if (isLegacyReadFileParams(params)) { + return this.executeLegacy(params.files, task, callbacks) } - // Enforce maxConcurrentFileReads limit - const { maxConcurrentFileReads = 5 } = (await task.providerRef.deref()?.getState()) ?? {} - if (fileEntries.length > maxConcurrentFileReads) { + return this.executeNew(params, task, callbacks) + } + + /** + * Execute new single-file format with slice/indentation mode support. + */ + private async executeNew(params: ReadFileParams, task: Task, callbacks: ToolCallbacks): Promise { + const { pushToolResult } = callbacks + const modelInfo = task.api.getModel().info + const filePath = params.path + + // Validate input + if (!filePath) { task.consecutiveMistakeCount++ task.recordToolError("read_file") - const errorMsg = `Too many files requested. You attempted to read ${fileEntries.length} files, but the concurrent file reads limit is ${maxConcurrentFileReads}. Please read files in batches of ${maxConcurrentFileReads} or fewer.` - await task.say("error", errorMsg) - const errorResult = `Error: ${errorMsg}` - pushToolResult(errorResult) + const errorMsg = await task.sayAndCreateMissingParamError("read_file", "path") + pushToolResult(`Error: ${errorMsg}`) return } const supportsImages = modelInfo.supportsImages ?? false - const fileResults: FileResult[] = fileEntries.map((entry) => ({ - path: entry.path, - status: "pending", - lineRanges: entry.lineRanges, - })) + // Initialize file results tracking + const fileEntry: InternalFileEntry = { + path: filePath, + mode: params.mode, + offset: params.offset, + limit: params.limit, + anchor_line: params.indentation?.anchor_line, + max_levels: params.indentation?.max_levels, + include_siblings: params.indentation?.include_siblings, + include_header: params.indentation?.include_header, + max_lines: params.indentation?.max_lines, + } + + const fileResults: FileResult[] = [ + { + path: filePath, + status: "pending" as const, + entry: fileEntry, + }, + ] const updateFileResult = (filePath: string, updates: Partial) => { const index = fileResults.findIndex((result) => result.path === filePath) @@ -89,187 +128,35 @@ export class ReadFileTool extends BaseTool<"read_file"> { } try { + // Phase 1: Validate and filter files for approval const filesToApprove: FileResult[] = [] for (const fileResult of fileResults) { const relPath = fileResult.path - const fullPath = path.resolve(task.cwd, relPath) - - if (fileResult.lineRanges) { - let hasRangeError = false - for (const range of fileResult.lineRanges) { - if (range.start > range.end) { - const errorMsg = "Invalid line range: end line cannot be less than start line" - updateFileResult(relPath, { - status: "blocked", - error: errorMsg, - nativeContent: `File: ${relPath}\nError: Error reading file: ${errorMsg}`, - }) - await task.say("error", `Error reading file ${relPath}: ${errorMsg}`) - hasRangeError = true - break - } - if (isNaN(range.start) || isNaN(range.end)) { - const errorMsg = "Invalid line range values" - updateFileResult(relPath, { - status: "blocked", - error: errorMsg, - nativeContent: `File: ${relPath}\nError: Error reading file: ${errorMsg}`, - }) - await task.say("error", `Error reading file ${relPath}: ${errorMsg}`) - hasRangeError = true - break - } - } - if (hasRangeError) continue - } - - if (fileResult.status === "pending") { - const accessAllowed = task.rooIgnoreController?.validateAccess(relPath) - if (!accessAllowed) { - await task.say("rooignore_error", relPath) - const errorMsg = formatResponse.rooIgnoreError(relPath) - updateFileResult(relPath, { - status: "blocked", - error: errorMsg, - nativeContent: `File: ${relPath}\nError: ${errorMsg}`, - }) - continue - } - - filesToApprove.push(fileResult) - } - } - - if (filesToApprove.length > 1) { - const { maxReadFileLine = -1 } = (await task.providerRef.deref()?.getState()) ?? {} - - const batchFiles = filesToApprove.map((fileResult) => { - const relPath = fileResult.path - const fullPath = path.resolve(task.cwd, relPath) - const isOutsideWorkspace = isPathOutsideWorkspace(fullPath) - - let lineSnippet = "" - if (fileResult.lineRanges && fileResult.lineRanges.length > 0) { - const ranges = fileResult.lineRanges.map((range) => - t("tools:readFile.linesRange", { start: range.start, end: range.end }), - ) - lineSnippet = ranges.join(", ") - } else if (maxReadFileLine === 0) { - lineSnippet = t("tools:readFile.definitionsOnly") - } else if (maxReadFileLine > 0) { - lineSnippet = t("tools:readFile.maxLines", { max: maxReadFileLine }) - } - - const readablePath = getReadablePath(task.cwd, relPath) - const key = `${readablePath}${lineSnippet ? ` (${lineSnippet})` : ""}` - - return { path: readablePath, lineSnippet, isOutsideWorkspace, key, content: fullPath } - }) - - const completeMessage = JSON.stringify({ tool: "readFile", batchFiles } satisfies ClineSayTool) - const { response, text, images } = await task.ask("tool", completeMessage, false) - - if (response === "yesButtonClicked") { - if (text) await task.say("user_feedback", text, images) - filesToApprove.forEach((fileResult) => { - updateFileResult(fileResult.path, { - status: "approved", - feedbackText: text, - feedbackImages: images, - }) - }) - } else if (response === "noButtonClicked") { - if (text) await task.say("user_feedback", text, images) - task.didRejectTool = true - filesToApprove.forEach((fileResult) => { - updateFileResult(fileResult.path, { - status: "denied", - nativeContent: `File: ${fileResult.path}\nStatus: Denied by user`, - feedbackText: text, - feedbackImages: images, - }) - }) - } else { - try { - const individualPermissions = JSON.parse(text || "{}") - let hasAnyDenial = false - - batchFiles.forEach((batchFile, index) => { - const fileResult = filesToApprove[index] - const approved = individualPermissions[batchFile.key] === true - - if (approved) { - updateFileResult(fileResult.path, { status: "approved" }) - } else { - hasAnyDenial = true - updateFileResult(fileResult.path, { - status: "denied", - nativeContent: `File: ${fileResult.path}\nStatus: Denied by user`, - }) - } - }) - - if (hasAnyDenial) task.didRejectTool = true - } catch (error) { - console.error("Failed to parse individual permissions:", error) - task.didRejectTool = true - filesToApprove.forEach((fileResult) => { - updateFileResult(fileResult.path, { - status: "denied", - nativeContent: `File: ${fileResult.path}\nStatus: Denied by user`, - }) - }) - } - } - } else if (filesToApprove.length === 1) { - const fileResult = filesToApprove[0] - const relPath = fileResult.path - const fullPath = path.resolve(task.cwd, relPath) - const isOutsideWorkspace = isPathOutsideWorkspace(fullPath) - const { maxReadFileLine = -1 } = (await task.providerRef.deref()?.getState()) ?? {} - - let lineSnippet = "" - if (fileResult.lineRanges && fileResult.lineRanges.length > 0) { - const ranges = fileResult.lineRanges.map((range) => - t("tools:readFile.linesRange", { start: range.start, end: range.end }), - ) - lineSnippet = ranges.join(", ") - } else if (maxReadFileLine === 0) { - lineSnippet = t("tools:readFile.definitionsOnly") - } else if (maxReadFileLine > 0) { - lineSnippet = t("tools:readFile.maxLines", { max: maxReadFileLine }) - } - - const completeMessage = JSON.stringify({ - tool: "readFile", - path: getReadablePath(task.cwd, relPath), - isOutsideWorkspace, - content: fullPath, - reason: lineSnippet, - } satisfies ClineSayTool) - - const { response, text, images } = await task.ask("tool", completeMessage, false) - if (response !== "yesButtonClicked") { - if (text) await task.say("user_feedback", text, images) - task.didRejectTool = true + // RooIgnore validation + const accessAllowed = task.rooIgnoreController?.validateAccess(relPath) + if (!accessAllowed) { + await task.say("rooignore_error", relPath) + const errorMsg = formatResponse.rooIgnoreError(relPath) updateFileResult(relPath, { - status: "denied", - nativeContent: `File: ${relPath}\nStatus: Denied by user`, - feedbackText: text, - feedbackImages: images, + status: "blocked", + error: errorMsg, + nativeContent: `File: ${relPath}\nError: ${errorMsg}`, }) - } else { - if (text) await task.say("user_feedback", text, images) - updateFileResult(relPath, { status: "approved", feedbackText: text, feedbackImages: images }) + continue } + + filesToApprove.push(fileResult) } + // Phase 2: Request user approval + await this.requestApproval(task, filesToApprove, updateFileResult) + + // Phase 3: Process approved files const imageMemoryTracker = new ImageMemoryTracker() const state = await task.providerRef.deref()?.getState() const { - maxReadFileLine = -1, maxImageFileSize = DEFAULT_MAX_IMAGE_FILE_SIZE_MB, maxTotalImageSize = DEFAULT_MAX_TOTAL_IMAGE_SIZE_MB, } = state ?? {} @@ -279,360 +166,469 @@ export class ReadFileTool extends BaseTool<"read_file"> { const relPath = fileResult.path const fullPath = path.resolve(task.cwd, relPath) + const entry = fileResult.entry! try { - // Check if the path is a directory before attempting to read it + // Check if path is a directory const stats = await fs.stat(fullPath) if (stats.isDirectory()) { - const errorMsg = `Cannot read '${relPath}' because it is a directory. To view the contents of a directory, use the list_files tool instead.` + const errorMsg = `Cannot read '${relPath}' because it is a directory. Use list_files tool instead.` updateFileResult(relPath, { status: "error", error: errorMsg, - nativeContent: `File: ${relPath}\nError: Error reading file: ${errorMsg}`, + nativeContent: `File: ${relPath}\nError: ${errorMsg}`, }) await task.say("error", `Error reading file ${relPath}: ${errorMsg}`) continue } - const [totalLines, isBinary] = await Promise.all([countFileLines(fullPath), isBinaryFile(fullPath)]) + // Check for binary file + const isBinary = await isBinaryFile(fullPath) if (isBinary) { - const fileExtension = path.extname(relPath).toLowerCase() - const supportedBinaryFormats = getSupportedBinaryFormats() - - if (isSupportedImageFormat(fileExtension)) { - try { - const validationResult = await validateImageForProcessing( - fullPath, - supportsImages, - maxImageFileSize, - maxTotalImageSize, - imageMemoryTracker.getTotalMemoryUsed(), - ) - - if (!validationResult.isValid) { - await task.fileContextTracker.trackFileContext(relPath, "read_tool" as RecordSource) - updateFileResult(relPath, { - nativeContent: `File: ${relPath}\nNote: ${validationResult.notice}`, - }) - continue - } - - const imageResult = await processImageFile(fullPath) - imageMemoryTracker.addMemoryUsage(imageResult.sizeInMB) - await task.fileContextTracker.trackFileContext(relPath, "read_tool" as RecordSource) - - updateFileResult(relPath, { - nativeContent: `File: ${relPath}\nNote: ${imageResult.notice}`, - imageDataUrl: imageResult.dataUrl, - }) - continue - } catch (error) { - const errorMsg = error instanceof Error ? error.message : String(error) - updateFileResult(relPath, { - status: "error", - error: `Error reading image file: ${errorMsg}`, - nativeContent: `File: ${relPath}\nError: Error reading image file: ${errorMsg}`, - }) - await task.say("error", `Error reading image file ${relPath}: ${errorMsg}`) - continue - } - } - - if (supportedBinaryFormats && supportedBinaryFormats.includes(fileExtension)) { - // Use extractTextFromFile for supported binary formats (PDF, DOCX, etc.) - try { - const content = await extractTextFromFile(fullPath) - const numberedContent = addLineNumbers(content) - const lines = content.split("\n") - const lineCount = lines.length - - await task.fileContextTracker.trackFileContext(relPath, "read_tool" as RecordSource) - - updateFileResult(relPath, { - nativeContent: - lineCount > 0 - ? `File: ${relPath}\nLines 1-${lineCount}:\n${numberedContent}` - : `File: ${relPath}\nNote: File is empty`, - }) - continue - } catch (error) { - const errorMsg = error instanceof Error ? error.message : String(error) - updateFileResult(relPath, { - status: "error", - error: `Error extracting text: ${errorMsg}`, - nativeContent: `File: ${relPath}\nError: Error extracting text: ${errorMsg}`, - }) - await task.say("error", `Error extracting text from ${relPath}: ${errorMsg}`) - continue - } - } else { - const fileFormat = fileExtension.slice(1) || "bin" - updateFileResult(relPath, { - notice: `Binary file format: ${fileFormat}`, - nativeContent: `File: ${relPath}\nBinary file (${fileFormat}) - content not displayed`, - }) - continue - } - } - - if (fileResult.lineRanges && fileResult.lineRanges.length > 0) { - const nativeRangeResults: string[] = [] - - for (const range of fileResult.lineRanges) { - const content = addLineNumbers( - await readLines(fullPath, range.end - 1, range.start - 1), - range.start, - ) - nativeRangeResults.push(`Lines ${range.start}-${range.end}:\n${content}`) - } - - updateFileResult(relPath, { - nativeContent: `File: ${relPath}\n${nativeRangeResults.join("\n\n")}`, - }) - continue - } - - if (maxReadFileLine === 0) { - try { - const defResult = await parseSourceCodeDefinitionsForFile( - fullPath, - task.rooIgnoreController, - ) - if (defResult) { - const notice = `Showing only ${maxReadFileLine} of ${totalLines} total lines. Use line_range if you need to read more lines` - updateFileResult(relPath, { - nativeContent: `File: ${relPath}\nCode Definitions:\n${defResult}\n\nNote: ${notice}`, - }) - } - } catch (error) { - if (error instanceof Error && error.message.startsWith("Unsupported language:")) { - console.warn(`[read_file] Warning: ${error.message}`) - } else { - console.error( - `[read_file] Unhandled error: ${error instanceof Error ? error.message : String(error)}`, - ) - } - } - continue - } - - if (maxReadFileLine > 0 && totalLines > maxReadFileLine) { - const content = addLineNumbers(await readLines(fullPath, maxReadFileLine - 1, 0)) - let toolInfo = `Lines 1-${maxReadFileLine}:\n${content}\n` - - try { - const defResult = await parseSourceCodeDefinitionsForFile( - fullPath, - task.rooIgnoreController, - ) - if (defResult) { - const truncatedDefs = truncateDefinitionsToLineLimit(defResult, maxReadFileLine) - toolInfo += `\nCode Definitions:\n${truncatedDefs}\n` - } - - const notice = `Showing only ${maxReadFileLine} of ${totalLines} total lines. Use line_range if you need to read more lines` - toolInfo += `\nNote: ${notice}` - - updateFileResult(relPath, { - nativeContent: `File: ${relPath}\n${toolInfo}`, - }) - } catch (error) { - if (error instanceof Error && error.message.startsWith("Unsupported language:")) { - console.warn(`[read_file] Warning: ${error.message}`) - } else { - console.error( - `[read_file] Unhandled error: ${error instanceof Error ? error.message : String(error)}`, - ) - } - } + await this.handleBinaryFile( + task, + relPath, + fullPath, + supportsImages, + maxImageFileSize, + maxTotalImageSize, + imageMemoryTracker, + updateFileResult, + ) continue } - const { id: modelId, info: modelInfo } = task.api.getModel() - const { contextTokens } = task.getTokenUsage() - const contextWindow = modelInfo.contextWindow - - const maxOutputTokens = - getModelMaxOutputTokens({ - modelId, - model: modelInfo, - settings: task.apiConfiguration, - }) ?? ANTHROPIC_DEFAULT_MAX_TOKENS - - // Calculate available token budget (60% of remaining context) - const remainingTokens = contextWindow - maxOutputTokens - (contextTokens || 0) - const safeReadBudget = Math.floor(remainingTokens * FILE_READ_BUDGET_PERCENT) - - let toolInfo = "" - - if (safeReadBudget <= 0) { - // No budget available - const notice = "No available context budget for file reading" - toolInfo = `Note: ${notice}` - } else { - // Read file with incremental token counting - const result = await readFileWithTokenBudget(fullPath, { - budgetTokens: safeReadBudget, - }) - - const content = addLineNumbers(result.content) - - if (!result.complete) { - // File was truncated - const notice = `File truncated: showing ${result.lineCount} lines (${result.tokenCount} tokens) due to context budget. Use line_range to read specific sections.` - toolInfo = - result.lineCount > 0 - ? `Lines 1-${result.lineCount}:\n${content}\n\nNote: ${notice}` - : `Note: ${notice}` - } else { - // Full file read - if (result.lineCount === 0) { - toolInfo = "Note: File is empty" - } else { - toolInfo = `Lines 1-${result.lineCount}:\n${content}` - } - } - } + // Read text file content with lossy UTF-8 conversion + // Reading as Buffer first allows graceful handling of non-UTF8 bytes + // (they become U+FFFD replacement characters instead of throwing) + const buffer = await fs.readFile(fullPath) + const fileContent = buffer.toString("utf-8") + const result = this.processTextFile(fileContent, entry) await task.fileContextTracker.trackFileContext(relPath, "read_tool" as RecordSource) updateFileResult(relPath, { - nativeContent: `File: ${relPath}\n${toolInfo}`, + nativeContent: `File: ${relPath}\n${result}`, }) } catch (error) { const errorMsg = error instanceof Error ? error.message : String(error) updateFileResult(relPath, { status: "error", error: `Error reading file: ${errorMsg}`, - nativeContent: `File: ${relPath}\nError: Error reading file: ${errorMsg}`, + nativeContent: `File: ${relPath}\nError: ${errorMsg}`, }) await task.say("error", `Error reading file ${relPath}: ${errorMsg}`) } } - // Check if any files had errors or were blocked and mark the turn as failed - const hasErrors = fileResults.some((result) => result.status === "error" || result.status === "blocked") + // Phase 4: Build and return result + const hasErrors = fileResults.some((r) => r.status === "error" || r.status === "blocked") if (hasErrors) { task.didToolFailInCurrentTurn = true } - // Build final result - const finalResult = fileResults - .filter((result) => result.nativeContent) - .map((result) => result.nativeContent) - .join("\n\n---\n\n") + this.buildAndPushResult(task, fileResults, pushToolResult) + } catch (error) { + const relPath = filePath || "unknown" + const errorMsg = error instanceof Error ? error.message : String(error) - const fileImageUrls = fileResults - .filter((result) => result.imageDataUrl) - .map((result) => result.imageDataUrl as string) + updateFileResult(relPath, { + status: "error", + error: `Error reading file: ${errorMsg}`, + nativeContent: `File: ${relPath}\nError: ${errorMsg}`, + }) - let statusMessage = "" - let feedbackImages: any[] = [] + await task.say("error", `Error reading file ${relPath}: ${errorMsg}`) + task.didToolFailInCurrentTurn = true - const deniedWithFeedback = fileResults.find((result) => result.status === "denied" && result.feedbackText) + const errorResult = fileResults + .filter((r) => r.nativeContent) + .map((r) => r.nativeContent) + .join("\n\n---\n\n") - if (deniedWithFeedback && deniedWithFeedback.feedbackText) { - statusMessage = formatResponse.toolDeniedWithFeedback(deniedWithFeedback.feedbackText) - feedbackImages = deniedWithFeedback.feedbackImages || [] - } else if (task.didRejectTool) { - statusMessage = formatResponse.toolDenied() - } else { - const approvedWithFeedback = fileResults.find( - (result) => result.status === "approved" && result.feedbackText, - ) + pushToolResult(errorResult || `Error: ${errorMsg}`) + } + } - if (approvedWithFeedback && approvedWithFeedback.feedbackText) { - statusMessage = formatResponse.toolApprovedWithFeedback(approvedWithFeedback.feedbackText) - feedbackImages = approvedWithFeedback.feedbackImages || [] - } + /** + * Process a text file according to the requested mode. + */ + private processTextFile(content: string, entry: InternalFileEntry): string { + const mode = entry.mode || "slice" + + if (mode === "indentation") { + // Indentation mode: semantic block extraction + // When anchor_line is not provided, default to offset (which defaults to 1) + const anchorLine = entry.anchor_line ?? entry.offset ?? 1 + const result = readWithIndentation(content, { + anchorLine, + maxLevels: entry.max_levels, + includeSiblings: entry.include_siblings, + includeHeader: entry.include_header, + limit: entry.limit ?? DEFAULT_LINE_LIMIT, + maxLines: entry.max_lines, + }) + + let output = result.content + + if (result.wasTruncated && result.includedRanges.length > 0) { + const [start, end] = result.includedRanges[0] + const nextOffset = end + 1 + const effectiveLimit = entry.limit ?? DEFAULT_LINE_LIMIT + // Put truncation warning at TOP (before content) to match @ mention format + output = `IMPORTANT: File content truncated. + Status: Showing lines ${start}-${end} of ${result.totalLines} total lines. + To read more: Use the read_file tool with offset=${nextOffset} and limit=${effectiveLimit}. + + ${result.content}` + } else if (result.includedRanges.length > 0) { + const rangeStr = result.includedRanges.map(([s, e]) => `${s}-${e}`).join(", ") + output += `\n\nIncluded ranges: ${rangeStr} (total: ${result.totalLines} lines)` } - const allImages = [...feedbackImages, ...fileImageUrls] + return output + } + + // Slice mode (default): simple offset/limit reading + // NOTE: read_file offset is 1-based externally; convert to 0-based for readWithSlice. + const offset1 = entry.offset ?? 1 + const offset0 = Math.max(0, offset1 - 1) + const limit = entry.limit ?? DEFAULT_LINE_LIMIT + + const result = readWithSlice(content, offset0, limit) + + let output = result.content + + if (result.wasTruncated) { + const startLine = offset1 + const endLine = offset1 + result.returnedLines - 1 + const nextOffset = endLine + 1 + // Put truncation warning at TOP (before content) to match @ mention format + output = `IMPORTANT: File content truncated. + Status: Showing lines ${startLine}-${endLine} of ${result.totalLines} total lines. + To read more: Use the read_file tool with offset=${nextOffset} and limit=${limit}. + + ${result.content}` + } else if (result.returnedLines === 0) { + output = "Note: File is empty" + } - const finalModelSupportsImages = task.api.getModel().info.supportsImages ?? false - const imagesToInclude = finalModelSupportsImages ? allImages : [] + return output + } - if (statusMessage || imagesToInclude.length > 0) { - const result = formatResponse.toolResult( - statusMessage || finalResult, - imagesToInclude.length > 0 ? imagesToInclude : undefined, + /** + * Handle binary file processing (images, PDF, DOCX, etc.). + */ + private async handleBinaryFile( + task: Task, + relPath: string, + fullPath: string, + supportsImages: boolean, + maxImageFileSize: number, + maxTotalImageSize: number, + imageMemoryTracker: ImageMemoryTracker, + updateFileResult: (path: string, updates: Partial) => void, + ): Promise { + const fileExtension = path.extname(relPath).toLowerCase() + const supportedBinaryFormats = getSupportedBinaryFormats() + + // Handle image files + if (isSupportedImageFormat(fileExtension)) { + try { + const validationResult = await validateImageForProcessing( + fullPath, + supportsImages, + maxImageFileSize, + maxTotalImageSize, + imageMemoryTracker.getTotalMemoryUsed(), ) - if (typeof result === "string") { - if (statusMessage) { - pushToolResult(`${result}\n${finalResult}`) - } else { - pushToolResult(result) - } - } else { - if (statusMessage) { - const textBlock = { type: "text" as const, text: finalResult } - pushToolResult([...result, textBlock]) - } else { - pushToolResult(result) - } + if (!validationResult.isValid) { + await task.fileContextTracker.trackFileContext(relPath, "read_tool" as RecordSource) + updateFileResult(relPath, { + nativeContent: `File: ${relPath}\nNote: ${validationResult.notice}`, + }) + return } - } else { - pushToolResult(finalResult) + + const imageResult = await processImageFile(fullPath) + imageMemoryTracker.addMemoryUsage(imageResult.sizeInMB) + await task.fileContextTracker.trackFileContext(relPath, "read_tool" as RecordSource) + + updateFileResult(relPath, { + nativeContent: `File: ${relPath}\nNote: ${imageResult.notice}`, + imageDataUrl: imageResult.dataUrl, + }) + return + } catch (error) { + const errorMsg = error instanceof Error ? error.message : String(error) + updateFileResult(relPath, { + status: "error", + error: `Error reading image file: ${errorMsg}`, + nativeContent: `File: ${relPath}\nError: ${errorMsg}`, + }) + await task.say("error", `Error reading image file ${relPath}: ${errorMsg}`) + return } - } catch (error) { - const relPath = fileEntries[0]?.path || "unknown" - const errorMsg = error instanceof Error ? error.message : String(error) + } + + // Handle other supported binary formats (PDF, DOCX, etc.) + if (supportedBinaryFormats && supportedBinaryFormats.includes(fileExtension)) { + try { + const content = await extractTextFromFile(fullPath) + const numberedContent = addLineNumbers(content) + const lineCount = content.split("\n").length - if (fileResults.length > 0) { + await task.fileContextTracker.trackFileContext(relPath, "read_tool" as RecordSource) + + updateFileResult(relPath, { + nativeContent: + lineCount > 0 + ? `File: ${relPath}\nLines 1-${lineCount}:\n${numberedContent}` + : `File: ${relPath}\nNote: File is empty`, + }) + return + } catch (error) { + const errorMsg = error instanceof Error ? error.message : String(error) updateFileResult(relPath, { status: "error", - error: `Error reading file: ${errorMsg}`, - nativeContent: `File: ${relPath}\nError: Error reading file: ${errorMsg}`, + error: `Error extracting text: ${errorMsg}`, + nativeContent: `File: ${relPath}\nError: ${errorMsg}`, }) + await task.say("error", `Error extracting text from ${relPath}: ${errorMsg}`) + return } + } - await task.say("error", `Error reading file ${relPath}: ${errorMsg}`) + // Unsupported binary format + const fileFormat = fileExtension.slice(1) || "bin" + updateFileResult(relPath, { + notice: `Binary file format: ${fileFormat}`, + nativeContent: `File: ${relPath}\nBinary file (${fileFormat}) - content not displayed`, + }) + } - // Mark that a tool failed in this turn - task.didToolFailInCurrentTurn = true + /** + * Request user approval for file reads. + */ + private async requestApproval( + task: Task, + filesToApprove: FileResult[], + updateFileResult: (path: string, updates: Partial) => void, + ): Promise { + if (filesToApprove.length === 0) return + + if (filesToApprove.length > 1) { + // Batch approval + const batchFiles = filesToApprove.map((fileResult) => { + const relPath = fileResult.path + const fullPath = path.resolve(task.cwd, relPath) + const isOutsideWorkspace = isPathOutsideWorkspace(fullPath) + const readablePath = getReadablePath(task.cwd, relPath) - const errorResult = fileResults - .filter((result) => result.nativeContent) - .map((result) => result.nativeContent) - .join("\n\n---\n\n") + const lineSnippet = this.getLineSnippet(fileResult.entry!) + const key = `${readablePath}${lineSnippet ? ` (${lineSnippet})` : ""}` + + return { path: readablePath, lineSnippet, isOutsideWorkspace, key, content: fullPath } + }) + + const completeMessage = JSON.stringify({ tool: "readFile", batchFiles } satisfies ClineSayTool) + const { response, text, images } = await task.ask("tool", completeMessage, false) + + if (response === "yesButtonClicked") { + if (text) await task.say("user_feedback", text, images) + filesToApprove.forEach((fr) => { + updateFileResult(fr.path, { status: "approved", feedbackText: text, feedbackImages: images }) + }) + } else if (response === "noButtonClicked") { + if (text) await task.say("user_feedback", text, images) + task.didRejectTool = true + filesToApprove.forEach((fr) => { + updateFileResult(fr.path, { + status: "denied", + nativeContent: `File: ${fr.path}\nStatus: Denied by user`, + feedbackText: text, + feedbackImages: images, + }) + }) + } else { + // Individual permissions + try { + const individualPermissions = JSON.parse(text || "{}") + let hasAnyDenial = false + + batchFiles.forEach((batchFile, index) => { + const fileResult = filesToApprove[index] + const approved = individualPermissions[batchFile.key] === true + + if (approved) { + updateFileResult(fileResult.path, { status: "approved" }) + } else { + hasAnyDenial = true + updateFileResult(fileResult.path, { + status: "denied", + nativeContent: `File: ${fileResult.path}\nStatus: Denied by user`, + }) + } + }) + + if (hasAnyDenial) task.didRejectTool = true + } catch { + task.didRejectTool = true + filesToApprove.forEach((fr) => { + updateFileResult(fr.path, { + status: "denied", + nativeContent: `File: ${fr.path}\nStatus: Denied by user`, + }) + }) + } + } + } else { + // Single file approval + const fileResult = filesToApprove[0] + const relPath = fileResult.path + const fullPath = path.resolve(task.cwd, relPath) + const isOutsideWorkspace = isPathOutsideWorkspace(fullPath) + const lineSnippet = this.getLineSnippet(fileResult.entry!) + + const startLine = this.getStartLine(fileResult.entry!) + + const completeMessage = JSON.stringify({ + tool: "readFile", + path: getReadablePath(task.cwd, relPath), + isOutsideWorkspace, + content: fullPath, + reason: lineSnippet, + startLine, + } satisfies ClineSayTool) + + const { response, text, images } = await task.ask("tool", completeMessage, false) + + if (response !== "yesButtonClicked") { + if (text) await task.say("user_feedback", text, images) + task.didRejectTool = true + updateFileResult(relPath, { + status: "denied", + nativeContent: `File: ${relPath}\nStatus: Denied by user`, + feedbackText: text, + feedbackImages: images, + }) + } else { + if (text) await task.say("user_feedback", text, images) + updateFileResult(relPath, { status: "approved", feedbackText: text, feedbackImages: images }) + } + } + } + + /** + * Get the starting line number for navigation purposes. + */ + private getStartLine(entry: InternalFileEntry): number | undefined { + if (entry.mode === "indentation" && entry.anchor_line !== undefined) { + return entry.anchor_line + } + const offset = entry.offset ?? 1 + return offset > 1 ? offset : undefined + } + + /** + * Generate a human-readable line snippet for approval messages. + */ + private getLineSnippet(entry: InternalFileEntry): string { + if (entry.mode === "indentation" && entry.anchor_line !== undefined) { + return `indentation mode at line ${entry.anchor_line}` + } - pushToolResult(errorResult) + const limit = entry.limit ?? DEFAULT_LINE_LIMIT + const offset1 = entry.offset ?? 1 + + if (offset1 > 1) { + return `lines ${offset1}-${offset1 + limit - 1}` } + + // Always show the line limit, even when using the default + return `up to ${limit} lines` } - getReadFileToolDescription(blockName: string, blockParams: any): string - getReadFileToolDescription(blockName: string, nativeArgs: { files: FileEntry[] }): string - getReadFileToolDescription(blockName: string, second: any): string { - // If native typed args ({ files: FileEntry[] }) were provided - if (second && typeof second === "object" && "files" in second && Array.isArray(second.files)) { - const paths = (second.files as FileEntry[]).map((f) => f?.path).filter(Boolean) as string[] - if (paths.length === 0) { - return `[${blockName} with no valid paths]` - } else if (paths.length === 1) { - return `[${blockName} for '${paths[0]}'. Reading multiple files at once is more efficient for the LLM. If other files are relevant to your current task, please read them simultaneously.]` - } else if (paths.length <= 3) { - const pathList = paths.map((p) => `'${p}'`).join(", ") - return `[${blockName} for ${pathList}]` + /** + * Build and push the final result to the tool output. + */ + private buildAndPushResult(task: Task, fileResults: FileResult[], pushToolResult: PushToolResult): void { + const finalResult = fileResults + .filter((r) => r.nativeContent) + .map((r) => r.nativeContent) + .join("\n\n---\n\n") + + const fileImageUrls = fileResults.filter((r) => r.imageDataUrl).map((r) => r.imageDataUrl as string) + + let statusMessage = "" + let feedbackImages: string[] = [] + + const deniedWithFeedback = fileResults.find((r) => r.status === "denied" && r.feedbackText) + + if (deniedWithFeedback?.feedbackText) { + statusMessage = formatResponse.toolDeniedWithFeedback(deniedWithFeedback.feedbackText) + feedbackImages = deniedWithFeedback.feedbackImages || [] + } else if (task.didRejectTool) { + statusMessage = formatResponse.toolDenied() + } else { + const approvedWithFeedback = fileResults.find((r) => r.status === "approved" && r.feedbackText) + if (approvedWithFeedback?.feedbackText) { + statusMessage = formatResponse.toolApprovedWithFeedback(approvedWithFeedback.feedbackText) + feedbackImages = approvedWithFeedback.feedbackImages || [] + } + } + + const allImages = [...feedbackImages, ...fileImageUrls] + const finalModelSupportsImages = task.api.getModel().info.supportsImages ?? false + const imagesToInclude = finalModelSupportsImages ? allImages : [] + + if (statusMessage || imagesToInclude.length > 0) { + const result = formatResponse.toolResult( + statusMessage || finalResult, + imagesToInclude.length > 0 ? imagesToInclude : undefined, + ) + + if (typeof result === "string") { + pushToolResult(statusMessage ? `${result}\n${finalResult}` : result) } else { - return `[${blockName} for ${paths.length} files]` + if (statusMessage) { + const textBlock = { type: "text" as const, text: finalResult } + pushToolResult([...result, textBlock] as any) + } else { + pushToolResult(result as any) + } } + } else { + pushToolResult(finalResult) + } + } + + getReadFileToolDescription(blockName: string, blockParams: { path?: string }): string + getReadFileToolDescription(blockName: string, nativeArgs: ReadFileParams): string + getReadFileToolDescription(blockName: string, second: unknown): string { + // If native typed args were provided + if (second && typeof second === "object" && "path" in second && typeof (second as any).path === "string") { + return `[${blockName} for '${(second as any).path}']` } - const blockParams = second as any + const blockParams = second as Record if (blockParams?.path) { - return `[${blockName} for '${blockParams.path}'. Reading multiple files at once is more efficient for the LLM. If other files are relevant to your current task, please read them simultaneously.]` + return `[${blockName} for '${blockParams.path}']` } - return `[${blockName} with missing files]` + return `[${blockName} with missing path]` } override async handlePartial(task: Task, block: ToolUse<"read_file">): Promise { + // Handle both legacy and new format for partial display let filePath = "" - if (block.nativeArgs && "files" in block.nativeArgs && Array.isArray(block.nativeArgs.files)) { - const files = block.nativeArgs.files - if (files.length > 0 && files[0]?.path) { - filePath = files[0].path + if (block.nativeArgs) { + if (isLegacyReadFileParams(block.nativeArgs)) { + // Legacy format - show first file + filePath = block.nativeArgs.files[0]?.path ?? "" + } else { + filePath = block.nativeArgs.path ?? "" } } @@ -648,6 +644,155 @@ export class ReadFileTool extends BaseTool<"read_file"> { } satisfies ClineSayTool) await task.ask("tool", partialMessage, block.partial).catch(() => {}) } + + /** + * Execute legacy multi-file format for backward compatibility. + * This handles the old format: { files: [{ path: string, lineRanges?: [...] }] } + */ + private async executeLegacy(fileEntries: FileEntry[], task: Task, callbacks: ToolCallbacks): Promise { + const { pushToolResult } = callbacks + const modelInfo = task.api.getModel().info + + // Temporary indicator for testing legacy format detection + console.warn("[read_file] Legacy format detected - using backward compatibility path") + + if (!fileEntries || fileEntries.length === 0) { + task.consecutiveMistakeCount++ + task.recordToolError("read_file") + const errorMsg = await task.sayAndCreateMissingParamError("read_file", "files") + pushToolResult(`Error: ${errorMsg}`) + return + } + + const supportsImages = modelInfo.supportsImages ?? false + + // Process each file sequentially (legacy behavior) + const results: string[] = [] + + for (const entry of fileEntries) { + const relPath = entry.path + const fullPath = path.resolve(task.cwd, relPath) + + // RooIgnore validation + const accessAllowed = task.rooIgnoreController?.validateAccess(relPath) + if (!accessAllowed) { + await task.say("rooignore_error", relPath) + const errorMsg = formatResponse.rooIgnoreError(relPath) + results.push(`File: ${relPath}\nError: ${errorMsg}`) + continue + } + + // Request approval for single file + const isOutsideWorkspace = isPathOutsideWorkspace(fullPath) + let lineSnippet = "" + if (entry.lineRanges && entry.lineRanges.length > 0) { + const ranges = entry.lineRanges.map((range: LineRange) => `lines ${range.start}-${range.end}`) + lineSnippet = ranges.join(", ") + } + + const completeMessage = JSON.stringify({ + tool: "readFile", + path: getReadablePath(task.cwd, relPath), + isOutsideWorkspace, + content: fullPath, + reason: lineSnippet || undefined, + } satisfies ClineSayTool) + + const { response, text, images } = await task.ask("tool", completeMessage, false) + + if (response !== "yesButtonClicked") { + if (text) await task.say("user_feedback", text, images) + task.didRejectTool = true + results.push(`File: ${relPath}\nStatus: Denied by user`) + continue + } + + if (text) await task.say("user_feedback", text, images) + + try { + // Check if the path is a directory + const stats = await fs.stat(fullPath) + if (stats.isDirectory()) { + const errorMsg = `Cannot read '${relPath}' because it is a directory.` + results.push(`File: ${relPath}\nError: ${errorMsg}`) + await task.say("error", `Error reading file ${relPath}: ${errorMsg}`) + continue + } + + const isBinary = await isBinaryFile(fullPath).catch(() => false) + + if (isBinary) { + // Handle binary files (images) + const fileExtension = path.extname(relPath).toLowerCase() + if (supportsImages && isSupportedImageFormat(fileExtension)) { + const state = await task.providerRef.deref()?.getState() + const { + maxImageFileSize = DEFAULT_MAX_IMAGE_FILE_SIZE_MB, + maxTotalImageSize = DEFAULT_MAX_TOTAL_IMAGE_SIZE_MB, + } = state ?? {} + const validation = await validateImageForProcessing( + fullPath, + supportsImages, + maxImageFileSize, + maxTotalImageSize, + 0, // Legacy path doesn't track cumulative memory + ) + if (!validation.isValid) { + results.push(`File: ${relPath}\nNotice: ${validation.notice ?? "Image validation failed"}`) + continue + } + const imageResult = await processImageFile(fullPath) + if (imageResult) { + results.push(`File: ${relPath}\n[Image file - content processed for vision model]`) + } + } else { + results.push(`File: ${relPath}\nError: Cannot read binary file`) + } + continue + } + + // Read text file + const rawContent = await fs.readFile(fullPath, "utf8") + + // Handle line ranges if specified + let content: string + if (entry.lineRanges && entry.lineRanges.length > 0) { + const lines = rawContent.split("\n") + const selectedLines: string[] = [] + + for (const range of entry.lineRanges) { + // Convert to 0-based index, ranges are 1-based inclusive + const startIdx = Math.max(0, range.start - 1) + const endIdx = Math.min(lines.length - 1, range.end - 1) + + for (let i = startIdx; i <= endIdx; i++) { + selectedLines.push(`${i + 1} | ${lines[i]}`) + } + } + content = selectedLines.join("\n") + } else { + // Read with default limits using slice mode + const result = readWithSlice(rawContent, 0, DEFAULT_LINE_LIMIT) + content = result.content + if (result.wasTruncated) { + content += `\n\n[File truncated: showing ${result.returnedLines} of ${result.totalLines} total lines]` + } + } + + results.push(`File: ${relPath}\n${content}`) + + // Track file in context + await task.fileContextTracker.trackFileContext(relPath, "read_tool") + } catch (error) { + const errorMsg = error instanceof Error ? error.message : String(error) + results.push(`File: ${relPath}\nError: ${errorMsg}`) + await task.say("error", `Error reading file ${relPath}: ${errorMsg}`) + } + } + + // Push combined results + pushToolResult(results.join("\n\n---\n\n")) + } } export const readFileTool = new ReadFileTool() diff --git a/src/core/tools/__tests__/ToolRepetitionDetector.spec.ts b/src/core/tools/__tests__/ToolRepetitionDetector.spec.ts index 3e156dd7c49..bda80d711f5 100644 --- a/src/core/tools/__tests__/ToolRepetitionDetector.spec.ts +++ b/src/core/tools/__tests__/ToolRepetitionDetector.spec.ts @@ -575,7 +575,7 @@ describe("ToolRepetitionDetector", () => { params: {}, // Empty for native protocol partial: false, nativeArgs: { - files: [{ path: "file1.ts" }], + path: "file1.ts", }, } @@ -585,7 +585,7 @@ describe("ToolRepetitionDetector", () => { params: {}, // Empty for native protocol partial: false, nativeArgs: { - files: [{ path: "file2.ts" }], + path: "file2.ts", }, } @@ -609,7 +609,7 @@ describe("ToolRepetitionDetector", () => { params: {}, // Empty for native protocol partial: false, nativeArgs: { - files: [{ path: "same-file.ts" }], + path: "same-file.ts", }, } @@ -625,7 +625,7 @@ describe("ToolRepetitionDetector", () => { expect(result.askUser).toBeDefined() }) - it("should differentiate read_file calls with multiple files in different orders", () => { + it("should treat different slice offsets as distinct read_file calls", () => { const detector = new ToolRepetitionDetector(2) const readFile1: ToolUse = { @@ -634,7 +634,9 @@ describe("ToolRepetitionDetector", () => { params: {}, partial: false, nativeArgs: { - files: [{ path: "a.ts" }, { path: "b.ts" }], + path: "a.ts", + offset: 1, + limit: 2000, }, } @@ -644,11 +646,13 @@ describe("ToolRepetitionDetector", () => { params: {}, partial: false, nativeArgs: { - files: [{ path: "b.ts" }, { path: "a.ts" }], + path: "a.ts", + offset: 2001, + limit: 2000, }, } - // Different order should be treated as different calls + // Different offsets should be treated as different calls expect(detector.check(readFile1).allowExecution).toBe(true) expect(detector.check(readFile2).allowExecution).toBe(true) }) diff --git a/src/core/tools/__tests__/readFileTool.spec.ts b/src/core/tools/__tests__/readFileTool.spec.ts index a79cfffb504..f1be93fe2a6 100644 --- a/src/core/tools/__tests__/readFileTool.spec.ts +++ b/src/core/tools/__tests__/readFileTool.spec.ts @@ -1,14 +1,33 @@ -// npx vitest src/core/tools/__tests__/readFileTool.spec.ts +/** + * Tests for ReadFileTool - Codex-inspired file reading with indentation mode support. + * + * These tests cover: + * - Input validation (missing path parameter) + * - RooIgnore blocking + * - Directory read error handling + * - Binary file handling (images, PDF, DOCX, unsupported) + * - Image memory limits + * - Approval flow (approve, deny, feedback) + * - Text file processing (slice and indentation modes) + * - Output structure formatting + */ + +import path from "path" -import * as path from "path" - -import { countFileLines } from "../../../integrations/misc/line-counter" -import { readLines } from "../../../integrations/misc/read-lines" -import { extractTextFromFile } from "../../../integrations/misc/extract-text" -import { parseSourceCodeDefinitionsForFile } from "../../../services/tree-sitter" import { isBinaryFile } from "isbinaryfile" -import { ReadFileToolUse, ToolResponse } from "../../../shared/tools" -import { readFileTool } from "../ReadFileTool" + +import { readFileTool, ReadFileTool } from "../ReadFileTool" +import { formatResponse } from "../../prompts/responses" +import { + validateImageForProcessing, + processImageFile, + isSupportedImageFormat, + ImageMemoryTracker, +} from "../helpers/imageHelpers" +import { extractTextFromFile, addLineNumbers, getSupportedBinaryFormats } from "../../../integrations/misc/extract-text" +import { readWithIndentation, readWithSlice } from "../../../integrations/misc/indentation-reader" + +// ─── Mocks ──────────────────────────────────────────────────────────────────── vi.mock("path", async () => { const originalPath = await vi.importActual("path") @@ -19,76 +38,39 @@ vi.mock("path", async () => { } }) -// Already mocked above with hoisted fsPromises - -vi.mock("isbinaryfile") - -vi.mock("../../../integrations/misc/line-counter") -vi.mock("../../../integrations/misc/read-lines") - -// Mock fs/promises readFile for image tests -const fsPromises = vi.hoisted(() => ({ +vi.mock("fs/promises", () => ({ readFile: vi.fn(), - stat: vi.fn().mockResolvedValue({ size: 1024 }), + stat: vi.fn(), })) -vi.mock("fs/promises", () => fsPromises) - -// Mock input content for tests -let mockInputContent = "" -// Create hoisted mocks that can be used in vi.mock factories -const { addLineNumbersMock, mockReadFileWithTokenBudget } = vi.hoisted(() => { - const addLineNumbersMock = vi.fn().mockImplementation((text: string, startLine = 1) => { - if (!text) return "" - const lines = typeof text === "string" ? text.split("\n") : [text] - return lines.map((line: string, i: number) => `${startLine + i} | ${line}`).join("\n") - }) - const mockReadFileWithTokenBudget = vi.fn() - return { addLineNumbersMock, mockReadFileWithTokenBudget } -}) +vi.mock("isbinaryfile") -// First create all the mocks vi.mock("../../../integrations/misc/extract-text", () => ({ extractTextFromFile: vi.fn(), - addLineNumbers: addLineNumbersMock, + addLineNumbers: vi.fn().mockImplementation((text: string, startLine = 1) => { + if (!text) return "" + const lines = text.split("\n") + return lines.map((line, i) => `${startLine + i} | ${line}`).join("\n") + }), getSupportedBinaryFormats: vi.fn(() => [".pdf", ".docx", ".ipynb"]), })) -vi.mock("../../../services/tree-sitter") -// Mock readFileWithTokenBudget - must be mocked to prevent actual file system access -vi.mock("../../../integrations/misc/read-file-with-budget", () => ({ - readFileWithTokenBudget: (...args: any[]) => mockReadFileWithTokenBudget(...args), +vi.mock("../../../integrations/misc/indentation-reader", () => ({ + readWithIndentation: vi.fn(), + readWithSlice: vi.fn(), })) -const extractTextFromFileMock = vi.fn() -const getSupportedBinaryFormatsMock = vi.fn(() => [".pdf", ".docx", ".ipynb"]) - -// Mock formatResponse - use vi.hoisted to ensure mocks are available before vi.mock -const { toolResultMock, imageBlocksMock } = vi.hoisted(() => { - const toolResultMock = vi.fn((text: string, images?: string[]) => { - if (images && images.length > 0) { - return [ - { type: "text", text }, - ...images.map((img) => { - const [header, data] = img.split(",") - const media_type = header.match(/:(.*?);/)?.[1] || "image/png" - return { type: "image", source: { type: "base64", media_type, data } } - }), - ] - } - return text - }) - const imageBlocksMock = vi.fn((images?: string[]) => { - return images - ? images.map((img) => { - const [header, data] = img.split(",") - const media_type = header.match(/:(.*?);/)?.[1] || "image/png" - return { type: "image", source: { type: "base64", media_type, data } } - }) - : [] - }) - return { toolResultMock, imageBlocksMock } -}) +vi.mock("../helpers/imageHelpers", () => ({ + DEFAULT_MAX_IMAGE_FILE_SIZE_MB: 5, + DEFAULT_MAX_TOTAL_IMAGE_SIZE_MB: 20, + isSupportedImageFormat: vi.fn(), + validateImageForProcessing: vi.fn(), + processImageFile: vi.fn(), + ImageMemoryTracker: vi.fn().mockImplementation(() => ({ + getTotalMemoryUsed: vi.fn().mockReturnValue(0), + addMemoryUsage: vi.fn(), + })), +})) vi.mock("../../prompts/responses", () => ({ formatResponse: { @@ -102,1904 +84,590 @@ vi.mock("../../prompts/responses", () => ({ `The user approved this operation and responded with the message:\n\n${feedback}\n`, ), rooIgnoreError: vi.fn( - (path: string) => - `Access to ${path} is blocked by the .rooignore file settings. You must try to continue in the task without using this file, or ask the user to update the .rooignore file.`, + (filePath: string) => + `Access to ${filePath} is blocked by the .rooignore file settings. You must try to continue in the task without using this file, or ask the user to update the .rooignore file.`, ), - toolResult: toolResultMock, - imageBlocks: imageBlocksMock, - }, -})) - -vi.mock("../../ignore/RooIgnoreController", () => ({ - RooIgnoreController: class { - initialize() { - return Promise.resolve() - } - validateAccess() { - return true - } + toolResult: vi.fn((text: string, images?: string[]) => { + if (images && images.length > 0) { + return [ + { type: "text", text }, + ...images.map((img) => { + const [header, data] = img.split(",") + const media_type = header.match(/:(.*?);/)?.[1] || "image/png" + return { type: "image", source: { type: "base64", media_type, data } } + }), + ] + } + return text + }), + imageBlocks: vi.fn((images?: string[]) => { + return images + ? images.map((img) => { + const [header, data] = img.split(",") + const media_type = header.match(/:(.*?);/)?.[1] || "image/png" + return { type: "image", source: { type: "base64", media_type, data } } + }) + : [] + }), }, })) -vi.mock("../../../utils/fs", () => ({ - fileExistsAtPath: vi.fn().mockReturnValue(true), -})) - -// Global beforeEach to ensure clean mock state between all test suites -beforeEach(() => { - // NOTE: Removed vi.clearAllMocks() to prevent interference with setImageSupport calls - // Instead, individual suites clear their specific mocks to maintain isolation - - // Explicitly reset the hoisted mock implementations to prevent cross-suite pollution - toolResultMock.mockImplementation((text: string, images?: string[]) => { - if (images && images.length > 0) { - return [ - { type: "text", text }, - ...images.map((img) => { - const [header, data] = img.split(",") - const media_type = header.match(/:(.*?);/)?.[1] || "image/png" - return { type: "image", source: { type: "base64", media_type, data } } - }), - ] - } - return text - }) - - imageBlocksMock.mockImplementation((images?: string[]) => { - return images - ? images.map((img) => { - const [header, data] = img.split(",") - const media_type = header.match(/:(.*?);/)?.[1] || "image/png" - return { type: "image", source: { type: "base64", media_type, data } } - }) - : [] - }) - - // Reset addLineNumbers mock to its default implementation (prevents cross-test pollution) - addLineNumbersMock.mockReset() - addLineNumbersMock.mockImplementation((text: string, startLine = 1) => { - if (!text) return "" - const lines = typeof text === "string" ? text.split("\n") : [text] - return lines.map((line: string, i: number) => `${startLine + i} | ${line}`).join("\n") - }) - - // Reset readFileWithTokenBudget mock with default implementation - mockReadFileWithTokenBudget.mockClear() - mockReadFileWithTokenBudget.mockImplementation(async (_filePath: string, _options: any) => { - // Default: return the mockInputContent with 5 lines - const lines = mockInputContent ? mockInputContent.split("\n") : [] - return { - content: mockInputContent, - tokenCount: mockInputContent.length / 4, // rough estimate - lineCount: lines.length, - complete: true, - } - }) -}) - -// Mock i18n translation function -vi.mock("../../../i18n", () => ({ - t: vi.fn((key: string, params?: Record) => { - // Map translation keys to English text - const translations: Record = { - "tools:readFile.imageWithSize": "Image file ({{size}} KB)", - "tools:readFile.imageTooLarge": - "Image file is too large ({{size}}). The maximum allowed size is {{max}} MB.", - "tools:readFile.linesRange": " (lines {{start}}-{{end}})", - "tools:readFile.definitionsOnly": " (definitions only)", - "tools:readFile.maxLines": " (max {{max}} lines)", - } - - let result = translations[key] || key - - // Simple template replacement - if (params) { - Object.entries(params).forEach(([param, value]) => { - result = result.replace(new RegExp(`{{${param}}}`, "g"), String(value)) - }) - } - - return result - }), -})) +// Mock fs/promises +const fsPromises = await import("fs/promises") +const mockedFsReadFile = vi.mocked(fsPromises.readFile) +const mockedFsStat = vi.mocked(fsPromises.stat) + +const mockedIsBinaryFile = vi.mocked(isBinaryFile) +const mockedExtractTextFromFile = vi.mocked(extractTextFromFile) +const mockedReadWithSlice = vi.mocked(readWithSlice) +const mockedReadWithIndentation = vi.mocked(readWithIndentation) +const mockedIsSupportedImageFormat = vi.mocked(isSupportedImageFormat) +const mockedValidateImageForProcessing = vi.mocked(validateImageForProcessing) +const mockedProcessImageFile = vi.mocked(processImageFile) + +// ─── Test Helpers ───────────────────────────────────────────────────────────── + +interface MockTaskOptions { + supportsImages?: boolean + rooIgnoreAllowed?: boolean + maxImageFileSize?: number + maxTotalImageSize?: number +} -// Shared mock setup function to ensure consistent state across all test suites -function createMockCline(): any { - const mockProvider = { - getState: vi.fn(), - deref: vi.fn().mockReturnThis(), - } +function createMockTask(options: MockTaskOptions = {}) { + const { supportsImages = false, rooIgnoreAllowed = true, maxImageFileSize = 5, maxTotalImageSize = 20 } = options - const mockCline: any = { - cwd: "/", - task: "Test", - providerRef: mockProvider, - rooIgnoreController: { - validateAccess: vi.fn().mockReturnValue(true), + return { + cwd: "/test/workspace", + api: { + getModel: vi.fn().mockReturnValue({ + info: { supportsImages }, + }), }, + consecutiveMistakeCount: 0, + didToolFailInCurrentTurn: false, + didRejectTool: false, + ask: vi.fn().mockResolvedValue({ response: "yesButtonClicked", text: undefined, images: undefined }), say: vi.fn().mockResolvedValue(undefined), - ask: vi.fn().mockResolvedValue({ response: "yesButtonClicked" }), - presentAssistantMessage: vi.fn(), - handleError: vi.fn().mockResolvedValue(undefined), - pushToolResult: vi.fn(), + sayAndCreateMissingParamError: vi.fn().mockResolvedValue("Missing required parameter: path"), + recordToolError: vi.fn(), + rooIgnoreController: { + validateAccess: vi.fn().mockReturnValue(rooIgnoreAllowed), + }, fileContextTracker: { trackFileContext: vi.fn().mockResolvedValue(undefined), }, - recordToolUsage: vi.fn().mockReturnValue(undefined), - recordToolError: vi.fn().mockReturnValue(undefined), - didRejectTool: false, - getTokenUsage: vi.fn().mockReturnValue({ - contextTokens: 10000, - }), - apiConfiguration: { - apiProvider: "anthropic", - }, - // CRITICAL: Always ensure image support is enabled - api: { - getModel: vi.fn().mockReturnValue({ - id: "test-model", - info: { - supportsImages: true, - contextWindow: 200000, - maxTokens: 4096, - supportsPromptCache: false, - // (native tool support is determined at request-time; no model flag) - }, + providerRef: { + deref: vi.fn().mockReturnValue({ + getState: vi.fn().mockResolvedValue({ + maxImageFileSize, + maxTotalImageSize, + }), }), }, } - - return { mockCline, mockProvider } } -// Helper function to set image support without affecting shared state -function setImageSupport(mockCline: any, supportsImages: boolean | undefined): void { - mockCline.api = { - getModel: vi.fn().mockReturnValue({ - id: "test-model", - info: { supportsImages }, - }), +function createMockCallbacks() { + return { + pushToolResult: vi.fn(), + askApproval: vi.fn(), + handleError: vi.fn(), } } -describe("read_file tool with maxReadFileLine setting", () => { - // Test data - const testFilePath = "test/file.txt" - const absoluteFilePath = "/test/file.txt" - const fileContent = "Line 1\nLine 2\nLine 3\nLine 4\nLine 5" - const numberedFileContent = "1 | Line 1\n2 | Line 2\n3 | Line 3\n4 | Line 4\n5 | Line 5\n" - const sourceCodeDef = "\n\n# file.txt\n1--5 | Content" - - // Mocked functions with correct types - const mockedCountFileLines = vi.mocked(countFileLines) - const mockedReadLines = vi.mocked(readLines) - const mockedExtractTextFromFile = vi.mocked(extractTextFromFile) - const mockedParseSourceCodeDefinitionsForFile = vi.mocked(parseSourceCodeDefinitionsForFile) - - const mockedIsBinaryFile = vi.mocked(isBinaryFile) - const mockedPathResolve = vi.mocked(path.resolve) - - let mockCline: any - let mockProvider: any - let toolResult: ToolResponse | undefined +// ─── Tests ──────────────────────────────────────────────────────────────────── +describe("ReadFileTool", () => { beforeEach(() => { - // Clear specific mocks (not all mocks to preserve shared state) - mockedCountFileLines.mockClear() - mockedExtractTextFromFile.mockClear() - mockedIsBinaryFile.mockClear() - mockedPathResolve.mockClear() - addLineNumbersMock.mockClear() - extractTextFromFileMock.mockClear() - toolResultMock.mockClear() - - // Use shared mock setup function - const mocks = createMockCline() - mockCline = mocks.mockCline - mockProvider = mocks.mockProvider - - // Explicitly disable image support for text file tests to prevent cross-suite pollution - setImageSupport(mockCline, false) - - mockedPathResolve.mockReturnValue(absoluteFilePath) - mockedIsBinaryFile.mockResolvedValue(false) - - // Mock fsPromises.stat to return a file (not directory) by default - fsPromises.stat.mockResolvedValue({ - isDirectory: () => false, - isFile: () => true, - isSymbolicLink: () => false, - } as any) - - mockInputContent = fileContent + vi.clearAllMocks() - // Setup the extractTextFromFile mock implementation with the current mockInputContent - // Reset the spy before each test - addLineNumbersMock.mockClear() - - // Setup the extractTextFromFile mock to call our spy - mockedExtractTextFromFile.mockImplementation((_filePath) => { - // Call the spy and return its result - return Promise.resolve(addLineNumbersMock(mockInputContent)) + // Default mock implementations + mockedFsStat.mockResolvedValue({ isDirectory: () => false } as any) + mockedIsBinaryFile.mockResolvedValue(false) + mockedFsReadFile.mockResolvedValue(Buffer.from("test content")) + mockedReadWithSlice.mockReturnValue({ + content: "1 | test content", + returnedLines: 1, + totalLines: 1, + wasTruncated: false, + includedRanges: [[1, 1]], }) - - toolResult = undefined }) - /** - * Helper function to execute the read file tool with different maxReadFileLine settings - */ - async function executeReadFileTool( - params: Partial = {}, - options: { - maxReadFileLine?: number - totalLines?: number - skipAddLineNumbersCheck?: boolean // Flag to skip addLineNumbers check - path?: string - start_line?: string - end_line?: string - } = {}, - ): Promise { - // Configure mocks based on test scenario - const maxReadFileLine = options.maxReadFileLine ?? 500 - const totalLines = options.totalLines ?? 5 - - mockProvider.getState.mockResolvedValue({ maxReadFileLine, maxImageFileSize: 20, maxTotalImageSize: 20 }) - mockedCountFileLines.mockResolvedValue(totalLines) - - // Reset the spy before each test - addLineNumbersMock.mockClear() - - const lineRanges = - options.start_line && options.end_line - ? [ - { - start: Number(options.start_line), - end: Number(options.end_line), - }, - ] - : [] - - // Create a tool use object - const toolUse: ReadFileToolUse = { - type: "tool_use", - name: "read_file", - params: { ...params }, - partial: false, - nativeArgs: { - files: [ - { - path: options.path || testFilePath, - lineRanges, - }, - ], - }, - } - - await readFileTool.handle(mockCline, toolUse, { - askApproval: mockCline.ask, - handleError: vi.fn(), - pushToolResult: (result: ToolResponse) => { - toolResult = result - }, - }) - - return toolResult - } - - describe("when maxReadFileLine is negative", () => { - it("should read the entire file using extractTextFromFile", async () => { - // Setup - use default mockInputContent - mockInputContent = fileContent + describe("input validation", () => { + it("should return error when path is missing", async () => { + const mockTask = createMockTask() + const callbacks = createMockCallbacks() - // Execute - const result = await executeReadFileTool({}, { maxReadFileLine: -1 }) + await readFileTool.execute({ path: "" } as any, mockTask as any, callbacks) - // Verify - check that the result contains the expected native format elements - expect(result).toContain(`File: ${testFilePath}`) - expect(result).toContain(`Lines 1-5:`) + expect(mockTask.consecutiveMistakeCount).toBe(1) + expect(mockTask.recordToolError).toHaveBeenCalledWith("read_file") + expect(mockTask.sayAndCreateMissingParamError).toHaveBeenCalledWith("read_file", "path") + expect(callbacks.pushToolResult).toHaveBeenCalledWith(expect.stringContaining("Error:")) }) - it("should not show line snippet in approval message when maxReadFileLine is -1", async () => { - // This test verifies the line snippet behavior for the approval message - // Setup - use default mockInputContent - mockInputContent = fileContent + it("should return error when path is undefined", async () => { + const mockTask = createMockTask() + const callbacks = createMockCallbacks() - // Execute - we'll reuse executeReadFileTool to run the tool - await executeReadFileTool({}, { maxReadFileLine: -1 }) + await readFileTool.execute({} as any, mockTask as any, callbacks) - // Verify the empty line snippet for full read was passed to the approval message - // Look at the parameters passed to the 'ask' method in the approval message - const askCall = mockCline.ask.mock.calls[0] - const completeMessage = JSON.parse(askCall[1]) - - // Verify the reason (lineSnippet) is empty or undefined for full read - expect(completeMessage.reason).toBeFalsy() + expect(mockTask.consecutiveMistakeCount).toBe(1) + expect(callbacks.pushToolResult).toHaveBeenCalledWith(expect.stringContaining("Error:")) }) }) - describe("when maxReadFileLine is 0", () => { - it("should return an empty content with source code definitions", async () => { - // Setup - for maxReadFileLine = 0, the implementation won't call readLines - mockedParseSourceCodeDefinitionsForFile.mockResolvedValue(sourceCodeDef) + describe("RooIgnore handling", () => { + it("should block access to rooignore-protected files", async () => { + const mockTask = createMockTask({ rooIgnoreAllowed: false }) + const callbacks = createMockCallbacks() - // Execute - skip addLineNumbers check as it's not called for maxReadFileLine=0 - const result = await executeReadFileTool( - {}, - { - maxReadFileLine: 0, - totalLines: 5, - skipAddLineNumbersCheck: true, - }, - ) + await readFileTool.execute({ path: "secret.env" }, mockTask as any, callbacks) - // Verify - native format - expect(result).toContain(`File: ${testFilePath}`) - expect(result).toContain(`Code Definitions:`) - - // Verify native structure - expect(result).toContain("Note: Showing only 0 of 5 total lines") - expect(result).toContain(sourceCodeDef.trim()) - expect(result).not.toContain("Lines 1-") // No content when maxReadFileLine is 0 + expect(mockTask.say).toHaveBeenCalledWith("rooignore_error", "secret.env") + expect(formatResponse.rooIgnoreError).toHaveBeenCalledWith("secret.env") + expect(callbacks.pushToolResult).toHaveBeenCalledWith(expect.stringContaining("blocked by the .rooignore")) }) }) - describe("when maxReadFileLine is less than file length", () => { - it("should read only maxReadFileLine lines and add source code definitions", async () => { - // Setup - const content = "Line 1\nLine 2\nLine 3" - const numberedContent = "1 | Line 1\n2 | Line 2\n3 | Line 3" - mockedReadLines.mockResolvedValue(content) - mockedParseSourceCodeDefinitionsForFile.mockResolvedValue(sourceCodeDef) - - // Setup addLineNumbers to always return numbered content - addLineNumbersMock.mockReturnValue(numberedContent) - - // Execute - const result = await executeReadFileTool({}, { maxReadFileLine: 3 }) - - // Verify - native format - expect(result).toContain(`File: ${testFilePath}`) - expect(result).toContain(`Lines 1-3:`) - expect(result).toContain(`Code Definitions:`) - expect(result).toContain("Note: Showing only 3 of 5 total lines") - }) + describe("directory handling", () => { + it("should return error when trying to read a directory", async () => { + const mockTask = createMockTask() + const callbacks = createMockCallbacks() - it("should truncate code definitions when file exceeds maxReadFileLine", async () => { - // Setup - file with 100 lines but we'll only read first 30 - const content = "Line 1\nLine 2\nLine 3" - const numberedContent = "1 | Line 1\n2 | Line 2\n3 | Line 3" - const fullDefinitions = `# file.txt -10--20 | function foo() { -50--60 | function bar() { -80--90 | function baz() {` - const truncatedDefinitions = `# file.txt -10--20 | function foo() {` - - mockedReadLines.mockResolvedValue(content) - mockedParseSourceCodeDefinitionsForFile.mockResolvedValue(fullDefinitions) - addLineNumbersMock.mockReturnValue(numberedContent) - - // Execute with maxReadFileLine = 30 - const result = await executeReadFileTool({}, { maxReadFileLine: 30, totalLines: 100 }) - - // Verify - native format - expect(result).toContain(`File: ${testFilePath}`) - expect(result).toContain(`Lines 1-30:`) - expect(result).toContain(`Code Definitions:`) - - // Should include foo (starts at line 10) but not bar (starts at line 50) or baz (starts at line 80) - expect(result).toContain("10--20 | function foo()") - expect(result).not.toContain("50--60 | function bar()") - expect(result).not.toContain("80--90 | function baz()") - - expect(result).toContain("Note: Showing only 30 of 100 total lines") - }) + mockedFsStat.mockResolvedValue({ isDirectory: () => true } as any) + + await readFileTool.execute({ path: "src/utils" }, mockTask as any, callbacks) - it("should handle truncation when all definitions are beyond the line limit", async () => { - // Setup - all definitions start after maxReadFileLine - const content = "Line 1\nLine 2\nLine 3" - const numberedContent = "1 | Line 1\n2 | Line 2\n3 | Line 3" - const fullDefinitions = `# file.txt -50--60 | function foo() { -80--90 | function bar() {` - - mockedReadLines.mockResolvedValue(content) - mockedParseSourceCodeDefinitionsForFile.mockResolvedValue(fullDefinitions) - addLineNumbersMock.mockReturnValue(numberedContent) - - // Execute with maxReadFileLine = 30 - const result = await executeReadFileTool({}, { maxReadFileLine: 30, totalLines: 100 }) - - // Verify - native format - expect(result).toContain(`File: ${testFilePath}`) - expect(result).toContain(`Lines 1-30:`) - expect(result).toContain(`Code Definitions:`) - expect(result).toContain("# file.txt") - expect(result).not.toContain("50--60 | function foo()") - expect(result).not.toContain("80--90 | function bar()") + expect(mockTask.say).toHaveBeenCalledWith( + "error", + expect.stringContaining("Cannot read 'src/utils' because it is a directory"), + ) + expect(callbacks.pushToolResult).toHaveBeenCalledWith(expect.stringContaining("it is a directory")) + expect(mockTask.didToolFailInCurrentTurn).toBe(true) }) }) - describe("when maxReadFileLine equals or exceeds file length", () => { - it("should use extractTextFromFile when maxReadFileLine > totalLines", async () => { - // Setup - mockedCountFileLines.mockResolvedValue(5) // File shorter than maxReadFileLine - mockInputContent = fileContent - - // Execute - const result = await executeReadFileTool({}, { maxReadFileLine: 10, totalLines: 5 }) - - // Verify - native format - expect(result).toContain(`File: ${testFilePath}`) - expect(result).toContain(`Lines 1-5:`) + describe("image handling", () => { + beforeEach(() => { + mockedIsBinaryFile.mockResolvedValue(true) + mockedIsSupportedImageFormat.mockReturnValue(true) }) - it("should read with extractTextFromFile when file has few lines", async () => { - // Setup - mockedCountFileLines.mockResolvedValue(3) // File shorter than maxReadFileLine - const threeLineContent = "Line 1\nLine 2\nLine 3" - mockInputContent = threeLineContent - - // Configure the mock to return the correct content for this test - mockReadFileWithTokenBudget.mockResolvedValueOnce({ - content: threeLineContent, - tokenCount: threeLineContent.length / 4, - lineCount: 3, - complete: true, + it("should process image file when model supports images", async () => { + const mockTask = createMockTask({ supportsImages: true }) + const callbacks = createMockCallbacks() + + mockedValidateImageForProcessing.mockResolvedValue({ + isValid: true, + sizeInMB: 0.5, + }) + mockedProcessImageFile.mockResolvedValue({ + dataUrl: "data:image/png;base64,abc123", + buffer: Buffer.from("test"), + sizeInKB: 512, + sizeInMB: 0.5, + notice: "Image processed successfully", }) - // Execute - const result = await executeReadFileTool({}, { maxReadFileLine: 5, totalLines: 3 }) + await readFileTool.execute({ path: "image.png" }, mockTask as any, callbacks) - // Verify - native format - expect(result).toContain(`File: ${testFilePath}`) - expect(result).toContain(`Lines 1-3:`) + expect(mockedValidateImageForProcessing).toHaveBeenCalled() + expect(mockedProcessImageFile).toHaveBeenCalled() + expect(callbacks.pushToolResult).toHaveBeenCalled() }) - }) - - describe("when file is binary", () => { - it("should always use extractTextFromFile regardless of maxReadFileLine", async () => { - // Setup - mockedIsBinaryFile.mockResolvedValue(true) - mockedCountFileLines.mockResolvedValue(3) - mockedExtractTextFromFile.mockResolvedValue("") - // Execute - const result = await executeReadFileTool({}, { maxReadFileLine: 3, totalLines: 3 }) + it("should skip image when model does not support images", async () => { + const mockTask = createMockTask({ supportsImages: false }) + const callbacks = createMockCallbacks() - // Verify - native format for binary files - expect(result).toContain(`File: ${testFilePath}`) - expect(typeof result).toBe("string") - }) - }) + mockedValidateImageForProcessing.mockResolvedValue({ + isValid: false, + reason: "unsupported_model", + notice: "Model does not support image processing", + }) - describe("with range parameters", () => { - it("should honor start_line and end_line when provided", async () => { - // Setup - mockedReadLines.mockResolvedValue("Line 2\nLine 3\nLine 4") + await readFileTool.execute({ path: "image.png" }, mockTask as any, callbacks) - // Execute using executeReadFileTool with range parameters - const rangeResult = await executeReadFileTool( - {}, - { - start_line: "2", - end_line: "4", - }, + expect(mockedValidateImageForProcessing).toHaveBeenCalled() + expect(mockedProcessImageFile).not.toHaveBeenCalled() + expect(callbacks.pushToolResult).toHaveBeenCalledWith( + expect.stringContaining("Model does not support image processing"), ) - - // Verify - native format - expect(rangeResult).toContain(`File: ${testFilePath}`) - expect(rangeResult).toContain(`Lines 2-4:`) }) - }) -}) -describe("read_file tool output structure", () => { - // Test basic native structure - const testFilePath = "test/file.txt" - const absoluteFilePath = "/test/file.txt" - const fileContent = "Line 1\nLine 2\nLine 3\nLine 4\nLine 5" - - const mockedCountFileLines = vi.mocked(countFileLines) - const mockedExtractTextFromFile = vi.mocked(extractTextFromFile) - const mockedIsBinaryFile = vi.mocked(isBinaryFile) - const mockedPathResolve = vi.mocked(path.resolve) - const mockedFsReadFile = vi.mocked(fsPromises.readFile) - const imageBuffer = Buffer.from( - "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==", - "base64", - ) - - let mockCline: any - let mockProvider: any - let toolResult: ToolResponse | undefined + it("should skip image when file exceeds size limit", async () => { + const mockTask = createMockTask({ supportsImages: true, maxImageFileSize: 1 }) + const callbacks = createMockCallbacks() - beforeEach(() => { - // Clear specific mocks (not all mocks to preserve shared state) - mockedCountFileLines.mockClear() - mockedExtractTextFromFile.mockClear() - mockedIsBinaryFile.mockClear() - mockedPathResolve.mockClear() - addLineNumbersMock.mockClear() - extractTextFromFileMock.mockClear() - toolResultMock.mockClear() - - // CRITICAL: Reset fsPromises mocks to prevent cross-test contamination - fsPromises.stat.mockClear() - fsPromises.stat.mockResolvedValue({ - size: 1024, - isDirectory: () => false, - isFile: () => true, - isSymbolicLink: () => false, - } as any) - fsPromises.readFile.mockClear() - - // Use shared mock setup function - const mocks = createMockCline() - mockCline = mocks.mockCline - mockProvider = mocks.mockProvider - - // Explicitly enable image support for this test suite (contains image memory tests) - setImageSupport(mockCline, true) - - mockedPathResolve.mockReturnValue(absoluteFilePath) - mockedIsBinaryFile.mockResolvedValue(false) + mockedValidateImageForProcessing.mockResolvedValue({ + isValid: false, + reason: "size_limit", + notice: "Image file size (10 MB) exceeds the maximum allowed size (1 MB)", + }) - // Set default implementation for extractTextFromFile - mockedExtractTextFromFile.mockImplementation((filePath) => { - return Promise.resolve(addLineNumbersMock(mockInputContent)) - }) + await readFileTool.execute({ path: "large-image.png" }, mockTask as any, callbacks) - mockInputContent = fileContent + expect(mockedProcessImageFile).not.toHaveBeenCalled() + expect(callbacks.pushToolResult).toHaveBeenCalledWith( + expect.stringContaining("exceeds the maximum allowed"), + ) + }) - // Setup mock provider with default maxReadFileLine - mockProvider.getState.mockResolvedValue({ maxReadFileLine: -1, maxImageFileSize: 20, maxTotalImageSize: 20 }) // Default to full file read + it("should skip image when total memory limit exceeded", async () => { + const mockTask = createMockTask({ supportsImages: true, maxTotalImageSize: 5 }) + const callbacks = createMockCallbacks() - // Add additional properties needed for missing param validation tests - mockCline.sayAndCreateMissingParamError = vi.fn().mockResolvedValue("Missing required parameter") + mockedValidateImageForProcessing.mockResolvedValue({ + isValid: false, + reason: "memory_limit", + notice: "Skipping image: would exceed total memory limit", + }) - toolResult = undefined - }) + await readFileTool.execute({ path: "another-image.png" }, mockTask as any, callbacks) - async function executeReadFileTool( - options: { - totalLines?: number - maxReadFileLine?: number - isBinary?: boolean - validateAccess?: boolean - filePath?: string - } = {}, - ): Promise { - // Configure mocks based on test scenario - const totalLines = options.totalLines ?? 5 - const maxReadFileLine = options.maxReadFileLine ?? 500 - const isBinary = options.isBinary ?? false - const validateAccess = options.validateAccess ?? true - - mockProvider.getState.mockResolvedValue({ maxReadFileLine, maxImageFileSize: 20, maxTotalImageSize: 20 }) - mockedCountFileLines.mockResolvedValue(totalLines) - mockedIsBinaryFile.mockResolvedValue(isBinary) - mockCline.rooIgnoreController.validateAccess = vi.fn().mockReturnValue(validateAccess) - const filePath = options.filePath ?? testFilePath - - // Create a tool use object - const toolUse: ReadFileToolUse = { - type: "tool_use", - name: "read_file", - params: {}, - partial: false, - nativeArgs: { - files: [{ path: filePath, lineRanges: [] }], - }, - } - - // Execute the tool - await readFileTool.handle(mockCline, toolUse, { - askApproval: mockCline.ask, - handleError: vi.fn(), - pushToolResult: (result: ToolResponse) => { - toolResult = result - }, + expect(mockedProcessImageFile).not.toHaveBeenCalled() + expect(callbacks.pushToolResult).toHaveBeenCalledWith(expect.stringContaining("would exceed total memory")) }) - return toolResult - } + it("should handle image read errors gracefully", async () => { + const mockTask = createMockTask({ supportsImages: true }) + const callbacks = createMockCallbacks() - describe("Basic Structure Tests", () => { - it("should produce native output with proper format", async () => { - // Setup - const numberedContent = "1 | Line 1\n2 | Line 2\n3 | Line 3\n4 | Line 4\n5 | Line 5" - - // Configure mockReadFileWithTokenBudget to return the 5-line content - mockReadFileWithTokenBudget.mockResolvedValueOnce({ - content: fileContent, // "Line 1\nLine 2\nLine 3\nLine 4\nLine 5" - tokenCount: fileContent.length / 4, - lineCount: 5, - complete: true, + mockedValidateImageForProcessing.mockResolvedValue({ + isValid: true, + sizeInMB: 0.5, }) + mockedProcessImageFile.mockRejectedValue(new Error("Failed to read image")) - mockProvider.getState.mockResolvedValue({ - maxReadFileLine: -1, - maxImageFileSize: 20, - maxTotalImageSize: 20, - }) // Allow up to 20MB per image and total size - - // Execute - const result = await executeReadFileTool() + await readFileTool.execute({ path: "corrupt.png" }, mockTask as any, callbacks) - // Verify native format - expect(result).toBe(`File: ${testFilePath}\nLines 1-5:\n${numberedContent}`) + expect(mockTask.say).toHaveBeenCalledWith("error", expect.stringContaining("Error reading image file")) + expect(callbacks.pushToolResult).toHaveBeenCalledWith(expect.stringContaining("Error")) }) + }) - it("should follow the correct native structure format", async () => { - // Setup - mockInputContent = fileContent - // Execute - const result = await executeReadFileTool({ maxReadFileLine: -1 }) - - // Verify using regex to check native structure - const nativeStructureRegex = new RegExp(`^File: ${testFilePath}\\nLines 1-5:\\n.*$`, "s") - expect(result).toMatch(nativeStructureRegex) + describe("binary file handling", () => { + beforeEach(() => { + mockedIsBinaryFile.mockResolvedValue(true) + mockedIsSupportedImageFormat.mockReturnValue(false) }) - it("should handle empty files correctly", async () => { - // Setup - mockedCountFileLines.mockResolvedValue(0) + it("should extract text from PDF files", async () => { + const mockTask = createMockTask() + const callbacks = createMockCallbacks() - // Configure mockReadFileWithTokenBudget to return empty content - mockReadFileWithTokenBudget.mockResolvedValueOnce({ - content: "", - tokenCount: 0, - lineCount: 0, - complete: true, - }) + mockedExtractTextFromFile.mockResolvedValue("PDF content here") - mockProvider.getState.mockResolvedValue({ - maxReadFileLine: -1, - maxImageFileSize: 20, - maxTotalImageSize: 20, - }) // Allow up to 20MB per image and total size + await readFileTool.execute({ path: "document.pdf" }, mockTask as any, callbacks) - // Execute - const result = await executeReadFileTool({ totalLines: 0 }) - - // Verify native format for empty file - expect(result).toBe(`File: ${testFilePath}\nNote: File is empty`) + expect(mockedExtractTextFromFile).toHaveBeenCalled() + expect(callbacks.pushToolResult).toHaveBeenCalledWith(expect.stringContaining("PDF content here")) }) - describe("Total Image Memory Limit", () => { - const testImages = [ - { path: "test/image1.png", sizeKB: 5120 }, // 5MB - { path: "test/image2.jpg", sizeKB: 10240 }, // 10MB - { path: "test/image3.gif", sizeKB: 8192 }, // 8MB - ] - - // Define imageBuffer for this test suite - const imageBuffer = Buffer.from( - "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==", - "base64", - ) - - beforeEach(() => { - // CRITICAL: Reset fsPromises mocks to prevent cross-test contamination within this suite - fsPromises.stat.mockClear() - fsPromises.readFile.mockClear() - }) + it("should extract text from DOCX files", async () => { + const mockTask = createMockTask() + const callbacks = createMockCallbacks() - async function executeReadMultipleImagesTool(imagePaths: string[]): Promise { - // Ensure image support is enabled before calling the tool - setImageSupport(mockCline, true) - - const toolUse: ReadFileToolUse = { - type: "tool_use", - name: "read_file", - params: {}, - partial: false, - nativeArgs: { - files: imagePaths.map((p) => ({ path: p, lineRanges: [] })), - }, - } - - let localResult: ToolResponse | undefined - await readFileTool.handle(mockCline, toolUse, { - askApproval: mockCline.ask, - handleError: vi.fn(), - pushToolResult: (result: ToolResponse) => { - localResult = result - }, - }) - // In multi-image scenarios, the result is pushed to pushToolResult, not returned directly. - // We need to check the mock's calls to get the result. - if (mockCline.pushToolResult.mock.calls.length > 0) { - return mockCline.pushToolResult.mock.calls[0][0] - } - - return localResult - } + mockedExtractTextFromFile.mockResolvedValue("DOCX content here") - it("should allow multiple images under the total memory limit", async () => { - // Setup required mocks (don't clear all mocks - preserve API setup) - mockedIsBinaryFile.mockResolvedValue(true) - mockedCountFileLines.mockResolvedValue(0) - fsPromises.readFile.mockResolvedValue( - Buffer.from( - "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==", - "base64", - ), - ) - - // Setup mockProvider - mockProvider.getState.mockResolvedValue({ - maxReadFileLine: -1, - maxImageFileSize: 20, - maxTotalImageSize: 20, - }) // Allow up to 20MB per image and total size - - // Setup mockCline properties (preserve existing API) - mockCline.cwd = "/" - mockCline.task = "Test" - mockCline.providerRef = mockProvider - mockCline.rooIgnoreController = { - validateAccess: vi.fn().mockReturnValue(true), - } - mockCline.say = vi.fn().mockResolvedValue(undefined) - mockCline.ask = vi.fn().mockResolvedValue({ response: "yesButtonClicked" }) - mockCline.presentAssistantMessage = vi.fn() - mockCline.handleError = vi.fn().mockResolvedValue(undefined) - mockCline.pushToolResult = vi.fn() - mockCline.fileContextTracker = { - trackFileContext: vi.fn().mockResolvedValue(undefined), - } - mockCline.recordToolUsage = vi.fn().mockReturnValue(undefined) - mockCline.recordToolError = vi.fn().mockReturnValue(undefined) - setImageSupport(mockCline, true) - - // Setup - images that fit within 20MB limit - const smallImages = [ - { path: "test/small1.png", sizeKB: 2048 }, // 2MB - { path: "test/small2.jpg", sizeKB: 3072 }, // 3MB - { path: "test/small3.gif", sizeKB: 4096 }, // 4MB - ] // Total: 9MB (under 20MB limit) - - // Mock file stats for each image - fsPromises.stat = vi.fn().mockImplementation((filePath) => { - const normalizedFilePath = path.normalize(filePath.toString()) - const image = smallImages.find((img) => normalizedFilePath.includes(path.normalize(img.path))) - return Promise.resolve({ size: (image?.sizeKB || 1024) * 1024, isDirectory: () => false }) - }) - - // Mock path.resolve for each image - mockedPathResolve.mockImplementation((cwd, relPath) => `/${relPath}`) - - // Execute - const result = await executeReadMultipleImagesTool(smallImages.map((img) => img.path)) - - // Verify all images were processed (should be a multi-part response) - expect(Array.isArray(result)).toBe(true) - const parts = result as any[] - - // Should have text part and 3 image parts - const textPart = parts.find((p) => p.type === "text")?.text - const imageParts = parts.filter((p) => p.type === "image") - - expect(textPart).toBeDefined() - expect(imageParts).toHaveLength(3) - - // Verify no memory limit notices - expect(textPart).not.toContain("Total image memory would exceed") - }) + await readFileTool.execute({ path: "document.docx" }, mockTask as any, callbacks) - it("should skip images that would exceed the total memory limit", async () => { - // Setup required mocks (don't clear all mocks) - mockedIsBinaryFile.mockResolvedValue(true) - mockedCountFileLines.mockResolvedValue(0) - fsPromises.readFile.mockResolvedValue( - Buffer.from( - "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==", - "base64", - ), - ) - - // Setup mockProvider - mockProvider.getState.mockResolvedValue({ - maxReadFileLine: -1, - maxImageFileSize: 15, - maxTotalImageSize: 20, - }) // Allow up to 15MB per image and 20MB total size - - // Setup mockCline properties - mockCline.cwd = "/" - mockCline.task = "Test" - mockCline.providerRef = mockProvider - mockCline.rooIgnoreController = { - validateAccess: vi.fn().mockReturnValue(true), - } - mockCline.say = vi.fn().mockResolvedValue(undefined) - mockCline.ask = vi.fn().mockResolvedValue({ response: "yesButtonClicked" }) - mockCline.presentAssistantMessage = vi.fn() - mockCline.handleError = vi.fn().mockResolvedValue(undefined) - mockCline.pushToolResult = vi.fn() - mockCline.fileContextTracker = { - trackFileContext: vi.fn().mockResolvedValue(undefined), - } - mockCline.recordToolUsage = vi.fn().mockReturnValue(undefined) - mockCline.recordToolError = vi.fn().mockReturnValue(undefined) - setImageSupport(mockCline, true) - - // Setup - images where later ones would exceed 20MB total limit - // Each must be under 5MB per-file limit (5120KB) - const largeImages = [ - { path: "test/large1.png", sizeKB: 5017 }, // ~4.9MB - { path: "test/large2.jpg", sizeKB: 5017 }, // ~4.9MB - { path: "test/large3.gif", sizeKB: 5017 }, // ~4.9MB - { path: "test/large4.png", sizeKB: 5017 }, // ~4.9MB - { path: "test/large5.jpg", sizeKB: 5017 }, // ~4.9MB - This should be skipped (total would be ~24.5MB > 20MB) - ] + expect(mockedExtractTextFromFile).toHaveBeenCalled() + expect(callbacks.pushToolResult).toHaveBeenCalledWith(expect.stringContaining("DOCX content here")) + }) - // Mock file stats for each image - fsPromises.stat = vi.fn().mockImplementation((filePath) => { - const normalizedFilePath = path.normalize(filePath.toString()) - const image = largeImages.find((img) => normalizedFilePath.includes(path.normalize(img.path))) - return Promise.resolve({ size: (image?.sizeKB || 1024) * 1024, isDirectory: () => false }) - }) + it("should handle unsupported binary formats", async () => { + const mockTask = createMockTask() + const callbacks = createMockCallbacks() - // Mock path.resolve for each image - mockedPathResolve.mockImplementation((cwd, relPath) => `/${relPath}`) + // Return empty array to indicate .exe is not supported + vi.mocked(getSupportedBinaryFormats).mockReturnValue([".pdf", ".docx"]) - // Execute - const result = await executeReadMultipleImagesTool(largeImages.map((img) => img.path)) + await readFileTool.execute({ path: "program.exe" }, mockTask as any, callbacks) - // Verify result structure - should be a mix of successful images and skipped notices - expect(Array.isArray(result)).toBe(true) - const parts = result as any[] + expect(callbacks.pushToolResult).toHaveBeenCalledWith(expect.stringContaining("Binary file")) + }) - const textPart = Array.isArray(result) ? result.find((p) => p.type === "text")?.text : result - const imageParts = Array.isArray(result) ? result.filter((p) => p.type === "image") : [] + it("should handle extraction errors gracefully", async () => { + const mockTask = createMockTask() + const callbacks = createMockCallbacks() - expect(textPart).toBeDefined() + mockedExtractTextFromFile.mockRejectedValue(new Error("Extraction failed")) - // Debug: Show what we actually got vs expected - if (imageParts.length !== 4) { - throw new Error( - `Expected 4 images, got ${imageParts.length}. Full result: ${JSON.stringify(result, null, 2)}. Text part: ${textPart}`, - ) - } - expect(imageParts).toHaveLength(4) // First 4 images should be included (~19.6MB total) + await readFileTool.execute({ path: "corrupt.pdf" }, mockTask as any, callbacks) - // Verify memory limit notice for the fifth image - expect(textPart).toContain("Image skipped to avoid size limit (20MB)") - expect(textPart).toMatch(/Current: \d+(\.\d+)? MB/) - expect(textPart).toMatch(/this file: \d+(\.\d+)? MB/) - }) - - it("should track memory usage correctly across multiple images", async () => { - // Setup mocks (don't clear all mocks) - - // Setup required mocks - mockedIsBinaryFile.mockResolvedValue(true) - mockedCountFileLines.mockResolvedValue(0) - fsPromises.readFile.mockResolvedValue( - Buffer.from( - "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==", - "base64", - ), - ) - - // Setup mockProvider - mockProvider.getState.mockResolvedValue({ - maxReadFileLine: -1, - maxImageFileSize: 15, - maxTotalImageSize: 20, - }) // Allow up to 15MB per image and 20MB total size - - // Setup mockCline properties - mockCline.cwd = "/" - mockCline.task = "Test" - mockCline.providerRef = mockProvider - mockCline.rooIgnoreController = { - validateAccess: vi.fn().mockReturnValue(true), - } - mockCline.say = vi.fn().mockResolvedValue(undefined) - mockCline.ask = vi.fn().mockResolvedValue({ response: "yesButtonClicked" }) - mockCline.presentAssistantMessage = vi.fn() - mockCline.handleError = vi.fn().mockResolvedValue(undefined) - mockCline.pushToolResult = vi.fn() - mockCline.fileContextTracker = { - trackFileContext: vi.fn().mockResolvedValue(undefined), - } - mockCline.recordToolUsage = vi.fn().mockReturnValue(undefined) - mockCline.recordToolError = vi.fn().mockReturnValue(undefined) - setImageSupport(mockCline, true) - - // Setup - images that exactly reach the limit - const exactLimitImages = [ - { path: "test/exact1.png", sizeKB: 10240 }, // 10MB - { path: "test/exact2.jpg", sizeKB: 10240 }, // 10MB - Total exactly 20MB - { path: "test/exact3.gif", sizeKB: 1024 }, // 1MB - This should be skipped - ] + expect(mockTask.say).toHaveBeenCalledWith("error", expect.stringContaining("Error extracting text")) + expect(mockTask.didToolFailInCurrentTurn).toBe(true) + }) + }) - // Mock file stats with simpler logic - fsPromises.stat = vi.fn().mockImplementation((filePath) => { - const normalizedFilePath = path.normalize(filePath.toString()) - const image = exactLimitImages.find((img) => normalizedFilePath.includes(path.normalize(img.path))) - if (image) { - return Promise.resolve({ size: image.sizeKB * 1024, isDirectory: () => false }) - } - return Promise.resolve({ size: 1024 * 1024, isDirectory: () => false }) // Default 1MB - }) - - // Mock path.resolve - mockedPathResolve.mockImplementation((cwd, relPath) => `/${relPath}`) - - // Execute - const result = await executeReadMultipleImagesTool(exactLimitImages.map((img) => img.path)) - - // Verify - const textPart = Array.isArray(result) ? result.find((p) => p.type === "text")?.text : result - const imageParts = Array.isArray(result) ? result.filter((p) => p.type === "image") : [] - - expect(imageParts).toHaveLength(2) // First 2 images should fit - expect(textPart).toContain("Image skipped to avoid size limit (20MB)") - expect(textPart).toMatch(/Current: \d+(\.\d+)? MB/) - expect(textPart).toMatch(/this file: \d+(\.\d+)? MB/) - }) + describe("text file processing", () => { + beforeEach(() => { + mockedIsBinaryFile.mockResolvedValue(false) + }) - it("should handle individual image size limit and total memory limit together", async () => { - // Setup mocks (don't clear all mocks) - - // Setup required mocks - mockedIsBinaryFile.mockResolvedValue(true) - mockedCountFileLines.mockResolvedValue(0) - fsPromises.readFile.mockResolvedValue( - Buffer.from( - "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==", - "base64", - ), - ) - - // Setup mockProvider - mockProvider.getState.mockResolvedValue({ - maxReadFileLine: -1, - maxImageFileSize: 20, - maxTotalImageSize: 20, - }) // Allow up to 20MB per image and total size - - // Setup mockCline properties (complete setup) - mockCline.cwd = "/" - mockCline.task = "Test" - mockCline.providerRef = mockProvider - mockCline.rooIgnoreController = { - validateAccess: vi.fn().mockReturnValue(true), - } - mockCline.say = vi.fn().mockResolvedValue(undefined) - mockCline.ask = vi.fn().mockResolvedValue({ response: "yesButtonClicked" }) - mockCline.presentAssistantMessage = vi.fn() - mockCline.handleError = vi.fn().mockResolvedValue(undefined) - mockCline.pushToolResult = vi.fn() - mockCline.fileContextTracker = { - trackFileContext: vi.fn().mockResolvedValue(undefined), - } - mockCline.recordToolUsage = vi.fn().mockReturnValue(undefined) - mockCline.recordToolError = vi.fn().mockReturnValue(undefined) - setImageSupport(mockCline, true) - - // Setup - mix of images with individual size violations and total memory issues - const mixedImages = [ - { path: "test/ok.png", sizeKB: 3072 }, // 3MB - OK - { path: "test/too-big.jpg", sizeKB: 6144 }, // 6MB - Exceeds individual 5MB limit - { path: "test/ok2.gif", sizeKB: 4096 }, // 4MB - OK individually but might exceed total - ] + it("should read text file with slice mode (default)", async () => { + const mockTask = createMockTask() + const callbacks = createMockCallbacks() - // Mock file stats - fsPromises.stat = vi.fn().mockImplementation((filePath) => { - const fileName = path.basename(filePath) - const baseName = path.parse(fileName).name - const image = mixedImages.find((img) => img.path.includes(baseName)) - return Promise.resolve({ size: (image?.sizeKB || 1024) * 1024, isDirectory: () => false }) - }) - - // Mock provider state with 5MB individual limit - mockProvider.getState.mockResolvedValue({ - maxReadFileLine: -1, - maxImageFileSize: 5, - maxTotalImageSize: 20, - }) - - // Mock path.resolve - mockedPathResolve.mockImplementation((cwd, relPath) => `/${relPath}`) - - // Execute - const result = await executeReadMultipleImagesTool(mixedImages.map((img) => img.path)) - - // Verify - expect(Array.isArray(result)).toBe(true) - const parts = result as any[] - - const textPart = parts.find((p) => p.type === "text")?.text - const imageParts = parts.filter((p) => p.type === "image") - - // Should have 2 images (ok.png and ok2.gif) - expect(imageParts).toHaveLength(2) - - // Should show individual size limit violation - expect(textPart).toMatch( - /Image file is too large \(\d+(\.\d+)? MB\)\. The maximum allowed size is 5 MB\./, - ) + const content = "line 1\nline 2\nline 3" + mockedFsReadFile.mockResolvedValue(Buffer.from(content)) + mockedReadWithSlice.mockReturnValue({ + content: "1 | line 1\n2 | line 2\n3 | line 3", + returnedLines: 3, + totalLines: 3, + wasTruncated: false, + includedRanges: [[1, 3]], }) - it("should correctly calculate total memory and skip the last image", async () => { - // Setup - const testImages = [ - { path: "test/image1.png", sizeMB: 8 }, - { path: "test/image2.png", sizeMB: 8 }, - { path: "test/image3.png", sizeMB: 8 }, // This one should be skipped - ] - - mockProvider.getState.mockResolvedValue({ - maxReadFileLine: -1, - maxImageFileSize: 10, // 10MB per image - maxTotalImageSize: 20, // 20MB total - }) - - mockedIsBinaryFile.mockResolvedValue(true) - mockedCountFileLines.mockResolvedValue(0) - mockedFsReadFile.mockResolvedValue(imageBuffer) - - fsPromises.stat.mockImplementation(async (filePath) => { - const normalizedFilePath = path.normalize(filePath.toString()) - const file = testImages.find((f) => normalizedFilePath.includes(path.normalize(f.path))) - if (file) { - return { size: file.sizeMB * 1024 * 1024, isDirectory: () => false } - } - return { size: 1024 * 1024, isDirectory: () => false } // Default 1MB - }) - - const imagePaths = testImages.map((img) => img.path) - const result = await executeReadMultipleImagesTool(imagePaths) - - expect(Array.isArray(result)).toBe(true) - const parts = result as any[] - const textPart = parts.find((p) => p.type === "text")?.text - const imageParts = parts.filter((p) => p.type === "image") - - expect(imageParts).toHaveLength(2) // First two images should be processed - expect(textPart).toContain("Image skipped to avoid size limit (20MB)") - expect(textPart).toMatch(/Current: \d+(\.\d+)? MB/) - expect(textPart).toMatch(/this file: \d+(\.\d+)? MB/) - }) + await readFileTool.execute({ path: "test.ts" }, mockTask as any, callbacks) - it("should reset total memory tracking for each tool invocation", async () => { - // Setup mocks (don't clear all mocks) - - // Setup required mocks for first batch - mockedIsBinaryFile.mockResolvedValue(true) - mockedCountFileLines.mockResolvedValue(0) - fsPromises.readFile.mockResolvedValue( - Buffer.from( - "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==", - "base64", - ), - ) - - // Setup mockProvider - mockProvider.getState.mockResolvedValue({ - maxReadFileLine: -1, - maxImageFileSize: 20, - maxTotalImageSize: 20, - }) - - // Setup mockCline properties (complete setup) - mockCline.cwd = "/" - mockCline.task = "Test" - mockCline.providerRef = mockProvider - mockCline.rooIgnoreController = { - validateAccess: vi.fn().mockReturnValue(true), - } - mockCline.say = vi.fn().mockResolvedValue(undefined) - mockCline.ask = vi.fn().mockResolvedValue({ response: "yesButtonClicked" }) - mockCline.presentAssistantMessage = vi.fn() - mockCline.handleError = vi.fn().mockResolvedValue(undefined) - mockCline.pushToolResult = vi.fn() - mockCline.fileContextTracker = { - trackFileContext: vi.fn().mockResolvedValue(undefined), - } - mockCline.recordToolUsage = vi.fn().mockReturnValue(undefined) - mockCline.recordToolError = vi.fn().mockReturnValue(undefined) - setImageSupport(mockCline, true) - - // Setup - first call with images that use memory - const firstBatch = [{ path: "test/first.png", sizeKB: 10240 }] // 10MB - - fsPromises.stat = vi.fn().mockResolvedValue({ size: 10240 * 1024, isDirectory: () => false }) - mockedPathResolve.mockImplementation((cwd, relPath) => `/${relPath}`) - - // Execute first batch - await executeReadMultipleImagesTool(firstBatch.map((img) => img.path)) - - // Setup second batch (don't clear all mocks) - mockedIsBinaryFile.mockResolvedValue(true) - mockedCountFileLines.mockResolvedValue(0) - fsPromises.readFile.mockResolvedValue( - Buffer.from( - "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==", - "base64", - ), - ) - mockProvider.getState.mockResolvedValue({ - maxReadFileLine: -1, - maxImageFileSize: 20, - maxTotalImageSize: 20, - }) - - // Reset path resolving for second batch - mockedPathResolve.mockClear() - - // Re-setup mockCline properties for second batch (complete setup) - mockCline.cwd = "/" - mockCline.task = "Test" - mockCline.providerRef = mockProvider - mockCline.rooIgnoreController = { - validateAccess: vi.fn().mockReturnValue(true), - } - mockCline.say = vi.fn().mockResolvedValue(undefined) - mockCline.ask = vi.fn().mockResolvedValue({ response: "yesButtonClicked" }) - mockCline.presentAssistantMessage = vi.fn() - mockCline.handleError = vi.fn().mockResolvedValue(undefined) - mockCline.pushToolResult = vi.fn() - mockCline.fileContextTracker = { - trackFileContext: vi.fn().mockResolvedValue(undefined), - } - mockCline.recordToolUsage = vi.fn().mockReturnValue(undefined) - mockCline.recordToolError = vi.fn().mockReturnValue(undefined) - setImageSupport(mockCline, true) - - const secondBatch = [{ path: "test/second.png", sizeKB: 15360 }] // 15MB - - // Clear and reset file system mocks for second batch - fsPromises.stat.mockClear() - fsPromises.readFile.mockClear() - mockedIsBinaryFile.mockClear() - mockedCountFileLines.mockClear() - - // Reset mocks for second batch - fsPromises.stat = vi.fn().mockResolvedValue({ size: 15360 * 1024, isDirectory: () => false }) - fsPromises.readFile.mockResolvedValue( - Buffer.from( - "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==", - "base64", - ), - ) - mockedIsBinaryFile.mockResolvedValue(true) - mockedCountFileLines.mockResolvedValue(0) - mockedPathResolve.mockImplementation((cwd, relPath) => `/${relPath}`) - - // Execute second batch - const result = await executeReadMultipleImagesTool(secondBatch.map((img) => img.path)) - - // Verify second batch is processed successfully (memory tracking was reset) - expect(Array.isArray(result)).toBe(true) - const parts = result as any[] - const imageParts = parts.filter((p) => p.type === "image") - - expect(imageParts).toHaveLength(1) // Second image should be processed - }) + expect(mockedReadWithSlice).toHaveBeenCalled() + expect(callbacks.pushToolResult).toHaveBeenCalledWith(expect.stringContaining("line 1")) + }) - it("should handle a folder with many images just under the individual size limit", async () => { - // Setup - Create many images that are each just under the 5MB individual limit - // but together approach the 20MB total limit - const manyImages = [ - { path: "test/img1.png", sizeKB: 4900 }, // 4.78MB - { path: "test/img2.png", sizeKB: 4900 }, // 4.78MB - { path: "test/img3.png", sizeKB: 4900 }, // 4.78MB - { path: "test/img4.png", sizeKB: 4900 }, // 4.78MB - { path: "test/img5.png", sizeKB: 4900 }, // 4.78MB - This should be skipped (total would be ~23.9MB) - ] + it("should read text file with offset and limit", async () => { + const mockTask = createMockTask() + const callbacks = createMockCallbacks() - // Setup mocks - mockedIsBinaryFile.mockResolvedValue(true) - mockedCountFileLines.mockResolvedValue(0) - fsPromises.readFile.mockResolvedValue(imageBuffer) - - // Setup provider with 5MB individual limit and 20MB total limit - mockProvider.getState.mockResolvedValue({ - maxReadFileLine: -1, - maxImageFileSize: 5, - maxTotalImageSize: 20, - }) - - // Mock file stats for each image - fsPromises.stat = vi.fn().mockImplementation((filePath) => { - const normalizedFilePath = path.normalize(filePath.toString()) - const image = manyImages.find((img) => normalizedFilePath.includes(path.normalize(img.path))) - return Promise.resolve({ size: (image?.sizeKB || 1024) * 1024, isDirectory: () => false }) - }) - - // Mock path.resolve - mockedPathResolve.mockImplementation((cwd, relPath) => `/${relPath}`) - - // Execute - const result = await executeReadMultipleImagesTool(manyImages.map((img) => img.path)) - - // Verify - expect(Array.isArray(result)).toBe(true) - const parts = result as any[] - const textPart = parts.find((p) => p.type === "text")?.text - const imageParts = parts.filter((p) => p.type === "image") - - // Should process first 4 images (total ~19.12MB, under 20MB limit) - expect(imageParts).toHaveLength(4) - - // Should show memory limit notice for the 5th image - expect(textPart).toContain("Image skipped to avoid size limit (20MB)") - expect(textPart).toContain("test/img5.png") - - // Verify memory tracking worked correctly - // The notice should show current memory usage around 20MB (4 * 4900KB ≈ 19.14MB, displayed as 20.1MB) - expect(textPart).toMatch(/Current: \d+(\.\d+)? MB/) + mockedFsReadFile.mockResolvedValue(Buffer.from("line 1\nline 2\nline 3\nline 4\nline 5")) + mockedReadWithSlice.mockReturnValue({ + content: "2 | line 2\n3 | line 3", + returnedLines: 2, + totalLines: 5, + wasTruncated: true, + includedRanges: [[2, 3]], }) - it("should reset memory tracking between separate tool invocations more explicitly", async () => { - // This test verifies that totalImageMemoryUsed is reset between calls - // by making two separate tool invocations and ensuring the second one - // starts with fresh memory tracking - - // Setup mocks - mockedIsBinaryFile.mockResolvedValue(true) - mockedCountFileLines.mockResolvedValue(0) - fsPromises.readFile.mockResolvedValue(imageBuffer) - - // Setup provider - mockProvider.getState.mockResolvedValue({ - maxReadFileLine: -1, - maxImageFileSize: 20, - maxTotalImageSize: 20, - }) - - // First invocation - use 15MB of memory - const firstBatch = [{ path: "test/large1.png", sizeKB: 15360 }] // 15MB - - fsPromises.stat = vi.fn().mockResolvedValue({ size: 15360 * 1024, isDirectory: () => false }) - mockedPathResolve.mockImplementation((cwd, relPath) => `/${relPath}`) - - // Execute first batch - const result1 = await executeReadMultipleImagesTool(firstBatch.map((img) => img.path)) - - // Verify first batch processed successfully - expect(Array.isArray(result1)).toBe(true) - const parts1 = result1 as any[] - const imageParts1 = parts1.filter((p) => p.type === "image") - expect(imageParts1).toHaveLength(1) - - // Second invocation - should start with 0 memory used, not 15MB - // If memory tracking wasn't reset, this 18MB image would be rejected - const secondBatch = [{ path: "test/large2.png", sizeKB: 18432 }] // 18MB - - // Reset mocks for second invocation - fsPromises.stat.mockClear() - fsPromises.readFile.mockClear() - mockedPathResolve.mockClear() - - fsPromises.stat = vi.fn().mockResolvedValue({ size: 18432 * 1024, isDirectory: () => false }) - fsPromises.readFile.mockResolvedValue(imageBuffer) - mockedPathResolve.mockImplementation((cwd, relPath) => `/${relPath}`) - - // Execute second batch - const result2 = await executeReadMultipleImagesTool(secondBatch.map((img) => img.path)) - - // Verify second batch processed successfully - expect(Array.isArray(result2)).toBe(true) - const parts2 = result2 as any[] - const imageParts2 = parts2.filter((p) => p.type === "image") - const textPart2 = parts2.find((p) => p.type === "text")?.text - - // The 18MB image should be processed successfully because memory was reset - expect(imageParts2).toHaveLength(1) - - // Should NOT contain any memory limit notices - expect(textPart2).not.toContain("Image skipped to avoid memory limit") + await readFileTool.execute( + { path: "test.ts", mode: "slice", offset: 2, limit: 2 }, + mockTask as any, + callbacks, + ) - // This proves memory tracking was reset between invocations - }) + expect(mockedReadWithSlice).toHaveBeenCalledWith(expect.any(String), 1, 2) // offset converted to 0-based }) - }) - describe("Error Handling Tests", () => { - it("should include error in output for invalid path", async () => { - // Setup - missing path parameter - const toolUse: ReadFileToolUse = { - type: "tool_use", - name: "read_file", - params: {}, - partial: false, - nativeArgs: { - files: [], - }, - } + it("should read text file with indentation mode", async () => { + const mockTask = createMockTask() + const callbacks = createMockCallbacks() - // Execute the tool - await readFileTool.handle(mockCline, toolUse, { - askApproval: mockCline.ask, - handleError: vi.fn(), - pushToolResult: (result: ToolResponse) => { - toolResult = result - }, + const content = "class Foo {\n method() {\n return 42\n }\n}" + mockedFsReadFile.mockResolvedValue(Buffer.from(content)) + mockedReadWithIndentation.mockReturnValue({ + content: "1 | class Foo {\n2 | method() {\n3 | return 42\n4 | }\n5 | }", + returnedLines: 5, + totalLines: 5, + wasTruncated: false, + includedRanges: [[1, 5]], }) - // Verify - native format for error - expect(toolResult).toBe(`Error: Missing required parameter`) - }) - - it("should include error for RooIgnore error", async () => { - // Execute - skip addLineNumbers check as it returns early with an error - const result = await executeReadFileTool({ validateAccess: false }) + await readFileTool.execute( + { + path: "test.ts", + mode: "indentation", + indentation: { anchor_line: 3 }, + }, + mockTask as any, + callbacks, + ) - // Verify - native format for error - expect(result).toBe( - `File: ${testFilePath}\nError: Access to ${testFilePath} is blocked by the .rooignore file settings. You must try to continue in the task without using this file, or ask the user to update the .rooignore file.`, + expect(mockedReadWithIndentation).toHaveBeenCalledWith( + content, + expect.objectContaining({ + anchorLine: 3, + }), ) }) - it("should provide helpful error when trying to read a directory", async () => { - // Setup - mock fsPromises.stat to indicate the path is a directory - const dirPath = "test/my-directory" - const absoluteDirPath = "/test/my-directory" - - mockedPathResolve.mockReturnValue(absoluteDirPath) - - // Mock fs/promises stat to return directory - fsPromises.stat.mockResolvedValue({ - isDirectory: () => true, - isFile: () => false, - isSymbolicLink: () => false, - } as any) + it("should show truncation notice when content is truncated", async () => { + const mockTask = createMockTask() + const callbacks = createMockCallbacks() - // Mock isBinaryFile won't be called since we check directory first - mockedIsBinaryFile.mockResolvedValue(false) - - // Execute - const result = await executeReadFileTool({ filePath: dirPath }) + mockedFsReadFile.mockResolvedValue(Buffer.from("lots of content...")) + mockedReadWithSlice.mockReturnValue({ + content: "1 | truncated content", + returnedLines: 100, + totalLines: 5000, + wasTruncated: true, + includedRanges: [[1, 100]], + }) - // Verify - native format for error - expect(result).toContain(`File: ${dirPath}`) - expect(result).toContain(`Error: Error reading file: Cannot read '${dirPath}' because it is a directory`) - expect(result).toContain("use the list_files tool instead") + await readFileTool.execute({ path: "large.ts" }, mockTask as any, callbacks) - // Verify that task.say was called with the error - expect(mockCline.say).toHaveBeenCalledWith("error", expect.stringContaining("Cannot read")) - expect(mockCline.say).toHaveBeenCalledWith("error", expect.stringContaining("is a directory")) - expect(mockCline.say).toHaveBeenCalledWith("error", expect.stringContaining("list_files tool")) + expect(callbacks.pushToolResult).toHaveBeenCalledWith(expect.stringContaining("truncated")) + expect(callbacks.pushToolResult).toHaveBeenCalledWith(expect.stringContaining("To read more")) }) - }) -}) - -describe("read_file tool with image support", () => { - const testImagePath = "test/image.png" - const absoluteImagePath = "/test/image.png" - const base64ImageData = - "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==" - const imageBuffer = Buffer.from(base64ImageData, "base64") - - const mockedCountFileLines = vi.mocked(countFileLines) - const mockedIsBinaryFile = vi.mocked(isBinaryFile) - const mockedPathResolve = vi.mocked(path.resolve) - const mockedFsReadFile = vi.mocked(fsPromises.readFile) - const mockedExtractTextFromFile = vi.mocked(extractTextFromFile) - let localMockCline: any - let localMockProvider: any - let toolResult: ToolResponse | undefined + it("should handle empty files", async () => { + const mockTask = createMockTask() + const callbacks = createMockCallbacks() - beforeEach(() => { - // Clear specific mocks (not all mocks to preserve shared state) - mockedPathResolve.mockClear() - mockedIsBinaryFile.mockClear() - mockedCountFileLines.mockClear() - mockedFsReadFile.mockClear() - mockedExtractTextFromFile.mockClear() - toolResultMock.mockClear() - - // CRITICAL: Reset fsPromises.stat to prevent cross-test contamination - fsPromises.stat.mockClear() - fsPromises.stat.mockResolvedValue({ - size: 1024, - isDirectory: () => false, - isFile: () => true, - isSymbolicLink: () => false, - } as any) - - // Use shared mock setup function with local variables - const mocks = createMockCline() - localMockCline = mocks.mockCline - localMockProvider = mocks.mockProvider - - // CRITICAL: Explicitly ensure image support is enabled for all tests in this suite - setImageSupport(localMockCline, true) - - mockedPathResolve.mockReturnValue(absoluteImagePath) - mockedIsBinaryFile.mockResolvedValue(true) - mockedCountFileLines.mockResolvedValue(0) - mockedFsReadFile.mockResolvedValue(imageBuffer) - - // Setup mock provider with default maxReadFileLine - localMockProvider.getState.mockResolvedValue({ maxReadFileLine: -1 }) - - toolResult = undefined - }) - - async function executeReadImageTool(imagePath: string = testImagePath): Promise { - const toolUse: ReadFileToolUse = { - type: "tool_use", - name: "read_file", - params: {}, - partial: false, - nativeArgs: { - files: [{ path: imagePath, lineRanges: [] }], - }, - } - - // Debug: Check if mock is working - console.log("Mock API:", localMockCline.api) - console.log("Supports images:", localMockCline.api?.getModel?.()?.info?.supportsImages) - - await readFileTool.handle(localMockCline, toolUse, { - askApproval: localMockCline.ask, - handleError: vi.fn(), - pushToolResult: (result: ToolResponse) => { - toolResult = result - }, - }) + mockedFsReadFile.mockResolvedValue(Buffer.from("")) + mockedReadWithSlice.mockReturnValue({ + content: "", + returnedLines: 0, + totalLines: 0, + wasTruncated: false, + includedRanges: [], + }) - console.log("Result type:", Array.isArray(toolResult) ? "array" : typeof toolResult) - console.log("Result:", toolResult) + await readFileTool.execute({ path: "empty.ts" }, mockTask as any, callbacks) - return toolResult - } - - describe("Image Format Detection", () => { - it.each([ - [".png", "image.png", "image/png"], - [".jpg", "photo.jpg", "image/jpeg"], - [".jpeg", "picture.jpeg", "image/jpeg"], - [".gif", "animation.gif", "image/gif"], - [".bmp", "bitmap.bmp", "image/bmp"], - [".svg", "vector.svg", "image/svg+xml"], - [".webp", "modern.webp", "image/webp"], - [".ico", "favicon.ico", "image/x-icon"], - [".avif", "new-format.avif", "image/avif"], - ])("should detect %s as an image format", async (ext, filename, expectedMimeType) => { - // Setup - const imagePath = `test/${filename}` - const absolutePath = `/test/${filename}` - mockedPathResolve.mockReturnValue(absolutePath) - - // Ensure API mock supports images - setImageSupport(localMockCline, true) - - // Execute - const result = await executeReadImageTool(imagePath) - - // Verify result is a multi-part response - expect(Array.isArray(result)).toBe(true) - const textPart = (result as any[]).find((p) => p.type === "text")?.text - const imagePart = (result as any[]).find((p) => p.type === "image") - - // Verify text part - native format - expect(textPart).toContain(`File: ${imagePath}`) - expect(textPart).not.toContain("") - expect(textPart).toContain(`Note: Image file`) - - // Verify image part - expect(imagePart).toBeDefined() - expect(imagePart.source.media_type).toBe(expectedMimeType) - expect(imagePart.source.data).toBe(base64ImageData) + expect(callbacks.pushToolResult).toHaveBeenCalledWith(expect.stringContaining("empty")) }) }) - describe("Image Reading Functionality", () => { - it("should read image file and return a multi-part response", async () => { - // Execute - const result = await executeReadImageTool() - - // Verify result is a multi-part response - expect(Array.isArray(result)).toBe(true) - const textPart = (result as any[]).find((p) => p.type === "text")?.text - const imagePart = (result as any[]).find((p) => p.type === "image") - - // Verify text part - native format - expect(textPart).toContain(`File: ${testImagePath}`) - expect(textPart).not.toContain(``) - expect(textPart).toContain(`Note: Image file`) - - // Verify image part - expect(imagePart).toBeDefined() - expect(imagePart.source.media_type).toBe("image/png") - expect(imagePart.source.data).toBe(base64ImageData) - }) + describe("approval flow", () => { + it("should approve file read when user clicks yes", async () => { + const mockTask = createMockTask() + const callbacks = createMockCallbacks() - it("should call formatResponse.toolResult with text and image data", async () => { - // Execute - await executeReadImageTool() - - // Verify toolResultMock was called correctly - expect(toolResultMock).toHaveBeenCalledTimes(1) - const callArgs = toolResultMock.mock.calls[0] - const textArg = callArgs[0] - const imagesArg = callArgs[1] - - // Native format - expect(textArg).toContain(`File: ${testImagePath}`) - expect(imagesArg).toBeDefined() - expect(imagesArg).toBeInstanceOf(Array) - expect(imagesArg!.length).toBe(1) - expect(imagesArg![0]).toBe(`data:image/png;base64,${base64ImageData}`) - }) + mockTask.ask.mockResolvedValue({ response: "yesButtonClicked", text: undefined, images: undefined }) - it("should handle large image files", async () => { - // Setup - simulate a large image - const largeBase64 = "A".repeat(1000000) // 1MB of base64 data - const largeBuffer = Buffer.from(largeBase64, "base64") - mockedFsReadFile.mockResolvedValue(largeBuffer) - - // Execute - const result = await executeReadImageTool() - - // Verify it still works with large data - expect(Array.isArray(result)).toBe(true) - const imagePart = (result as any[]).find((p) => p.type === "image") - expect(imagePart).toBeDefined() - expect(imagePart.source.media_type).toBe("image/png") - expect(imagePart.source.data).toBe(largeBase64) - }) + await readFileTool.execute({ path: "test.ts" }, mockTask as any, callbacks) - it("should exclude images when model does not support images", async () => { - // Setup - mock API handler that doesn't support images - setImageSupport(localMockCline, false) - - // Execute - const result = await executeReadImageTool() - - // When images are not supported, the tool should return just text (not call formatResponse.toolResult) - expect(toolResultMock).not.toHaveBeenCalled() - expect(typeof result).toBe("string") - // Native format - expect(result).toContain(`File: ${testImagePath}`) - expect(result).toContain(`Note: Image file`) + expect(mockTask.ask).toHaveBeenCalledWith("tool", expect.any(String), false) + expect(mockTask.didRejectTool).toBe(false) }) - it("should include images when model supports images", async () => { - // Setup - mock API handler that supports images - setImageSupport(localMockCline, true) - - // Execute - const result = await executeReadImageTool() - - // Verify toolResultMock was called with images - expect(toolResultMock).toHaveBeenCalledTimes(1) - const callArgs = toolResultMock.mock.calls[0] - const textArg = callArgs[0] - const imagesArg = callArgs[1] - - // Native format - expect(textArg).toContain(`File: ${testImagePath}`) - expect(imagesArg).toBeDefined() // Images should be included - expect(imagesArg).toBeInstanceOf(Array) - expect(imagesArg!.length).toBe(1) - expect(imagesArg![0]).toBe(`data:image/png;base64,${base64ImageData}`) - }) + it("should deny file read when user clicks no", async () => { + const mockTask = createMockTask() + const callbacks = createMockCallbacks() - it("should handle undefined supportsImages gracefully", async () => { - // Setup - mock API handler with undefined supportsImages - setImageSupport(localMockCline, undefined) + mockTask.ask.mockResolvedValue({ response: "noButtonClicked", text: undefined, images: undefined }) - // Execute - const result = await executeReadImageTool() + await readFileTool.execute({ path: "test.ts" }, mockTask as any, callbacks) - // When supportsImages is undefined, should default to false and return just text - expect(toolResultMock).not.toHaveBeenCalled() - expect(typeof result).toBe("string") - // Native format - expect(result).toContain(`File: ${testImagePath}`) - expect(result).toContain(`Note: Image file`) + expect(mockTask.didRejectTool).toBe(true) + expect(callbacks.pushToolResult).toHaveBeenCalledWith(expect.stringContaining("Denied by user")) }) - it("should handle errors when reading image files", async () => { - // Setup - simulate read error - mockedFsReadFile.mockRejectedValue(new Error("Failed to read image")) - - // Execute - const toolUse: ReadFileToolUse = { - type: "tool_use", - name: "read_file", - params: {}, - partial: false, - nativeArgs: { - files: [{ path: testImagePath, lineRanges: [] }], - }, - } + it("should include user feedback when provided with approval", async () => { + const mockTask = createMockTask() + const callbacks = createMockCallbacks() - await readFileTool.handle(localMockCline, toolUse, { - askApproval: localMockCline.ask, - handleError: vi.fn(), - pushToolResult: (result: ToolResponse) => { - toolResult = result - }, + mockTask.ask.mockResolvedValue({ + response: "yesButtonClicked", + text: "Please be careful with this file", + images: undefined, + }) + mockedFsReadFile.mockResolvedValue(Buffer.from("content")) + mockedReadWithSlice.mockReturnValue({ + content: "1 | content", + returnedLines: 1, + totalLines: 1, + wasTruncated: false, + includedRanges: [[1, 1]], }) - // Verify error handling - native format - expect(toolResult).toContain("Error: Error reading image file: Failed to read image") - // Verify that say was called to show error to user - expect(localMockCline.say).toHaveBeenCalledWith("error", expect.stringContaining("Failed to read image")) - }) - }) - - describe("Binary File Handling", () => { - it("should not treat non-image binary files as images", async () => { - // Setup - const binaryPath = "test/document.pdf" - const absolutePath = "/test/document.pdf" - mockedPathResolve.mockReturnValue(absolutePath) - mockedExtractTextFromFile.mockResolvedValue("PDF content extracted") - - // Execute - const result = await executeReadImageTool(binaryPath) - - // Verify it uses extractTextFromFile instead - expect(result).not.toContain("") - // Make the test platform-agnostic by checking the call was made (path normalization can vary) - expect(mockedExtractTextFromFile).toHaveBeenCalledTimes(1) - const callArgs = mockedExtractTextFromFile.mock.calls[0] - expect(callArgs[0]).toMatch(/[\\\/]test[\\\/]document\.pdf$/) - }) - - it("should handle unknown binary formats", async () => { - // Setup - const binaryPath = "test/unknown.bin" - const absolutePath = "/test/unknown.bin" - mockedPathResolve.mockReturnValue(absolutePath) - mockedExtractTextFromFile.mockResolvedValue("") - - // Execute - const result = await executeReadImageTool(binaryPath) + await readFileTool.execute({ path: "test.ts" }, mockTask as any, callbacks) - // Verify - native format for binary files - expect(result).not.toContain("") - expect(result).toContain("Binary file (bin)") + expect(mockTask.say).toHaveBeenCalledWith("user_feedback", "Please be careful with this file", undefined) + expect(formatResponse.toolApprovedWithFeedback).toHaveBeenCalledWith("Please be careful with this file") }) - }) - describe("Edge Cases", () => { - it("should handle case-insensitive image extensions", async () => { - // Test uppercase extensions - const uppercasePath = "test/IMAGE.PNG" - const absolutePath = "/test/IMAGE.PNG" - mockedPathResolve.mockReturnValue(absolutePath) - - // Execute - const result = await executeReadImageTool(uppercasePath) - - // Verify - expect(Array.isArray(result)).toBe(true) - const imagePart = (result as any[]).find((p) => p.type === "image") - expect(imagePart).toBeDefined() - expect(imagePart.source.media_type).toBe("image/png") - }) + it("should include user feedback when provided with denial", async () => { + const mockTask = createMockTask() + const callbacks = createMockCallbacks() - it("should handle files with multiple dots in name", async () => { - // Setup - const complexPath = "test/my.photo.backup.png" - const absolutePath = "/test/my.photo.backup.png" - mockedPathResolve.mockReturnValue(absolutePath) - - // Execute - const result = await executeReadImageTool(complexPath) - - // Verify - expect(Array.isArray(result)).toBe(true) - const imagePart = (result as any[]).find((p) => p.type === "image") - expect(imagePart).toBeDefined() - expect(imagePart.source.media_type).toBe("image/png") - }) - - it("should handle empty image files", async () => { - // Setup - empty buffer - mockedFsReadFile.mockResolvedValue(Buffer.from("")) + mockTask.ask.mockResolvedValue({ + response: "noButtonClicked", + text: "This file contains secrets", + images: undefined, + }) - // Execute - const result = await executeReadImageTool() + await readFileTool.execute({ path: "secrets.env" }, mockTask as any, callbacks) - // Verify - should still create valid data URL - expect(Array.isArray(result)).toBe(true) - const imagePart = (result as any[]).find((p) => p.type === "image") - expect(imagePart).toBeDefined() - expect(imagePart.source.media_type).toBe("image/png") - expect(imagePart.source.data).toBe("") + expect(mockTask.say).toHaveBeenCalledWith("user_feedback", "This file contains secrets", undefined) + expect(formatResponse.toolDeniedWithFeedback).toHaveBeenCalledWith("This file contains secrets") }) }) -}) -describe("read_file tool concurrent file reads limit", () => { - const mockedCountFileLines = vi.mocked(countFileLines) - const mockedIsBinaryFile = vi.mocked(isBinaryFile) - const mockedPathResolve = vi.mocked(path.resolve) - - let mockCline: any - let mockProvider: any - let toolResult: ToolResponse | undefined - - beforeEach(() => { - // Clear specific mocks - mockedCountFileLines.mockClear() - mockedIsBinaryFile.mockClear() - mockedPathResolve.mockClear() - addLineNumbersMock.mockClear() - toolResultMock.mockClear() - - // Use shared mock setup function - const mocks = createMockCline() - mockCline = mocks.mockCline - mockProvider = mocks.mockProvider - - // Disable image support for these tests - setImageSupport(mockCline, false) - - mockedPathResolve.mockImplementation((cwd, relPath) => `/${relPath}`) - mockedIsBinaryFile.mockResolvedValue(false) - mockedCountFileLines.mockResolvedValue(10) - - // Mock fsPromises.stat to return a file (not directory) by default - fsPromises.stat.mockResolvedValue({ - isDirectory: () => false, - isFile: () => true, - isSymbolicLink: () => false, - } as any) + describe("output structure", () => { + it("should include file path in output", async () => { + const mockTask = createMockTask() + const callbacks = createMockCallbacks() + + mockedFsReadFile.mockResolvedValue(Buffer.from("content")) + mockedReadWithSlice.mockReturnValue({ + content: "1 | content", + returnedLines: 1, + totalLines: 1, + wasTruncated: false, + includedRanges: [[1, 1]], + }) - toolResult = undefined - }) + await readFileTool.execute({ path: "src/app.ts" }, mockTask as any, callbacks) - async function executeReadFileToolWithLimit( - fileCount: number, - maxConcurrentFileReads: number, - ): Promise { - // Setup provider state with the specified limit - mockProvider.getState.mockResolvedValue({ - maxReadFileLine: -1, - maxConcurrentFileReads, - maxImageFileSize: 20, - maxTotalImageSize: 20, + expect(callbacks.pushToolResult).toHaveBeenCalledWith(expect.stringContaining("File: src/app.ts")) }) - const toolUse: ReadFileToolUse = { - type: "tool_use", - name: "read_file", - params: {}, - partial: false, - nativeArgs: { - files: Array.from({ length: fileCount }, (_, i) => ({ path: `file${i + 1}.txt`, lineRanges: [] })), - }, - } - - // Configure mocks for successful file reads - mockReadFileWithTokenBudget.mockResolvedValue({ - content: "test content", - tokenCount: 10, - lineCount: 1, - complete: true, - }) + it("should track file context after successful read", async () => { + const mockTask = createMockTask() + const callbacks = createMockCallbacks() - await readFileTool.handle(mockCline, toolUse, { - askApproval: mockCline.ask, - handleError: vi.fn(), - pushToolResult: (result: ToolResponse) => { - toolResult = result - }, - }) - - return toolResult - } - - it("should reject when file count exceeds maxConcurrentFileReads", async () => { - // Try to read 6 files when limit is 5 - const result = await executeReadFileToolWithLimit(6, 5) + mockedFsReadFile.mockResolvedValue(Buffer.from("content")) + mockedReadWithSlice.mockReturnValue({ + content: "1 | content", + returnedLines: 1, + totalLines: 1, + wasTruncated: false, + includedRanges: [[1, 1]], + }) - // Verify error result - expect(result).toContain("Error: Too many files requested") - expect(result).toContain("You attempted to read 6 files") - expect(result).toContain("but the concurrent file reads limit is 5") - expect(result).toContain("Please read files in batches of 5 or fewer") + await readFileTool.execute({ path: "test.ts" }, mockTask as any, callbacks) - // Verify error tracking - expect(mockCline.say).toHaveBeenCalledWith("error", expect.stringContaining("Too many files requested")) + expect(mockTask.fileContextTracker.trackFileContext).toHaveBeenCalledWith("test.ts", "read_tool") + }) }) - it("should allow reading files when count equals maxConcurrentFileReads", async () => { - // Try to read exactly 5 files when limit is 5 - const result = await executeReadFileToolWithLimit(5, 5) + describe("error handling", () => { + it("should handle file read errors gracefully", async () => { + const mockTask = createMockTask() + const callbacks = createMockCallbacks() - // Should not contain error - expect(result).not.toContain("Error: Too many files requested") + mockedFsReadFile.mockRejectedValue(new Error("ENOENT: no such file or directory")) - // Should contain file results - expect(typeof result === "string" ? result : JSON.stringify(result)).toContain("file1.txt") - }) + await readFileTool.execute({ path: "nonexistent.ts" }, mockTask as any, callbacks) - it("should allow reading files when count is below maxConcurrentFileReads", async () => { - // Try to read 3 files when limit is 5 - const result = await executeReadFileToolWithLimit(3, 5) + expect(mockTask.say).toHaveBeenCalledWith("error", expect.stringContaining("Error reading file")) + expect(mockTask.didToolFailInCurrentTurn).toBe(true) + }) - // Should not contain error - expect(result).not.toContain("Error: Too many files requested") + it("should handle stat errors gracefully", async () => { + const mockTask = createMockTask() + const callbacks = createMockCallbacks() - // Should contain file results - expect(typeof result === "string" ? result : JSON.stringify(result)).toContain("file1.txt") - }) + mockedFsStat.mockRejectedValue(new Error("Permission denied")) - it("should respect custom maxConcurrentFileReads value of 1", async () => { - // Try to read 2 files when limit is 1 - const result = await executeReadFileToolWithLimit(2, 1) + await readFileTool.execute({ path: "protected.ts" }, mockTask as any, callbacks) - // Verify error result with limit of 1 - expect(result).toContain("Error: Too many files requested") - expect(result).toContain("You attempted to read 2 files") - expect(result).toContain("but the concurrent file reads limit is 1") + expect(mockTask.say).toHaveBeenCalledWith("error", expect.stringContaining("Error reading file")) + expect(mockTask.didToolFailInCurrentTurn).toBe(true) + }) }) - it("should allow single file read when maxConcurrentFileReads is 1", async () => { - // Try to read 1 file when limit is 1 - const result = await executeReadFileToolWithLimit(1, 1) + describe("getReadFileToolDescription", () => { + it("should return description with path when nativeArgs provided", () => { + const description = readFileTool.getReadFileToolDescription("read_file", { path: "src/app.ts" }) - // Should not contain error - expect(result).not.toContain("Error: Too many files requested") - - // Should contain file result - expect(typeof result === "string" ? result : JSON.stringify(result)).toContain("file1.txt") - }) + expect(description).toBe("[read_file for 'src/app.ts']") + }) - it("should respect higher maxConcurrentFileReads value", async () => { - // Try to read 15 files when limit is 10 - const result = await executeReadFileToolWithLimit(15, 10) + it("should return description with path when params provided", () => { + const description = readFileTool.getReadFileToolDescription("read_file", { path: "src/app.ts" }) - // Verify error result - expect(result).toContain("Error: Too many files requested") - expect(result).toContain("You attempted to read 15 files") - expect(result).toContain("but the concurrent file reads limit is 10") - }) - - it("should use default value of 5 when maxConcurrentFileReads is not set", async () => { - // Setup provider state without maxConcurrentFileReads - mockProvider.getState.mockResolvedValue({ - maxReadFileLine: -1, - maxImageFileSize: 20, - maxTotalImageSize: 20, + expect(description).toBe("[read_file for 'src/app.ts']") }) - const toolUse: ReadFileToolUse = { - type: "tool_use", - name: "read_file", - params: {}, - partial: false, - nativeArgs: { - files: Array.from({ length: 6 }, (_, i) => ({ path: `file${i + 1}.txt`, lineRanges: [] })), - }, - } - - mockReadFileWithTokenBudget.mockResolvedValue({ - content: "test content", - tokenCount: 10, - lineCount: 1, - complete: true, - }) + it("should return description indicating missing path", () => { + const description = readFileTool.getReadFileToolDescription("read_file", {}) - await readFileTool.handle(mockCline, toolUse, { - askApproval: mockCline.ask, - handleError: vi.fn(), - pushToolResult: (result: ToolResponse) => { - toolResult = result - }, + expect(description).toBe("[read_file with missing path]") }) - - // Should use default limit of 5 and reject 6 files - expect(toolResult).toContain("Error: Too many files requested") - expect(toolResult).toContain("but the concurrent file reads limit is 5") }) }) diff --git a/src/core/tools/helpers/__tests__/truncateDefinitions.spec.ts b/src/core/tools/helpers/__tests__/truncateDefinitions.spec.ts deleted file mode 100644 index a221b574055..00000000000 --- a/src/core/tools/helpers/__tests__/truncateDefinitions.spec.ts +++ /dev/null @@ -1,160 +0,0 @@ -import { describe, it, expect } from "vitest" -import { truncateDefinitionsToLineLimit } from "../truncateDefinitions" - -describe("truncateDefinitionsToLineLimit", () => { - it("should not truncate when maxReadFileLine is -1 (no limit)", () => { - const definitions = `# test.ts -10--20 | function foo() { -30--40 | function bar() { -50--60 | function baz() {` - - const result = truncateDefinitionsToLineLimit(definitions, -1) - expect(result).toBe(definitions) - }) - - it("should not truncate when maxReadFileLine is 0 (definitions only mode)", () => { - const definitions = `# test.ts -10--20 | function foo() { -30--40 | function bar() { -50--60 | function baz() {` - - const result = truncateDefinitionsToLineLimit(definitions, 0) - expect(result).toBe(definitions) - }) - - it("should truncate definitions beyond the line limit", () => { - const definitions = `# test.ts -10--20 | function foo() { -30--40 | function bar() { -50--60 | function baz() {` - - const result = truncateDefinitionsToLineLimit(definitions, 25) - const expected = `# test.ts -10--20 | function foo() {` - - expect(result).toBe(expected) - }) - - it("should include definitions that start within limit even if they end beyond it", () => { - const definitions = `# test.ts -10--50 | function foo() { -60--80 | function bar() {` - - const result = truncateDefinitionsToLineLimit(definitions, 30) - const expected = `# test.ts -10--50 | function foo() {` - - expect(result).toBe(expected) - }) - - it("should handle single-line definitions", () => { - const definitions = `# test.ts -10 | const foo = 1 -20 | const bar = 2 -30 | const baz = 3` - - const result = truncateDefinitionsToLineLimit(definitions, 25) - const expected = `# test.ts -10 | const foo = 1 -20 | const bar = 2` - - expect(result).toBe(expected) - }) - - it("should preserve header line when all definitions are beyond limit", () => { - const definitions = `# test.ts -100--200 | function foo() {` - - const result = truncateDefinitionsToLineLimit(definitions, 50) - const expected = `# test.ts` - - expect(result).toBe(expected) - }) - - it("should handle empty definitions", () => { - const definitions = `# test.ts` - - const result = truncateDefinitionsToLineLimit(definitions, 50) - expect(result).toBe(definitions) - }) - - it("should handle definitions without header", () => { - const definitions = `10--20 | function foo() { -30--40 | function bar() {` - - const result = truncateDefinitionsToLineLimit(definitions, 25) - const expected = `10--20 | function foo() {` - - expect(result).toBe(expected) - }) - - it("should not preserve empty lines (only definition lines)", () => { - const definitions = `# test.ts -10--20 | function foo() { - -30--40 | function bar() {` - - const result = truncateDefinitionsToLineLimit(definitions, 25) - const expected = `# test.ts -10--20 | function foo() {` - - expect(result).toBe(expected) - }) - - it("should handle mixed single and range definitions", () => { - const definitions = `# test.ts -5 | const x = 1 -10--20 | function foo() { -25 | const y = 2 -30--40 | function bar() {` - - const result = truncateDefinitionsToLineLimit(definitions, 26) - const expected = `# test.ts -5 | const x = 1 -10--20 | function foo() { -25 | const y = 2` - - expect(result).toBe(expected) - }) - - it("should handle definitions at exactly the limit", () => { - const definitions = `# test.ts -10--20 | function foo() { -30--40 | function bar() { -50--60 | function baz() {` - - const result = truncateDefinitionsToLineLimit(definitions, 30) - const expected = `# test.ts -10--20 | function foo() { -30--40 | function bar() {` - - expect(result).toBe(expected) - }) - - it("should handle definitions with leading whitespace", () => { - const definitions = `# test.ts - 10--20 | function foo() { - 30--40 | function bar() { - 50--60 | function baz() {` - - const result = truncateDefinitionsToLineLimit(definitions, 25) - const expected = `# test.ts - 10--20 | function foo() {` - - expect(result).toBe(expected) - }) - - it("should handle definitions with mixed whitespace patterns", () => { - const definitions = `# test.ts -10--20 | function foo() { - 30--40 | function bar() { - 50--60 | function baz() {` - - const result = truncateDefinitionsToLineLimit(definitions, 35) - const expected = `# test.ts -10--20 | function foo() { - 30--40 | function bar() {` - - expect(result).toBe(expected) - }) -}) diff --git a/src/core/tools/helpers/fileTokenBudget.ts b/src/core/tools/helpers/fileTokenBudget.ts deleted file mode 100644 index 4023802680f..00000000000 --- a/src/core/tools/helpers/fileTokenBudget.ts +++ /dev/null @@ -1,9 +0,0 @@ -// Re-export the new incremental token-based file reader -export { readFileWithTokenBudget } from "../../../integrations/misc/read-file-with-budget" -export type { ReadWithBudgetResult, ReadWithBudgetOptions } from "../../../integrations/misc/read-file-with-budget" - -/** - * Percentage of available context to reserve for file reading. - * The remaining percentage is reserved for the model's response and overhead. - */ -export const FILE_READ_BUDGET_PERCENT = 0.6 // 60% for file, 40% for response diff --git a/src/core/tools/helpers/truncateDefinitions.ts b/src/core/tools/helpers/truncateDefinitions.ts deleted file mode 100644 index 7c193ef52a5..00000000000 --- a/src/core/tools/helpers/truncateDefinitions.ts +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Truncate code definitions to only include those within the line limit - * @param definitions - The full definitions string from parseSourceCodeDefinitionsForFile - * @param maxReadFileLine - Maximum line number to include (-1 for no limit, 0 for definitions only) - * @returns Truncated definitions string - */ -export function truncateDefinitionsToLineLimit(definitions: string, maxReadFileLine: number): string { - // If no limit or definitions-only mode (0), return as-is - if (maxReadFileLine <= 0) { - return definitions - } - - const lines = definitions.split("\n") - const result: string[] = [] - let startIndex = 0 - - // Keep the header line (e.g., "# filename.ts") - if (lines.length > 0 && lines[0].startsWith("#")) { - result.push(lines[0]) - startIndex = 1 - } - - // Process definition lines - for (let i = startIndex; i < lines.length; i++) { - const line = lines[i] - - // Match definition format: "startLine--endLine | content" or "lineNumber | content" - // Allow optional leading whitespace to handle indented output or CRLF artifacts - const rangeMatch = line.match(/^\s*(\d+)(?:--(\d+))?\s*\|/) - - if (rangeMatch) { - const startLine = parseInt(rangeMatch[1], 10) - - // Only include definitions that start within the truncated range - if (startLine <= maxReadFileLine) { - result.push(line) - } - } - // Note: We don't preserve empty lines or other non-definition content - // as they're not part of the actual code definitions - } - - return result.join("\n") -} diff --git a/src/core/webview/ClineProvider.ts b/src/core/webview/ClineProvider.ts index e722ce37f85..8de7cf35e84 100644 --- a/src/core/webview/ClineProvider.ts +++ b/src/core/webview/ClineProvider.ts @@ -2041,7 +2041,6 @@ export class ClineProvider showRooIgnoredFiles, enableSubfolderRules, language, - maxReadFileLine, maxImageFileSize, maxTotalImageSize, historyPreviewCollapsed, @@ -2053,7 +2052,6 @@ export class ClineProvider publicSharingEnabled, organizationAllowList, organizationSettingsVersion, - maxConcurrentFileReads, customCondensingPrompt, codebaseIndexConfig, codebaseIndexModels, @@ -2183,10 +2181,8 @@ export class ClineProvider enableSubfolderRules: enableSubfolderRules ?? false, language: language ?? formatLanguage(vscode.env.language), renderContext: this.renderContext, - maxReadFileLine: maxReadFileLine ?? -1, maxImageFileSize: maxImageFileSize ?? 5, maxTotalImageSize: maxTotalImageSize ?? 20, - maxConcurrentFileReads: maxConcurrentFileReads ?? 5, settingsImportedAt: this.settingsImportedAt, hasSystemPromptOverride, historyPreviewCollapsed: historyPreviewCollapsed ?? false, @@ -2423,10 +2419,8 @@ export class ClineProvider telemetrySetting: stateValues.telemetrySetting || "unset", showRooIgnoredFiles: stateValues.showRooIgnoredFiles ?? false, enableSubfolderRules: stateValues.enableSubfolderRules ?? false, - maxReadFileLine: stateValues.maxReadFileLine ?? -1, maxImageFileSize: stateValues.maxImageFileSize ?? 5, maxTotalImageSize: stateValues.maxTotalImageSize ?? 20, - maxConcurrentFileReads: stateValues.maxConcurrentFileReads ?? 5, historyPreviewCollapsed: stateValues.historyPreviewCollapsed ?? false, reasoningBlockCollapsed: stateValues.reasoningBlockCollapsed ?? true, enterBehavior: stateValues.enterBehavior ?? "send", diff --git a/src/core/webview/__tests__/ClineProvider.spec.ts b/src/core/webview/__tests__/ClineProvider.spec.ts index c08ff8cad92..b65b137597c 100644 --- a/src/core/webview/__tests__/ClineProvider.spec.ts +++ b/src/core/webview/__tests__/ClineProvider.spec.ts @@ -568,7 +568,6 @@ describe("ClineProvider", () => { showRooIgnoredFiles: false, enableSubfolderRules: false, renderContext: "sidebar", - maxReadFileLine: 500, maxImageFileSize: 5, maxTotalImageSize: 20, cloudUserInfo: null, diff --git a/src/core/webview/__tests__/generateSystemPrompt.browser-capability.spec.ts b/src/core/webview/__tests__/generateSystemPrompt.browser-capability.spec.ts index 3b521c0f14b..9ad2709b613 100644 --- a/src/core/webview/__tests__/generateSystemPrompt.browser-capability.spec.ts +++ b/src/core/webview/__tests__/generateSystemPrompt.browser-capability.spec.ts @@ -62,8 +62,6 @@ function makeProviderStub() { experiments: {}, browserToolEnabled: true, // critical: enabled in settings language: "en", - maxReadFileLine: -1, - maxConcurrentFileReads: 5, }), } as any } diff --git a/src/core/webview/generateSystemPrompt.ts b/src/core/webview/generateSystemPrompt.ts index b6f77d3842c..abfe36f7ace 100644 --- a/src/core/webview/generateSystemPrompt.ts +++ b/src/core/webview/generateSystemPrompt.ts @@ -19,8 +19,6 @@ export const generateSystemPrompt = async (provider: ClineProvider, message: Web experiments, browserToolEnabled, language, - maxReadFileLine, - maxConcurrentFileReads, enableSubfolderRules, } = await provider.getState() @@ -70,9 +68,7 @@ export const generateSystemPrompt = async (provider: ClineProvider, message: Web experiments, language, rooIgnoreInstructions, - maxReadFileLine !== -1, { - maxConcurrentFileReads: maxConcurrentFileReads ?? 5, todoListEnabled: apiConfiguration?.todoListEnabled ?? true, useAgentRules: vscode.workspace.getConfiguration(Package.name).get("useAgentRules") ?? true, enableSubfolderRules: enableSubfolderRules ?? false, diff --git a/src/integrations/misc/__tests__/extract-text-large-files.spec.ts b/src/integrations/misc/__tests__/extract-text-large-files.spec.ts deleted file mode 100644 index c9e2f181f5a..00000000000 --- a/src/integrations/misc/__tests__/extract-text-large-files.spec.ts +++ /dev/null @@ -1,221 +0,0 @@ -// npx vitest run integrations/misc/__tests__/extract-text-large-files.spec.ts - -import * as fs from "fs/promises" - -import { extractTextFromFile } from "../extract-text" -import { countFileLines } from "../line-counter" -import { readLines } from "../read-lines" -import { isBinaryFile } from "isbinaryfile" - -// Mock all dependencies -vi.mock("fs/promises") -vi.mock("../line-counter") -vi.mock("../read-lines") -vi.mock("isbinaryfile") - -describe("extractTextFromFile - Large File Handling", () => { - // Type the mocks - const mockedFs = vi.mocked(fs) - const mockedCountFileLines = vi.mocked(countFileLines) - const mockedReadLines = vi.mocked(readLines) - const mockedIsBinaryFile = vi.mocked(isBinaryFile) - - beforeEach(() => { - vi.clearAllMocks() - // Set default mock behavior - mockedFs.access.mockResolvedValue(undefined) - mockedIsBinaryFile.mockResolvedValue(false) - }) - - it("should truncate files that exceed maxReadFileLine limit", async () => { - const largeFileContent = Array(150) - .fill(null) - .map((_, i) => `Line ${i + 1}: This is a test line with some content`) - .join("\n") - - mockedCountFileLines.mockResolvedValue(150) - mockedReadLines.mockResolvedValue( - Array(100) - .fill(null) - .map((_, i) => `Line ${i + 1}: This is a test line with some content`) - .join("\n"), - ) - - const result = await extractTextFromFile("/test/large-file.ts", 100) - - // Should only include first 100 lines with line numbers - expect(result).toContain(" 1 | Line 1: This is a test line with some content") - expect(result).toContain("100 | Line 100: This is a test line with some content") - expect(result).not.toContain("101 | Line 101: This is a test line with some content") - - // Should include truncation message - expect(result).toContain( - "[File truncated: showing 100 of 150 total lines. The file is too large and may exhaust the context window if read in full.]", - ) - }) - - it("should not truncate files within the maxReadFileLine limit", async () => { - const smallFileContent = Array(50) - .fill(null) - .map((_, i) => `Line ${i + 1}: This is a test line`) - .join("\n") - - mockedCountFileLines.mockResolvedValue(50) - mockedFs.readFile.mockResolvedValue(smallFileContent as any) - - const result = await extractTextFromFile("/test/small-file.ts", 100) - - // Should include all lines with line numbers - expect(result).toContain(" 1 | Line 1: This is a test line") - expect(result).toContain("50 | Line 50: This is a test line") - - // Should not include truncation message - expect(result).not.toContain("[File truncated:") - }) - - it("should handle files with exactly maxReadFileLine lines", async () => { - const exactFileContent = Array(100) - .fill(null) - .map((_, i) => `Line ${i + 1}`) - .join("\n") - - mockedCountFileLines.mockResolvedValue(100) - mockedFs.readFile.mockResolvedValue(exactFileContent as any) - - const result = await extractTextFromFile("/test/exact-file.ts", 100) - - // Should include all lines with line numbers - expect(result).toContain(" 1 | Line 1") - expect(result).toContain("100 | Line 100") - - // Should not include truncation message - expect(result).not.toContain("[File truncated:") - }) - - it("should handle undefined maxReadFileLine by not truncating", async () => { - const largeFileContent = Array(200) - .fill(null) - .map((_, i) => `Line ${i + 1}`) - .join("\n") - - mockedFs.readFile.mockResolvedValue(largeFileContent as any) - - const result = await extractTextFromFile("/test/large-file.ts", undefined) - - // Should include all lines with line numbers when maxReadFileLine is undefined - expect(result).toContain(" 1 | Line 1") - expect(result).toContain("200 | Line 200") - - // Should not include truncation message - expect(result).not.toContain("[File truncated:") - }) - - it("should handle empty files", async () => { - mockedFs.readFile.mockResolvedValue("" as any) - - const result = await extractTextFromFile("/test/empty-file.ts", 100) - - expect(result).toBe("") - expect(result).not.toContain("[File truncated:") - }) - - it("should handle files with only newlines", async () => { - const newlineOnlyContent = "\n\n\n\n\n" - - mockedCountFileLines.mockResolvedValue(6) // 5 newlines = 6 lines - mockedReadLines.mockResolvedValue("\n\n") - - const result = await extractTextFromFile("/test/newline-file.ts", 3) - - // Should truncate at line 3 - expect(result).toContain("[File truncated: showing 3 of 6 total lines") - }) - - it("should handle very large files efficiently", async () => { - // Simulate a 10,000 line file - mockedCountFileLines.mockResolvedValue(10000) - mockedReadLines.mockResolvedValue( - Array(500) - .fill(null) - .map((_, i) => `Line ${i + 1}: Some content here`) - .join("\n"), - ) - - const result = await extractTextFromFile("/test/very-large-file.ts", 500) - - // Should only include first 500 lines with line numbers - expect(result).toContain(" 1 | Line 1: Some content here") - expect(result).toContain("500 | Line 500: Some content here") - expect(result).not.toContain("501 | Line 501: Some content here") - - // Should show truncation message - expect(result).toContain("[File truncated: showing 500 of 10000 total lines") - }) - - it("should handle maxReadFileLine of 0 by throwing an error", async () => { - const fileContent = "Line 1\nLine 2\nLine 3" - - mockedFs.readFile.mockResolvedValue(fileContent as any) - - // maxReadFileLine of 0 should throw an error - await expect(extractTextFromFile("/test/file.ts", 0)).rejects.toThrow( - "Invalid maxReadFileLine: 0. Must be a positive integer or -1 for unlimited.", - ) - }) - - it("should handle negative maxReadFileLine by treating as undefined", async () => { - const fileContent = "Line 1\nLine 2\nLine 3" - - mockedFs.readFile.mockResolvedValue(fileContent as any) - - const result = await extractTextFromFile("/test/file.ts", -1) - - // Should include all content with line numbers when negative - expect(result).toContain("1 | Line 1") - expect(result).toContain("2 | Line 2") - expect(result).toContain("3 | Line 3") - expect(result).not.toContain("[File truncated:") - }) - - it("should preserve file content structure when truncating", async () => { - const structuredContent = [ - "function example() {", - " const x = 1;", - " const y = 2;", - " return x + y;", - "}", - "", - "// More code below", - ].join("\n") - - mockedCountFileLines.mockResolvedValue(7) - mockedReadLines.mockResolvedValue(["function example() {", " const x = 1;", " const y = 2;"].join("\n")) - - const result = await extractTextFromFile("/test/structured.ts", 3) - - // Should preserve the first 3 lines with line numbers - expect(result).toContain("1 | function example() {") - expect(result).toContain("2 | const x = 1;") - expect(result).toContain("3 | const y = 2;") - expect(result).not.toContain("4 | return x + y;") - - // Should include truncation info - expect(result).toContain("[File truncated: showing 3 of 7 total lines") - }) - - it("should handle binary files by throwing an error", async () => { - mockedIsBinaryFile.mockResolvedValue(true) - - await expect(extractTextFromFile("/test/binary.bin", 100)).rejects.toThrow( - "Cannot read text for file type: .bin", - ) - }) - - it("should handle file not found errors", async () => { - mockedFs.access.mockRejectedValue(new Error("ENOENT")) - - await expect(extractTextFromFile("/test/nonexistent.ts", 100)).rejects.toThrow( - "File not found: /test/nonexistent.ts", - ) - }) -}) diff --git a/src/integrations/misc/__tests__/indentation-reader.spec.ts b/src/integrations/misc/__tests__/indentation-reader.spec.ts new file mode 100644 index 00000000000..d46cb542775 --- /dev/null +++ b/src/integrations/misc/__tests__/indentation-reader.spec.ts @@ -0,0 +1,639 @@ +import { describe, it, expect } from "vitest" +import { + parseLines, + formatWithLineNumbers, + readWithIndentation, + readWithSlice, + computeEffectiveIndents, + type LineRecord, + type IndentationReadResult, +} from "../indentation-reader" + +// ─── Test Fixtures ──────────────────────────────────────────────────────────── + +const PYTHON_CODE = `#!/usr/bin/env python3 +"""Module docstring.""" +import os +import sys +from typing import List + +class Calculator: + """A simple calculator class.""" + + def __init__(self, value: int = 0): + self.value = value + + def add(self, n: int) -> int: + """Add a number.""" + self.value += n + return self.value + + def subtract(self, n: int) -> int: + """Subtract a number.""" + self.value -= n + return self.value + + def reset(self): + """Reset to zero.""" + self.value = 0 + +def main(): + calc = Calculator() + calc.add(5) + print(calc.value) + +if __name__ == "__main__": + main() +` + +const TYPESCRIPT_CODE = `import { something } from "./module" +import type { SomeType } from "./types" + +// Constants +const MAX_VALUE = 100 + +interface Config { + name: string + value: number +} + +class Handler { + private config: Config + + constructor(config: Config) { + this.config = config + } + + process(input: string): string { + // Process the input + const result = input.toUpperCase() + if (result.length > MAX_VALUE) { + return result.slice(0, MAX_VALUE) + } + return result + } + + validate(data: unknown): boolean { + if (typeof data !== "string") { + return false + } + return data.length > 0 + } +} + +export function createHandler(config: Config): Handler { + return new Handler(config) +} +` + +const SIMPLE_CODE = `function outer() { + function inner() { + console.log("hello") + } + inner() +} +` + +const CODE_WITH_BLANKS = `class Example: + def method_one(self): + x = 1 + + y = 2 + + return x + y + + def method_two(self): + return 42 +` + +// ─── parseLines Tests ───────────────────────────────────────────────────────── + +describe("parseLines", () => { + it("should parse lines with correct line numbers", () => { + const content = "line1\nline2\nline3" + const lines = parseLines(content) + + expect(lines).toHaveLength(3) + expect(lines[0].lineNumber).toBe(1) + expect(lines[1].lineNumber).toBe(2) + expect(lines[2].lineNumber).toBe(3) + }) + + it("should calculate indentation levels correctly", () => { + const content = "no indent\n one level\n two levels\n\t\ttab indent" + const lines = parseLines(content) + + expect(lines[0].indentLevel).toBe(0) + expect(lines[1].indentLevel).toBe(1) // 4 spaces = 1 level + expect(lines[2].indentLevel).toBe(2) // 8 spaces = 2 levels + expect(lines[3].indentLevel).toBe(2) // 2 tabs = 2 levels (tabs = 4 spaces each) + }) + + it("should identify blank lines", () => { + const content = "content\n\n \nmore content" + const lines = parseLines(content) + + expect(lines[0].isBlank).toBe(false) + expect(lines[1].isBlank).toBe(true) // empty + expect(lines[2].isBlank).toBe(true) // whitespace only + expect(lines[3].isBlank).toBe(false) + }) + + it("should identify block starts (Python style)", () => { + const content = "def foo():\n pass\nclass Bar:\n pass" + const lines = parseLines(content) + + expect(lines[0].isBlockStart).toBe(true) // def foo(): + expect(lines[1].isBlockStart).toBe(false) // pass + expect(lines[2].isBlockStart).toBe(true) // class Bar: + }) + + it("should identify block starts (C-style)", () => { + const content = "function foo() {\n return\n}\nif (x) {" + const lines = parseLines(content) + + expect(lines[0].isBlockStart).toBe(true) // function foo() { + expect(lines[1].isBlockStart).toBe(false) // return + expect(lines[2].isBlockStart).toBe(false) // } + expect(lines[3].isBlockStart).toBe(true) // if (x) { + }) + + it("should handle empty content", () => { + const lines = parseLines("") + expect(lines).toHaveLength(1) + expect(lines[0].isBlank).toBe(true) + }) +}) + +// ─── computeEffectiveIndents Tests ──────────────────────────────────────────── + +describe("computeEffectiveIndents", () => { + it("should return same indents for non-blank lines", () => { + const content = "line1\n line2\n line3" + const lines = parseLines(content) + const effective = computeEffectiveIndents(lines) + + expect(effective[0]).toBe(0) + expect(effective[1]).toBe(1) + expect(effective[2]).toBe(2) + }) + + it("should inherit previous indent for blank lines", () => { + const content = "line1\n line2\n\n line3" + const lines = parseLines(content) + const effective = computeEffectiveIndents(lines) + + expect(effective[0]).toBe(0) // line1 + expect(effective[1]).toBe(1) // line2 (indent 1) + expect(effective[2]).toBe(1) // blank line inherits from line2 + expect(effective[3]).toBe(1) // line3 + }) + + it("should handle multiple consecutive blank lines", () => { + const content = " start\n\n\n\n end" + const lines = parseLines(content) + const effective = computeEffectiveIndents(lines) + + expect(effective[0]).toBe(1) // start + expect(effective[1]).toBe(1) // blank inherits + expect(effective[2]).toBe(1) // blank inherits + expect(effective[3]).toBe(1) // blank inherits + expect(effective[4]).toBe(1) // end + }) + + it("should handle blank line at start", () => { + const content = "\n content" + const lines = parseLines(content) + const effective = computeEffectiveIndents(lines) + + expect(effective[0]).toBe(0) // blank at start has no previous, defaults to 0 + expect(effective[1]).toBe(1) // content + }) +}) + +// ─── formatWithLineNumbers Tests ────────────────────────────────────────────── + +describe("formatWithLineNumbers", () => { + it("should format lines with line numbers", () => { + const lines: LineRecord[] = [ + { lineNumber: 1, content: "first", indentLevel: 0, isBlank: false, isBlockStart: false }, + { lineNumber: 2, content: "second", indentLevel: 0, isBlank: false, isBlockStart: false }, + ] + + const result = formatWithLineNumbers(lines) + expect(result).toBe("1 | first\n2 | second") + }) + + it("should pad line numbers for alignment", () => { + const lines: LineRecord[] = [ + { lineNumber: 1, content: "a", indentLevel: 0, isBlank: false, isBlockStart: false }, + { lineNumber: 10, content: "b", indentLevel: 0, isBlank: false, isBlockStart: false }, + { lineNumber: 100, content: "c", indentLevel: 0, isBlank: false, isBlockStart: false }, + ] + + const result = formatWithLineNumbers(lines) + expect(result).toBe(" 1 | a\n 10 | b\n100 | c") + }) + + it("should truncate long lines", () => { + const longLine = "x".repeat(600) + const lines: LineRecord[] = [ + { lineNumber: 1, content: longLine, indentLevel: 0, isBlank: false, isBlockStart: false }, + ] + + const result = formatWithLineNumbers(lines, 100) + expect(result.length).toBeLessThan(longLine.length) + expect(result).toContain("...") + }) + + it("should handle empty array", () => { + const result = formatWithLineNumbers([]) + expect(result).toBe("") + }) +}) + +// ─── readWithSlice Tests ────────────────────────────────────────────────────── + +describe("readWithSlice", () => { + it("should read from beginning with default offset", () => { + const result = readWithSlice(SIMPLE_CODE, 0, 10) + + expect(result.totalLines).toBe(7) // 6 lines + empty trailing + expect(result.returnedLines).toBe(7) + expect(result.wasTruncated).toBe(false) + expect(result.content).toContain("1 | function outer()") + }) + + it("should respect offset parameter", () => { + const result = readWithSlice(SIMPLE_CODE, 2, 10) + + expect(result.content).not.toContain("function outer()") + expect(result.content).toContain("console.log") + expect(result.includedRanges[0][0]).toBe(3) // 1-based, offset 2 = line 3 + }) + + it("should respect limit parameter", () => { + const result = readWithSlice(TYPESCRIPT_CODE, 0, 5) + + expect(result.returnedLines).toBe(5) + expect(result.wasTruncated).toBe(true) + }) + + it("should handle offset beyond file end", () => { + const result = readWithSlice(SIMPLE_CODE, 1000, 10) + + expect(result.returnedLines).toBe(0) + expect(result.content).toContain("Error") + }) + + it("should handle negative offset", () => { + const result = readWithSlice(SIMPLE_CODE, -5, 10) + + // Should normalize to 0 + expect(result.includedRanges[0][0]).toBe(1) + }) +}) + +// ─── readWithIndentation Tests ──────────────────────────────────────────────── + +describe("readWithIndentation", () => { + describe("basic block extraction", () => { + it("should extract content around the anchor line", () => { + const result = readWithIndentation(PYTHON_CODE, { + anchorLine: 15, // Inside add() method + maxLevels: 0, // unlimited + includeHeader: false, + includeSiblings: false, + }) + + expect(result.content).toContain("def add") + expect(result.content).toContain("self.value += n") + expect(result.content).toContain("return self.value") + }) + + it("should handle anchor at first line", () => { + const result = readWithIndentation(SIMPLE_CODE, { + anchorLine: 1, + maxLevels: 0, + includeHeader: false, + }) + + expect(result.returnedLines).toBeGreaterThan(0) + expect(result.content).toContain("function outer()") + }) + + it("should handle anchor at last line", () => { + const lines = PYTHON_CODE.trim().split("\n").length + const result = readWithIndentation(PYTHON_CODE, { + anchorLine: lines, + maxLevels: 0, + includeHeader: false, + }) + + expect(result.returnedLines).toBeGreaterThan(0) + }) + }) + + describe("max_levels behavior", () => { + it("should include all content when maxLevels=0 (unlimited)", () => { + const result = readWithIndentation(SIMPLE_CODE, { + anchorLine: 3, // Inside inner() + maxLevels: 0, + includeHeader: false, + includeSiblings: false, + }) + + // With unlimited levels, should get the whole file + expect(result.content).toContain("function outer()") + expect(result.content).toContain("function inner()") + expect(result.content).toContain("console.log") + }) + + it("should limit expansion when maxLevels > 0", () => { + const result = readWithIndentation(SIMPLE_CODE, { + anchorLine: 3, // Inside inner() + maxLevels: 1, + includeHeader: false, + includeSiblings: false, + }) + + // With 1 level, should include inner() context but may not reach outer() + expect(result.content).toContain("console.log") + }) + + it("should handle deeply nested code with unlimited levels", () => { + const result = readWithIndentation(PYTHON_CODE, { + anchorLine: 15, // Inside add() method body + maxLevels: 0, // unlimited + includeHeader: false, + includeSiblings: false, + }) + + // Should expand to include class context + expect(result.content).toContain("class Calculator") + }) + }) + + describe("sibling blocks", () => { + it("should exclude siblings when includeSiblings is false", () => { + const result = readWithIndentation(PYTHON_CODE, { + anchorLine: 15, // Inside add() method + maxLevels: 1, + includeSiblings: false, + includeHeader: false, + }) + + // Should focus on add() but not include subtract() or other siblings + expect(result.content).toContain("def add") + }) + + it("should include siblings when includeSiblings is true", () => { + const result = readWithIndentation(PYTHON_CODE, { + anchorLine: 15, // Inside add() method + maxLevels: 1, + includeSiblings: true, + includeHeader: false, + }) + + // Should include sibling methods + expect(result.content).toContain("def add") + // May include other siblings depending on limit + }) + }) + + describe("file header (includeHeader option)", () => { + it("should allow comment lines at min indent when includeHeader is true", () => { + // The Codex algorithm's includeHeader option allows comment lines at the + // minimum indent level to be included during upward expansion. + // This is different from prepending the file's import header. + const result = readWithIndentation(PYTHON_CODE, { + anchorLine: 15, + maxLevels: 0, // unlimited - will expand to indent 0 + includeHeader: true, + includeSiblings: false, + }) + + // With unlimited levels, bidirectional expansion will include content + // at indent level 0. includeHeader allows comment lines to be included. + expect(result.returnedLines).toBeGreaterThan(0) + expect(result.content).toContain("def add") + }) + + it("should expand to top-level content with maxLevels=0", () => { + const result = readWithIndentation(PYTHON_CODE, { + anchorLine: 15, + maxLevels: 0, // unlimited + includeHeader: false, + includeSiblings: false, + }) + + // With unlimited levels, expansion goes to indent 0 + // which includes the class definition + expect(result.content).toContain("class Calculator") + }) + + it("should include class content when anchored inside a method", () => { + const result = readWithIndentation(TYPESCRIPT_CODE, { + anchorLine: 20, // Inside Handler class + maxLevels: 0, + includeHeader: true, + includeSiblings: false, + }) + + // Should include class context + expect(result.content).toContain("class Handler") + }) + }) + + describe("line limit and max_lines", () => { + it("should truncate output when exceeding limit", () => { + const result = readWithIndentation(TYPESCRIPT_CODE, { + anchorLine: 15, + maxLevels: 0, + includeHeader: true, + includeSiblings: true, + limit: 10, + }) + + expect(result.returnedLines).toBeLessThanOrEqual(10) + expect(result.wasTruncated).toBe(true) + }) + + it("should not truncate when under limit", () => { + const result = readWithIndentation(SIMPLE_CODE, { + anchorLine: 3, + maxLevels: 1, + includeHeader: false, + limit: 100, + }) + + expect(result.wasTruncated).toBe(false) + }) + + it("should respect maxLines as separate hard cap", () => { + const result = readWithIndentation(TYPESCRIPT_CODE, { + anchorLine: 20, + maxLevels: 0, + includeHeader: true, + includeSiblings: true, + limit: 100, + maxLines: 5, // Hard cap at 5 + }) + + expect(result.returnedLines).toBeLessThanOrEqual(5) + }) + + it("should use min of limit and maxLines", () => { + const result = readWithIndentation(TYPESCRIPT_CODE, { + anchorLine: 20, + maxLevels: 0, + includeHeader: true, + includeSiblings: true, + limit: 3, // More restrictive than maxLines + maxLines: 10, + }) + + expect(result.returnedLines).toBeLessThanOrEqual(3) + }) + }) + + describe("blank line handling", () => { + it("should treat blank lines with inherited indentation", () => { + const result = readWithIndentation(CODE_WITH_BLANKS, { + anchorLine: 4, // blank line inside method_one + maxLevels: 1, + includeHeader: false, + includeSiblings: false, + }) + + // Blank line should inherit previous indent and be included in expansion + expect(result.returnedLines).toBeGreaterThan(0) + }) + + it("should trim empty lines from edges of result", () => { + const result = readWithIndentation(CODE_WITH_BLANKS, { + anchorLine: 3, // x = 1 + maxLevels: 1, + includeHeader: false, + includeSiblings: false, + }) + + // Check that result doesn't start or end with blank lines + const lines = result.content.split("\n") + if (lines.length > 0) { + const firstLine = lines[0] + const lastLine = lines[lines.length - 1] + // Lines should have content after the line number prefix + expect(firstLine).toMatch(/\d+\s*\|/) + expect(lastLine).toMatch(/\d+\s*\|/) + } + }) + }) + + describe("error handling", () => { + it("should handle invalid anchor line (too low)", () => { + const result = readWithIndentation(PYTHON_CODE, { + anchorLine: 0, + maxLevels: 1, + }) + + expect(result.content).toContain("Error") + expect(result.returnedLines).toBe(0) + }) + + it("should handle invalid anchor line (too high)", () => { + const result = readWithIndentation(PYTHON_CODE, { + anchorLine: 9999, + maxLevels: 1, + }) + + expect(result.content).toContain("Error") + expect(result.returnedLines).toBe(0) + }) + }) + + describe("bidirectional expansion", () => { + it("should expand both up and down from anchor", () => { + const result = readWithIndentation(SIMPLE_CODE, { + anchorLine: 3, // console.log("hello") - in the middle + maxLevels: 0, + includeHeader: false, + includeSiblings: false, + limit: 10, + }) + + // Should include lines both before and after anchor + expect(result.content).toContain("function inner()") + expect(result.content).toContain("console.log") + }) + + it("should return single line when limit is 1", () => { + const result = readWithIndentation(SIMPLE_CODE, { + anchorLine: 3, + maxLevels: 0, + includeHeader: false, + includeSiblings: false, + limit: 1, + }) + + expect(result.returnedLines).toBe(1) + expect(result.content).toContain("console.log") + }) + + it("should stop expansion when hitting lower indent", () => { + const result = readWithIndentation(PYTHON_CODE, { + anchorLine: 15, // Inside add() method body (return self.value) + maxLevels: 2, // Only go up 2 levels from anchor indent + includeHeader: false, + includeSiblings: false, + }) + + // Should include method but respect maxLevels + expect(result.content).toContain("def add") + }) + }) + + describe("real-world scenarios", () => { + it("should extract a function with its context", () => { + const result = readWithIndentation(TYPESCRIPT_CODE, { + anchorLine: 37, // Inside createHandler function body (return statement) + maxLevels: 0, + includeHeader: true, + includeSiblings: false, + }) + + expect(result.content).toContain("export function createHandler") + expect(result.content).toContain("return new Handler") + }) + + it("should extract a class method with class context", () => { + const result = readWithIndentation(TYPESCRIPT_CODE, { + anchorLine: 19, // Inside process() method + maxLevels: 1, + includeHeader: false, + includeSiblings: false, + }) + + expect(result.content).toContain("process(input: string)") + }) + }) + + describe("includedRanges", () => { + it("should return correct contiguous range", () => { + const result = readWithIndentation(SIMPLE_CODE, { + anchorLine: 3, + maxLevels: 0, + includeHeader: false, + includeSiblings: false, + limit: 10, + }) + + expect(result.includedRanges.length).toBeGreaterThan(0) + // Each range should be [start, end] with start <= end + for (const [start, end] of result.includedRanges) { + expect(start).toBeLessThanOrEqual(end) + expect(start).toBeGreaterThan(0) + } + }) + }) +}) diff --git a/src/integrations/misc/__tests__/read-file-tool.spec.ts b/src/integrations/misc/__tests__/read-file-tool.spec.ts deleted file mode 100644 index fabc5bc8299..00000000000 --- a/src/integrations/misc/__tests__/read-file-tool.spec.ts +++ /dev/null @@ -1,147 +0,0 @@ -// npx vitest run integrations/misc/__tests__/read-file-tool.spec.ts - -import type { Mock } from "vitest" -import * as path from "path" -import { countFileLines } from "../line-counter" -import { readLines } from "../read-lines" -import { extractTextFromFile, addLineNumbers } from "../extract-text" - -// Mock the required functions -vitest.mock("../line-counter") -vitest.mock("../read-lines") -vitest.mock("../extract-text") - -describe("read_file tool with maxReadFileLine setting", () => { - // Mock original implementation first to use in tests - let originalCountFileLines: any - let originalReadLines: any - let originalExtractTextFromFile: any - let originalAddLineNumbers: any - - beforeEach(async () => { - // Import actual implementations - originalCountFileLines = ((await vitest.importActual("../line-counter")) as any).countFileLines - originalReadLines = ((await vitest.importActual("../read-lines")) as any).readLines - originalExtractTextFromFile = ((await vitest.importActual("../extract-text")) as any).extractTextFromFile - originalAddLineNumbers = ((await vitest.importActual("../extract-text")) as any).addLineNumbers - - vitest.resetAllMocks() - // Reset mocks to simulate original behavior - ;(countFileLines as Mock).mockImplementation(originalCountFileLines) - ;(readLines as Mock).mockImplementation(originalReadLines) - ;(extractTextFromFile as Mock).mockImplementation(originalExtractTextFromFile) - ;(addLineNumbers as Mock).mockImplementation(originalAddLineNumbers) - }) - - // Test for the case when file size is smaller than maxReadFileLine - it("should read entire file when line count is less than maxReadFileLine", async () => { - // Mock necessary functions - ;(countFileLines as Mock).mockResolvedValue(100) - ;(extractTextFromFile as Mock).mockResolvedValue("Small file content") - - // Create mock implementation that would simulate the behavior - // Note: We're not testing the Cline class directly as it would be too complex - // We're testing the logic flow that would happen in the read_file implementation - - const filePath = path.resolve("/test", "smallFile.txt") - const maxReadFileLine = 500 - - // Check line count - const lineCount = await countFileLines(filePath) - expect(lineCount).toBeLessThan(maxReadFileLine) - - // Should use extractTextFromFile for small files - if (lineCount < maxReadFileLine) { - await extractTextFromFile(filePath) - } - - expect(extractTextFromFile).toHaveBeenCalledWith(filePath) - expect(readLines).not.toHaveBeenCalled() - }) - - // Test for the case when file size is larger than maxReadFileLine - it("should truncate file when line count exceeds maxReadFileLine", async () => { - // Mock necessary functions - ;(countFileLines as Mock).mockResolvedValue(5000) - ;(readLines as Mock).mockResolvedValue("First 500 lines of large file") - ;(addLineNumbers as Mock).mockReturnValue("1 | First line\n2 | Second line\n...") - - const filePath = path.resolve("/test", "largeFile.txt") - const maxReadFileLine = 500 - - // Check line count - const lineCount = await countFileLines(filePath) - expect(lineCount).toBeGreaterThan(maxReadFileLine) - - // Should use readLines for large files - if (lineCount > maxReadFileLine) { - const content = await readLines(filePath, maxReadFileLine - 1, 0) - const numberedContent = addLineNumbers(content) - - // Verify the truncation message is shown (simulated) - const truncationMsg = `\n\n[File truncated: showing ${maxReadFileLine} of ${lineCount} total lines]` - const fullResult = numberedContent + truncationMsg - - expect(fullResult).toContain("File truncated") - } - - expect(readLines).toHaveBeenCalledWith(filePath, maxReadFileLine - 1, 0) - expect(addLineNumbers).toHaveBeenCalled() - expect(extractTextFromFile).not.toHaveBeenCalled() - }) - - // Test for the case when the file is a source code file - it("should add source code file type info for large source code files", async () => { - // Mock necessary functions - ;(countFileLines as Mock).mockResolvedValue(5000) - ;(readLines as Mock).mockResolvedValue("First 500 lines of large JavaScript file") - ;(addLineNumbers as Mock).mockReturnValue('1 | const foo = "bar";\n2 | function test() {...') - - const filePath = path.resolve("/test", "largeFile.js") - const maxReadFileLine = 500 - - // Check line count - const lineCount = await countFileLines(filePath) - expect(lineCount).toBeGreaterThan(maxReadFileLine) - - // Check if the file is a source code file - const fileExt = path.extname(filePath).toLowerCase() - const isSourceCode = [ - ".js", - ".ts", - ".jsx", - ".tsx", - ".py", - ".java", - ".c", - ".cpp", - ".cs", - ".go", - ".rb", - ".php", - ".swift", - ".rs", - ].includes(fileExt) - expect(isSourceCode).toBeTruthy() - - // Should use readLines for large files - if (lineCount > maxReadFileLine) { - const content = await readLines(filePath, maxReadFileLine - 1, 0) - const numberedContent = addLineNumbers(content) - - // Verify the truncation message and source code message are shown (simulated) - let truncationMsg = `\n\n[File truncated: showing ${maxReadFileLine} of ${lineCount} total lines]` - if (isSourceCode) { - truncationMsg += - "\n\nThis appears to be a source code file. Consider using list_code_definition_names to understand its structure." - } - const fullResult = numberedContent + truncationMsg - - expect(fullResult).toContain("source code file") - expect(fullResult).toContain("list_code_definition_names") - } - - expect(readLines).toHaveBeenCalledWith(filePath, maxReadFileLine - 1, 0) - expect(addLineNumbers).toHaveBeenCalled() - }) -}) diff --git a/src/integrations/misc/__tests__/read-file-with-budget.spec.ts b/src/integrations/misc/__tests__/read-file-with-budget.spec.ts deleted file mode 100644 index 7a4e99ce694..00000000000 --- a/src/integrations/misc/__tests__/read-file-with-budget.spec.ts +++ /dev/null @@ -1,321 +0,0 @@ -import fs from "fs/promises" -import path from "path" -import os from "os" -import { readFileWithTokenBudget } from "../read-file-with-budget" - -describe("readFileWithTokenBudget", () => { - let tempDir: string - - beforeEach(async () => { - // Create a temporary directory for test files - tempDir = path.join(os.tmpdir(), `read-file-budget-test-${Date.now()}`) - await fs.mkdir(tempDir, { recursive: true }) - }) - - afterEach(async () => { - // Clean up temporary directory - await fs.rm(tempDir, { recursive: true, force: true }) - }) - - describe("Basic functionality", () => { - test("reads entire small file when within budget", async () => { - const filePath = path.join(tempDir, "small.txt") - const content = "Line 1\nLine 2\nLine 3" - await fs.writeFile(filePath, content) - - const result = await readFileWithTokenBudget(filePath, { - budgetTokens: 1000, // Large budget - }) - - expect(result.content).toBe(content) - expect(result.lineCount).toBe(3) - expect(result.complete).toBe(true) - expect(result.tokenCount).toBeGreaterThan(0) - expect(result.tokenCount).toBeLessThan(1000) - }) - - test("returns correct token count", async () => { - const filePath = path.join(tempDir, "token-test.txt") - const content = "This is a test file with some content." - await fs.writeFile(filePath, content) - - const result = await readFileWithTokenBudget(filePath, { - budgetTokens: 1000, - }) - - // Token count should be reasonable (rough estimate: 1 token per 3-4 chars) - expect(result.tokenCount).toBeGreaterThan(5) - expect(result.tokenCount).toBeLessThan(20) - }) - - test("returns complete: true for files within budget", async () => { - const filePath = path.join(tempDir, "within-budget.txt") - const lines = Array.from({ length: 10 }, (_, i) => `Line ${i + 1}`) - await fs.writeFile(filePath, lines.join("\n")) - - const result = await readFileWithTokenBudget(filePath, { - budgetTokens: 1000, - }) - - expect(result.complete).toBe(true) - expect(result.lineCount).toBe(10) - }) - }) - - describe("Truncation behavior", () => { - test("stops reading when token budget reached", async () => { - const filePath = path.join(tempDir, "large.txt") - // Create a file with many lines - const lines = Array.from({ length: 1000 }, (_, i) => `This is line number ${i + 1} with some content`) - await fs.writeFile(filePath, lines.join("\n")) - - const result = await readFileWithTokenBudget(filePath, { - budgetTokens: 50, // Small budget - }) - - expect(result.complete).toBe(false) - expect(result.lineCount).toBeLessThan(1000) - expect(result.lineCount).toBeGreaterThan(0) - expect(result.tokenCount).toBeLessThanOrEqual(50) - }) - - test("returns complete: false when truncated", async () => { - const filePath = path.join(tempDir, "truncated.txt") - const lines = Array.from({ length: 500 }, (_, i) => `Line ${i + 1}`) - await fs.writeFile(filePath, lines.join("\n")) - - const result = await readFileWithTokenBudget(filePath, { - budgetTokens: 20, - }) - - expect(result.complete).toBe(false) - expect(result.tokenCount).toBeLessThanOrEqual(20) - }) - - test("content ends at line boundary (no partial lines)", async () => { - const filePath = path.join(tempDir, "line-boundary.txt") - const lines = Array.from({ length: 100 }, (_, i) => `Line ${i + 1}`) - await fs.writeFile(filePath, lines.join("\n")) - - const result = await readFileWithTokenBudget(filePath, { - budgetTokens: 30, - }) - - // Content should not end mid-line - const contentLines = result.content.split("\n") - expect(contentLines.length).toBe(result.lineCount) - // Last line should be complete (not cut off) - expect(contentLines[contentLines.length - 1]).toMatch(/^Line \d+$/) - }) - - test("works with different chunk sizes", async () => { - const filePath = path.join(tempDir, "chunks.txt") - const lines = Array.from({ length: 1000 }, (_, i) => `Line ${i + 1}`) - await fs.writeFile(filePath, lines.join("\n")) - - // Test with small chunk size - const result1 = await readFileWithTokenBudget(filePath, { - budgetTokens: 50, - chunkLines: 10, - }) - - // Test with large chunk size - const result2 = await readFileWithTokenBudget(filePath, { - budgetTokens: 50, - chunkLines: 500, - }) - - // Both should truncate, but may differ slightly in exact line count - expect(result1.complete).toBe(false) - expect(result2.complete).toBe(false) - expect(result1.tokenCount).toBeLessThanOrEqual(50) - expect(result2.tokenCount).toBeLessThanOrEqual(50) - }) - }) - - describe("Edge cases", () => { - test("handles empty file", async () => { - const filePath = path.join(tempDir, "empty.txt") - await fs.writeFile(filePath, "") - - const result = await readFileWithTokenBudget(filePath, { - budgetTokens: 100, - }) - - expect(result.content).toBe("") - expect(result.lineCount).toBe(0) - expect(result.tokenCount).toBe(0) - expect(result.complete).toBe(true) - }) - - test("handles single line file", async () => { - const filePath = path.join(tempDir, "single-line.txt") - await fs.writeFile(filePath, "Single line content") - - const result = await readFileWithTokenBudget(filePath, { - budgetTokens: 100, - }) - - expect(result.content).toBe("Single line content") - expect(result.lineCount).toBe(1) - expect(result.complete).toBe(true) - }) - - test("handles budget of 0 tokens", async () => { - const filePath = path.join(tempDir, "zero-budget.txt") - await fs.writeFile(filePath, "Line 1\nLine 2\nLine 3") - - const result = await readFileWithTokenBudget(filePath, { - budgetTokens: 0, - }) - - expect(result.content).toBe("") - expect(result.lineCount).toBe(0) - expect(result.tokenCount).toBe(0) - expect(result.complete).toBe(false) - }) - - test("handles very small budget (fewer tokens than first line)", async () => { - const filePath = path.join(tempDir, "tiny-budget.txt") - const longLine = "This is a very long line with lots of content that will exceed a tiny token budget" - await fs.writeFile(filePath, `${longLine}\nLine 2\nLine 3`) - - const result = await readFileWithTokenBudget(filePath, { - budgetTokens: 2, // Very small budget - }) - - // Should return empty since first line exceeds budget - expect(result.content).toBe("") - expect(result.lineCount).toBe(0) - expect(result.complete).toBe(false) - }) - - test("throws error for non-existent file", async () => { - const filePath = path.join(tempDir, "does-not-exist.txt") - - await expect( - readFileWithTokenBudget(filePath, { - budgetTokens: 100, - }), - ).rejects.toThrow("File not found") - }) - - test("handles file with no trailing newline", async () => { - const filePath = path.join(tempDir, "no-trailing-newline.txt") - await fs.writeFile(filePath, "Line 1\nLine 2\nLine 3") - - const result = await readFileWithTokenBudget(filePath, { - budgetTokens: 1000, - }) - - expect(result.content).toBe("Line 1\nLine 2\nLine 3") - expect(result.lineCount).toBe(3) - expect(result.complete).toBe(true) - }) - - test("handles file with trailing newline", async () => { - const filePath = path.join(tempDir, "trailing-newline.txt") - await fs.writeFile(filePath, "Line 1\nLine 2\nLine 3\n") - - const result = await readFileWithTokenBudget(filePath, { - budgetTokens: 1000, - }) - - expect(result.content).toBe("Line 1\nLine 2\nLine 3") - expect(result.lineCount).toBe(3) - expect(result.complete).toBe(true) - }) - }) - - describe("Token counting accuracy", () => { - test("returned tokenCount matches actual tokens in content", async () => { - const filePath = path.join(tempDir, "accuracy.txt") - const content = "Hello world\nThis is a test\nWith some content" - await fs.writeFile(filePath, content) - - const result = await readFileWithTokenBudget(filePath, { - budgetTokens: 1000, - }) - - // Verify the token count is reasonable - // Rough estimate: 1 token per 3-4 characters - const minExpected = Math.floor(content.length / 5) - const maxExpected = Math.ceil(content.length / 2) - - expect(result.tokenCount).toBeGreaterThanOrEqual(minExpected) - expect(result.tokenCount).toBeLessThanOrEqual(maxExpected) - }) - - test("handles special characters correctly", async () => { - const filePath = path.join(tempDir, "special-chars.txt") - const content = "Special chars: @#$%^&*()\nUnicode: 你好世界\nEmoji: 😀🎉" - await fs.writeFile(filePath, content) - - const result = await readFileWithTokenBudget(filePath, { - budgetTokens: 1000, - }) - - expect(result.content).toBe(content) - expect(result.tokenCount).toBeGreaterThan(0) - expect(result.complete).toBe(true) - }) - - test("handles code content", async () => { - const filePath = path.join(tempDir, "code.ts") - const code = `function hello(name: string): string {\n return \`Hello, \${name}!\`\n}` - await fs.writeFile(filePath, code) - - const result = await readFileWithTokenBudget(filePath, { - budgetTokens: 1000, - }) - - expect(result.content).toBe(code) - expect(result.tokenCount).toBeGreaterThan(0) - expect(result.complete).toBe(true) - }) - }) - - describe("Performance", () => { - test("handles large files efficiently", async () => { - const filePath = path.join(tempDir, "large-file.txt") - // Create a 1MB file - const lines = Array.from({ length: 10000 }, (_, i) => `Line ${i + 1} with some additional content`) - await fs.writeFile(filePath, lines.join("\n")) - - const startTime = Date.now() - - const result = await readFileWithTokenBudget(filePath, { - budgetTokens: 100, - }) - - const endTime = Date.now() - const duration = endTime - startTime - - // Should complete in reasonable time (less than 5 seconds) - expect(duration).toBeLessThan(5000) - expect(result.complete).toBe(false) - expect(result.tokenCount).toBeLessThanOrEqual(100) - }) - - test("early exits when budget is reached", async () => { - const filePath = path.join(tempDir, "early-exit.txt") - // Create a very large file - const lines = Array.from({ length: 50000 }, (_, i) => `Line ${i + 1}`) - await fs.writeFile(filePath, lines.join("\n")) - - const startTime = Date.now() - - const result = await readFileWithTokenBudget(filePath, { - budgetTokens: 50, // Small budget should trigger early exit - }) - - const endTime = Date.now() - const duration = endTime - startTime - - // Should be much faster than reading entire file (less than 2 seconds) - expect(duration).toBeLessThan(2000) - expect(result.complete).toBe(false) - expect(result.lineCount).toBeLessThan(50000) - }) - }) -}) diff --git a/src/integrations/misc/extract-text.ts b/src/integrations/misc/extract-text.ts index bafa7a5bab1..f29fa915d13 100644 --- a/src/integrations/misc/extract-text.ts +++ b/src/integrations/misc/extract-text.ts @@ -5,8 +5,8 @@ import mammoth from "mammoth" import fs from "fs/promises" import { isBinaryFile } from "isbinaryfile" import { extractTextFromXLSX } from "./extract-text-from-xlsx" -import { countFileLines } from "./line-counter" -import { readLines } from "./read-lines" +import { readWithSlice } from "./indentation-reader" +import { DEFAULT_LINE_LIMIT } from "../../core/prompts/tools/native-tools/read_file" async function extractTextFromPDF(filePath: string): Promise { const dataBuffer = await fs.readFile(filePath) @@ -51,26 +51,34 @@ export function getSupportedBinaryFormats(): string[] { } /** - * Extracts text content from a file, with support for various formats including PDF, DOCX, XLSX, and plain text. - * For large text files, can limit the number of lines read to prevent context exhaustion. + * Result of extracting text with metadata about truncation + */ +export interface ExtractTextResult { + /** The extracted content with line numbers */ + content: string + /** Total lines in the file */ + totalLines: number + /** Lines actually returned */ + returnedLines: number + /** Whether output was truncated */ + wasTruncated: boolean + /** Line range shown [start, end] (1-based) */ + linesShown?: [number, number] +} + +/** + * Extracts text content from a file with truncation support. + * Returns structured result with metadata about truncation. * * @param filePath - Path to the file to extract text from - * @param maxReadFileLine - Maximum number of lines to read from text files. - * Use UNLIMITED_LINES (-1) or undefined for no limit. - * Must be a positive integer or UNLIMITED_LINES. - * @returns Promise resolving to the extracted text content with line numbers - * @throws {Error} If file not found, unsupported format, or invalid parameters + * @param limit - Maximum lines to return (default: 2000) + * @returns Promise resolving to extracted text with metadata + * @throws {Error} If file not found or unsupported binary format */ -export async function extractTextFromFile(filePath: string, maxReadFileLine?: number): Promise { - // Validate maxReadFileLine parameter - if (maxReadFileLine !== undefined && maxReadFileLine !== -1) { - if (!Number.isInteger(maxReadFileLine) || maxReadFileLine < 1) { - throw new Error( - `Invalid maxReadFileLine: ${maxReadFileLine}. Must be a positive integer or -1 for unlimited.`, - ) - } - } - +export async function extractTextFromFileWithMetadata( + filePath: string, + limit: number = DEFAULT_LINE_LIMIT, +): Promise { try { await fs.access(filePath) } catch (error) { @@ -82,33 +90,49 @@ export async function extractTextFromFile(filePath: string, maxReadFileLine?: nu // Check if we have a specific extractor for this format const extractor = SUPPORTED_BINARY_FORMATS[fileExtension as keyof typeof SUPPORTED_BINARY_FORMATS] if (extractor) { - return extractor(filePath) + // For binary formats, extract and count lines + const content = await extractor(filePath) + const lines = content.split("\n") + return { + content, + totalLines: lines.length, + returnedLines: lines.length, + wasTruncated: false, + } } // Handle other files const isBinary = await isBinaryFile(filePath).catch(() => false) if (!isBinary) { - // Check if we need to apply line limit - if (maxReadFileLine !== undefined && maxReadFileLine !== -1) { - const totalLines = await countFileLines(filePath) - if (totalLines > maxReadFileLine) { - // Read only up to maxReadFileLine (endLine is 0-based and inclusive) - const content = await readLines(filePath, maxReadFileLine - 1, 0) - const numberedContent = addLineNumbers(content) - return ( - numberedContent + - `\n\n[File truncated: showing ${maxReadFileLine} of ${totalLines} total lines. The file is too large and may exhaust the context window if read in full.]` - ) - } + const rawContent = await fs.readFile(filePath, "utf8") + const result = readWithSlice(rawContent, 0, limit) + + return { + content: result.content, + totalLines: result.totalLines, + returnedLines: result.returnedLines, + wasTruncated: result.wasTruncated, + linesShown: result.includedRanges.length > 0 ? result.includedRanges[0] : undefined, } - // Read the entire file if no limit or file is within limit - return addLineNumbers(await fs.readFile(filePath, "utf8")) } else { throw new Error(`Cannot read text for file type: ${fileExtension}`) } } +/** + * Extracts text content from a file, with support for various formats including PDF, DOCX, XLSX, and plain text. + * Now uses truncation to limit large files to DEFAULT_LINE_LIMIT lines. + * + * @param filePath - Path to the file to extract text from + * @returns Promise resolving to the extracted text content with line numbers + * @throws {Error} If file not found or unsupported binary format + */ +export async function extractTextFromFile(filePath: string): Promise { + const result = await extractTextFromFileWithMetadata(filePath) + return result.content +} + export function addLineNumbers(content: string, startLine: number = 1): string { // If content is empty, return empty string - empty files should not have line numbers // If content is empty but startLine > 1, return "startLine | " because we know the file is not empty diff --git a/src/integrations/misc/indentation-reader.ts b/src/integrations/misc/indentation-reader.ts new file mode 100644 index 00000000000..aecabd5982b --- /dev/null +++ b/src/integrations/misc/indentation-reader.ts @@ -0,0 +1,469 @@ +/** + * Indentation-based semantic code block extraction. + * + * Inspired by Codex's indentation mode, this module extracts meaningful code blocks + * based on indentation hierarchy rather than arbitrary line ranges. + * + * The algorithm uses bidirectional expansion from an anchor line: + * 1. Parse the file to determine indentation level of each line + * 2. Compute effective indents (blank lines inherit previous non-blank line's indent) + * 3. Expand up and down from anchor simultaneously + * 4. Apply sibling exclusion counters to limit scope + * 5. Trim empty lines from edges + * 6. Apply line limit + */ + +import { + DEFAULT_LINE_LIMIT, + DEFAULT_MAX_LEVELS, + MAX_LINE_LENGTH, +} from "../../core/prompts/tools/native-tools/read_file" + +// ─── Types ──────────────────────────────────────────────────────────────────── + +export interface LineRecord { + /** 1-based line number */ + lineNumber: number + /** Original line content */ + content: string + /** Computed indentation level (number of leading whitespace units) */ + indentLevel: number + /** Whether this line is blank (empty or whitespace only) */ + isBlank: boolean + /** Whether this line starts a new block (has content followed by colon, brace, etc.) */ + isBlockStart: boolean +} + +export interface IndentationReadOptions { + /** 1-based anchor line number */ + anchorLine: number + /** Maximum indentation levels to include above anchor (0 = unlimited, default: 0) */ + maxLevels?: number + /** Include sibling blocks at the same indentation level (default: false) */ + includeSiblings?: boolean + /** Include file header content (imports, comments at top) (default: true) */ + includeHeader?: boolean + /** Maximum lines to return from bidirectional expansion (default: 2000) */ + limit?: number + /** Hard cap on lines returned, separate from limit (optional) */ + maxLines?: number +} + +export interface IndentationReadResult { + /** The extracted content with line numbers */ + content: string + /** Line ranges that were included [start, end] tuples (1-based) */ + includedRanges: Array<[number, number]> + /** Total lines in the file */ + totalLines: number + /** Lines actually returned */ + returnedLines: number + /** Whether output was truncated due to limit */ + wasTruncated: boolean +} + +// ─── Constants ──────────────────────────────────────────────────────────────── + +/** Indentation unit size (spaces) */ +const INDENT_SIZE = 4 + +/** Tab width for indent measurement (Codex standard) */ +const TAB_WIDTH = 4 + +/** Patterns that indicate a block start */ +const BLOCK_START_PATTERNS = [ + /:\s*$/, // Python-style (def foo():) + /\{\s*$/, // C-style opening brace + /=>\s*\{?\s*$/, // Arrow functions + /\bthen\s*$/, // Lua/some languages + /\bdo\s*$/, // Ruby, Lua +] + +/** Patterns for file header lines (imports, comments, etc.) */ +const HEADER_PATTERNS = [ + /^import\s/, // ES6 imports + /^from\s.*import/, // Python imports + /^const\s.*=\s*require/, // CommonJS requires + /^#!/, // Shebang + /^\/\*/, // Block comment start + /^\*/, // Block comment continuation + /^\s*\*\//, // Block comment end + /^\/\//, // Line comment + /^#(?!include)/, // Python/shell comment (not C #include) + /^"""/, // Python docstring + /^'''/, // Python docstring + /^use\s/, // Rust use + /^package\s/, // Go/Java package + /^require\s/, // Lua require + /^@/, // Decorators (Python, TypeScript) + /^"use\s/, // "use strict", "use client" +] + +/** Comment prefixes for header detection (Codex standard) */ +const COMMENT_PREFIXES = ["#", "//", "--", "/*", "*", "'''", '"""'] + +// ─── Core Functions ─────────────────────────────────────────────────────────── + +/** + * Parse a file's lines into LineRecord objects with indentation information. + */ +export function parseLines(content: string): LineRecord[] { + const lines = content.split("\n") + return lines.map((line, index) => { + const trimmed = line.trimStart() + const leadingWhitespace = line.length - trimmed.length + + // Calculate indent in spaces (tabs = TAB_WIDTH spaces each) + let indentSpaces = 0 + for (let i = 0; i < leadingWhitespace; i++) { + if (line[i] === "\t") { + indentSpaces += TAB_WIDTH + } else { + indentSpaces += 1 + } + } + // Convert to indent level (number of INDENT_SIZE units) + const indentLevel = Math.floor(indentSpaces / INDENT_SIZE) + + const isBlank = trimmed.length === 0 + const isBlockStart = !isBlank && BLOCK_START_PATTERNS.some((pattern) => pattern.test(line)) + + return { + lineNumber: index + 1, + content: line, + indentLevel, + isBlank, + isBlockStart, + } + }) +} + +/** + * Compute effective indents where blank lines inherit the previous non-blank line's indent. + * This matches the Codex algorithm behavior. + */ +export function computeEffectiveIndents(lines: LineRecord[]): number[] { + const effective: number[] = [] + let previousIndent = 0 + + for (const line of lines) { + if (line.isBlank) { + effective.push(previousIndent) + } else { + previousIndent = line.indentLevel + effective.push(previousIndent) + } + } + return effective +} + +/** + * Check if a line is a comment (for include_header behavior). + */ +function isComment(line: LineRecord): boolean { + const trimmed = line.content.trim() + return COMMENT_PREFIXES.some((prefix) => trimmed.startsWith(prefix)) +} + +/** + * Trim empty lines from the front and back of a line array. + */ +function trimEmptyLines(lines: LineRecord[]): void { + // Trim from front + while (lines.length > 0 && lines[0].isBlank) { + lines.shift() + } + // Trim from back + while (lines.length > 0 && lines[lines.length - 1].isBlank) { + lines.pop() + } +} + +/** + * Find the file header (imports, top-level comments, etc.). + * Returns the end index of the header section. + */ +function findHeaderEnd(lines: LineRecord[]): number { + let lastHeaderIdx = -1 + let inBlockComment = false + + for (let i = 0; i < lines.length; i++) { + const line = lines[i] + const trimmed = line.content.trim() + + // Track block comments + if (trimmed.startsWith("/*")) inBlockComment = true + if (trimmed.endsWith("*/")) { + inBlockComment = false + lastHeaderIdx = i + continue + } + if (inBlockComment) { + lastHeaderIdx = i + continue + } + + // Check if this is a header line + if (line.isBlank) { + // Blank lines are part of header if we haven't seen content yet + if (lastHeaderIdx === i - 1) { + lastHeaderIdx = i + } + continue + } + + const isHeader = HEADER_PATTERNS.some((pattern) => pattern.test(trimmed)) + if (isHeader) { + lastHeaderIdx = i + } else if (line.indentLevel === 0) { + // Hit first non-header top-level content + break + } + } + + return lastHeaderIdx +} + +/** + * Format lines with line numbers, applying truncation to long lines. + */ +export function formatWithLineNumbers(lines: LineRecord[], maxLineLength: number = MAX_LINE_LENGTH): string { + if (lines.length === 0) return "" + const maxLineNumWidth = String(lines[lines.length - 1]?.lineNumber || 1).length + + return lines + .map((line) => { + const lineNum = String(line.lineNumber).padStart(maxLineNumWidth, " ") + let content = line.content + + // Truncate long lines + if (content.length > maxLineLength) { + content = content.substring(0, maxLineLength - 3) + "..." + } + + return `${lineNum} | ${content}` + }) + .join("\n") +} + +/** + * Convert a contiguous array of LineRecords into merged ranges for output. + */ +function computeIncludedRanges(lines: LineRecord[]): Array<[number, number]> { + if (lines.length === 0) return [] + + const ranges: Array<[number, number]> = [] + let rangeStart = lines[0].lineNumber + let rangeEnd = lines[0].lineNumber + + for (let i = 1; i < lines.length; i++) { + const lineNum = lines[i].lineNumber + if (lineNum === rangeEnd + 1) { + // Contiguous + rangeEnd = lineNum + } else { + // Gap - save current range and start new one + ranges.push([rangeStart, rangeEnd]) + rangeStart = lineNum + rangeEnd = lineNum + } + } + // Don't forget the last range + ranges.push([rangeStart, rangeEnd]) + + return ranges +} + +// ─── Main Export ────────────────────────────────────────────────────────────── + +/** + * Read a file using indentation-based semantic extraction (Codex algorithm). + * + * Uses bidirectional expansion from the anchor line with sibling exclusion counters. + * + * @param content - The file content to process + * @param options - Extraction options + * @returns The extracted content with metadata + */ +export function readWithIndentation(content: string, options: IndentationReadOptions): IndentationReadResult { + const { + anchorLine, + maxLevels = DEFAULT_MAX_LEVELS, + includeSiblings = false, + includeHeader = true, + limit = DEFAULT_LINE_LIMIT, + maxLines, + } = options + + const lines = parseLines(content) + const totalLines = lines.length + + // Validate anchor line + if (anchorLine < 1 || anchorLine > totalLines) { + return { + content: `Error: anchor_line ${anchorLine} is out of range (1-${totalLines})`, + includedRanges: [], + totalLines, + returnedLines: 0, + wasTruncated: false, + } + } + + const anchorIdx = anchorLine - 1 // Convert to 0-based + const effectiveIndents = computeEffectiveIndents(lines) + const anchorIndent = effectiveIndents[anchorIdx] + + // Calculate minimum indent threshold + // maxLevels = 0 means unlimited (minIndent = 0) + // maxLevels > 0 means limit to that many levels above anchor + let minIndent: number + if (maxLevels === 0) { + minIndent = 0 + } else { + // Each "level" is INDENT_SIZE spaces worth of indentation + // We subtract maxLevels from the anchor's indent level + minIndent = Math.max(0, anchorIndent - maxLevels) + } + + // Calculate final limit (use maxLines as hard cap if provided) + const guardLimit = maxLines ?? limit + const finalLimit = Math.min(limit, guardLimit, totalLines) + + // Edge case: if limit is 1, just return the anchor line + if (finalLimit === 1) { + const singleLine = [lines[anchorIdx]] + return { + content: formatWithLineNumbers(singleLine), + includedRanges: [[anchorLine, anchorLine]], + totalLines, + returnedLines: 1, + wasTruncated: totalLines > 1, + } + } + + // Bidirectional expansion from anchor (Codex algorithm) + const result: LineRecord[] = [lines[anchorIdx]] + let i = anchorIdx - 1 // Up cursor + let j = anchorIdx + 1 // Down cursor + let iMinCount = 0 // Count of min-indent lines seen going up + let jMinCount = 0 // Count of min-indent lines seen going down + + while (result.length < finalLimit) { + let progressed = false + + // Expand upward + if (i >= 0 && effectiveIndents[i] >= minIndent) { + result.unshift(lines[i]) + progressed = true + + // Handle sibling exclusion at min indent + if (effectiveIndents[i] === minIndent && !includeSiblings) { + const allowHeader = includeHeader && isComment(lines[i]) + const canTake = allowHeader || iMinCount === 0 + + if (canTake) { + iMinCount++ + } else { + // Reject this line - remove it and stop expanding up + result.shift() + progressed = false + i = -1 // Stop expanding up + } + } + + if (i >= 0) i-- + } else if (i >= 0) { + i = -1 // Stop expanding up (hit lower indent) + } + + if (result.length >= finalLimit) break + + // Expand downward + if (j < lines.length && effectiveIndents[j] >= minIndent) { + result.push(lines[j]) + progressed = true + + // Handle sibling exclusion at min indent + if (effectiveIndents[j] === minIndent && !includeSiblings) { + if (jMinCount > 0) { + // Already saw one min-indent block going down, reject this + result.pop() + progressed = false + j = lines.length // Stop expanding down + } + jMinCount++ + } + + if (j < lines.length) j++ + } else if (j < lines.length) { + j = lines.length // Stop expanding down (hit lower indent) + } + + if (!progressed) break + } + + // Trim leading/trailing empty lines + trimEmptyLines(result) + + // Check if we were truncated + const wasTruncated = result.length >= finalLimit || i >= 0 || j < lines.length + + // Format output + const formattedContent = formatWithLineNumbers(result) + + // Compute included ranges + const includedRanges = computeIncludedRanges(result) + + return { + content: formattedContent, + includedRanges, + totalLines, + returnedLines: result.length, + wasTruncated: wasTruncated && result.length < totalLines, + } +} + +/** + * Simple slice mode reading - read lines with offset/limit. + * + * @param content - The file content to process + * @param offset - 0-based line offset to start from (default: 0) + * @param limit - Maximum lines to return (default: 2000) + * @returns The extracted content with metadata + */ +export function readWithSlice( + content: string, + offset: number = 0, + limit: number = DEFAULT_LINE_LIMIT, +): IndentationReadResult { + const lines = parseLines(content) + const totalLines = lines.length + + // Validate offset + if (offset < 0) offset = 0 + if (offset >= totalLines) { + return { + content: `Error: offset ${offset} is beyond file end (${totalLines} lines)`, + includedRanges: [], + totalLines, + returnedLines: 0, + wasTruncated: false, + } + } + + // Slice lines + const endIdx = Math.min(offset + limit, totalLines) + const selectedLines = lines.slice(offset, endIdx) + const wasTruncated = endIdx < totalLines + + // Format output + const formattedContent = formatWithLineNumbers(selectedLines) + + return { + content: formattedContent, + includedRanges: [[offset + 1, endIdx]], // 1-based + totalLines, + returnedLines: selectedLines.length, + wasTruncated, + } +} diff --git a/src/integrations/misc/read-file-with-budget.ts b/src/integrations/misc/read-file-with-budget.ts deleted file mode 100644 index 15aa4f1144f..00000000000 --- a/src/integrations/misc/read-file-with-budget.ts +++ /dev/null @@ -1,182 +0,0 @@ -import { createReadStream } from "fs" -import fs from "fs/promises" -import { createInterface } from "readline" -import { countTokens } from "../../utils/countTokens" -import { Anthropic } from "@anthropic-ai/sdk" - -export interface ReadWithBudgetResult { - /** The content read up to the token budget */ - content: string - /** Actual token count of returned content */ - tokenCount: number - /** Total lines in the returned content */ - lineCount: number - /** Whether the entire file was read (false if truncated) */ - complete: boolean -} - -export interface ReadWithBudgetOptions { - /** Maximum tokens allowed. Required. */ - budgetTokens: number - /** Number of lines to buffer before token counting (default: 256) */ - chunkLines?: number -} - -/** - * Reads a file while incrementally counting tokens, stopping when budget is reached. - * - * Unlike validateFileTokenBudget + extractTextFromFile, this is a single-pass - * operation that returns the actual content up to the token limit. - * - * @param filePath - Path to the file to read - * @param options - Budget and chunking options - * @returns Content read, token count, and completion status - */ -export async function readFileWithTokenBudget( - filePath: string, - options: ReadWithBudgetOptions, -): Promise { - const { budgetTokens, chunkLines = 256 } = options - - // Verify file exists - try { - await fs.access(filePath) - } catch { - throw new Error(`File not found: ${filePath}`) - } - - return new Promise((resolve, reject) => { - let content = "" - let lineCount = 0 - let tokenCount = 0 - let lineBuffer: string[] = [] - let complete = true - let isProcessing = false - let shouldClose = false - - const readStream = createReadStream(filePath) - const rl = createInterface({ - input: readStream, - crlfDelay: Infinity, - }) - - const processBuffer = async (): Promise => { - if (lineBuffer.length === 0) return true - - const bufferText = lineBuffer.join("\n") - const currentBuffer = [...lineBuffer] - lineBuffer = [] - - // Count tokens for this chunk - let chunkTokens: number - try { - const contentBlocks: Anthropic.Messages.ContentBlockParam[] = [{ type: "text", text: bufferText }] - chunkTokens = await countTokens(contentBlocks) - } catch { - // Fallback: conservative estimate (2 chars per token) - chunkTokens = Math.ceil(bufferText.length / 2) - } - - // Check if adding this chunk would exceed budget - if (tokenCount + chunkTokens > budgetTokens) { - // Need to find cutoff within this chunk using binary search - let low = 0 - let high = currentBuffer.length - let bestFit = 0 - let bestTokens = 0 - - while (low < high) { - const mid = Math.floor((low + high + 1) / 2) - const testContent = currentBuffer.slice(0, mid).join("\n") - let testTokens: number - try { - const blocks: Anthropic.Messages.ContentBlockParam[] = [{ type: "text", text: testContent }] - testTokens = await countTokens(blocks) - } catch { - testTokens = Math.ceil(testContent.length / 2) - } - - if (tokenCount + testTokens <= budgetTokens) { - bestFit = mid - bestTokens = testTokens - low = mid - } else { - high = mid - 1 - } - } - - // Add best fit lines - if (bestFit > 0) { - const fitContent = currentBuffer.slice(0, bestFit).join("\n") - content += (content.length > 0 ? "\n" : "") + fitContent - tokenCount += bestTokens - lineCount += bestFit - } - complete = false - return false - } - - // Entire chunk fits - add it all - content += (content.length > 0 ? "\n" : "") + bufferText - tokenCount += chunkTokens - lineCount += currentBuffer.length - return true - } - - rl.on("line", (line) => { - lineBuffer.push(line) - - if (lineBuffer.length >= chunkLines && !isProcessing) { - isProcessing = true - rl.pause() - - processBuffer() - .then((continueReading) => { - isProcessing = false - if (!continueReading) { - shouldClose = true - rl.close() - readStream.destroy() - } else if (!shouldClose) { - rl.resume() - } - }) - .catch((err) => { - isProcessing = false - shouldClose = true - rl.close() - readStream.destroy() - reject(err) - }) - } - }) - - rl.on("close", async () => { - // Wait for any ongoing processing with timeout - const maxWaitTime = 30000 // 30 seconds - const startWait = Date.now() - while (isProcessing) { - if (Date.now() - startWait > maxWaitTime) { - reject(new Error("Timeout waiting for buffer processing to complete")) - return - } - await new Promise((r) => setTimeout(r, 10)) - } - - // Process remaining buffer - if (!shouldClose) { - try { - await processBuffer() - } catch (err) { - reject(err) - return - } - } - - resolve({ content, tokenCount, lineCount, complete }) - }) - - rl.on("error", reject) - readStream.on("error", reject) - }) -} diff --git a/src/shared/tools.ts b/src/shared/tools.ts index 5d7435573c8..4be71aa1bed 100644 --- a/src/shared/tools.ts +++ b/src/shared/tools.ts @@ -5,7 +5,6 @@ import type { ToolProgressStatus, ToolGroup, ToolName, - FileEntry, BrowserActionParams, GenerateImageParams, } from "@roo-code/types" @@ -66,7 +65,7 @@ export const toolParamNames = [ "todos", "prompt", "image", - "files", // Native protocol parameter for read_file + // read_file parameters (native protocol) "operations", // search_and_replace parameter for multiple operations "patch", // apply_patch parameter "file_path", // search_replace and edit_file parameter @@ -75,8 +74,18 @@ export const toolParamNames = [ "expected_replacements", // edit_file parameter for multiple occurrences "artifact_id", // read_command_output parameter "search", // read_command_output parameter for grep-like search - "offset", // read_command_output parameter for pagination - "limit", // read_command_output parameter for max bytes to return + "offset", // read_command_output and read_file parameter + "limit", // read_command_output and read_file parameter + // read_file indentation mode parameters + "indentation", + "anchor_line", + "max_levels", + "include_siblings", + "include_header", + "max_lines", + // read_file legacy format parameter (backward compatibility) + "files", + "line_ranges", ] as const export type ToolParamName = (typeof toolParamNames)[number] @@ -87,7 +96,7 @@ export type ToolParamName = (typeof toolParamNames)[number] */ export type NativeToolArgs = { access_mcp_resource: { server_name: string; uri: string } - read_file: { files: FileEntry[] } + read_file: import("@roo-code/types").ReadFileToolParams read_command_output: { artifact_id: string; search?: string; offset?: number; limit?: number } attempt_completion: { result: string } execute_command: { command: string; cwd?: string } @@ -135,6 +144,11 @@ export interface ToolUse { partial: boolean // nativeArgs is properly typed based on TName if it's in NativeToolArgs, otherwise never nativeArgs?: TName extends keyof NativeToolArgs ? NativeToolArgs[TName] : never + /** + * Flag indicating whether the tool call used a legacy/deprecated format. + * Used for telemetry tracking to monitor migration from old formats. + */ + usedLegacyFormat?: boolean } /** @@ -165,7 +179,23 @@ export interface ExecuteCommandToolUse extends ToolUse<"execute_command"> { export interface ReadFileToolUse extends ToolUse<"read_file"> { name: "read_file" - params: Partial, "args" | "path" | "start_line" | "end_line" | "files">> + params: Partial< + Pick< + Record, + | "args" + | "path" + | "start_line" + | "end_line" + | "mode" + | "offset" + | "limit" + | "indentation" + | "anchor_line" + | "max_levels" + | "include_siblings" + | "include_header" + > + > } export interface WriteToFileToolUse extends ToolUse<"write_to_file"> { diff --git a/src/utils/__tests__/json-schema.spec.ts b/src/utils/__tests__/json-schema.spec.ts index c939095340a..6f2096e626e 100644 --- a/src/utils/__tests__/json-schema.spec.ts +++ b/src/utils/__tests__/json-schema.spec.ts @@ -86,9 +86,9 @@ describe("normalizeToolSchema", () => { type: "object", properties: { path: { type: "string" }, - line_ranges: { + tags: { type: ["array", "null"], - items: { type: "integer" }, + items: { type: "string" }, }, }, }, @@ -104,8 +104,8 @@ describe("normalizeToolSchema", () => { type: "object", properties: { path: { type: "string" }, - line_ranges: { - anyOf: [{ type: "array", items: { type: "integer" } }, { type: "null" }], + tags: { + anyOf: [{ type: "array", items: { type: "string" } }, { type: "null" }], }, }, additionalProperties: false, @@ -123,7 +123,7 @@ describe("normalizeToolSchema", () => { type: "object", properties: { path: { type: "string" }, - line_ranges: { + ranges: { type: ["array", "null"], items: { type: "array", @@ -131,7 +131,7 @@ describe("normalizeToolSchema", () => { }, }, }, - required: ["path", "line_ranges"], + required: ["path", "ranges"], }, }, }, @@ -144,7 +144,7 @@ describe("normalizeToolSchema", () => { const filesItems = properties.files.items as Record const filesItemsProps = filesItems.properties as Record> // Array-specific properties (items) should be moved inside the array variant - expect(filesItemsProps.line_ranges.anyOf).toEqual([ + expect(filesItemsProps.ranges.anyOf).toEqual([ { type: "array", items: { type: "array", items: { type: "integer" } } }, { type: "null" }, ]) @@ -224,60 +224,32 @@ describe("normalizeToolSchema", () => { const input = { type: "object", properties: { - files: { - type: "array", - description: "List of files to read", - items: { - type: "object", - properties: { - path: { - type: "string", - description: "Path to the file", - }, - line_ranges: { - type: ["array", "null"], - description: "Optional line ranges", - items: { - type: "array", - items: { type: "integer" }, - minItems: 2, - maxItems: 2, - }, - }, + path: { + type: "string", + description: "Path to the file", + }, + indentation: { + type: ["object", "null"], + properties: { + anchor_line: { + type: ["integer", "null"], }, - required: ["path", "line_ranges"], - additionalProperties: false, }, - minItems: 1, }, }, - required: ["files"], + required: ["path"], additionalProperties: false, } const result = normalizeToolSchema(input) - // Verify the line_ranges was transformed with items inside the array variant - const files = (result.properties as Record).files as Record - const items = files.items as Record - const props = items.properties as Record> - // Array-specific properties (items, minItems, maxItems) should be moved inside the array variant - expect(props.line_ranges.anyOf).toEqual([ - { - type: "array", - items: { - type: "array", - items: { type: "integer" }, - minItems: 2, - maxItems: 2, - }, - }, - { type: "null" }, - ]) - // items should NOT be at root level anymore - expect(props.line_ranges.items).toBeUndefined() - // Other properties are preserved at root level - expect(props.line_ranges.description).toBe("Optional line ranges") + // Verify nested nullable objects are transformed correctly + const props = result.properties as Record> + expect(props.indentation.anyOf).toEqual([{ type: "object" }, { type: "null" }]) + expect(props.indentation.additionalProperties).toBe(false) + expect((props.indentation.properties as Record).anchor_line).toEqual({ + anyOf: [{ type: "integer" }, { type: "null" }], + }) }) describe("format field handling", () => { diff --git a/webview-ui/src/components/chat/ChatRow.tsx b/webview-ui/src/components/chat/ChatRow.tsx index 25bcd61ee3f..74639ff5ac5 100644 --- a/webview-ui/src/components/chat/ChatRow.tsx +++ b/webview-ui/src/components/chat/ChatRow.tsx @@ -649,7 +649,13 @@ export const ChatRowContent = ({ vscode.postMessage({ type: "openFile", text: tool.content })}> + onClick={() => + vscode.postMessage({ + type: "openFile", + text: tool.content, + values: tool.startLine ? { line: tool.startLine } : undefined, + }) + }> {tool.path?.startsWith(".") && .} diff --git a/webview-ui/src/components/chat/ChatView.tsx b/webview-ui/src/components/chat/ChatView.tsx index 5dcdf1998e7..6e9bea03893 100644 --- a/webview-ui/src/components/chat/ChatView.tsx +++ b/webview-ui/src/components/chat/ChatView.tsx @@ -1136,7 +1136,76 @@ const ChatViewComponent: React.ForwardRefRenderFunction { // Only filter out the launch ask and result messages - browser actions appear in chat - const result: ClineMessage[] = visibleMessages.filter((msg) => !isBrowserSessionMessage(msg)) + const filtered: ClineMessage[] = visibleMessages.filter((msg) => !isBrowserSessionMessage(msg)) + + // Helper to check if a message is a read_file ask that should be batched + const isReadFileAsk = (msg: ClineMessage): boolean => { + if (msg.type !== "ask" || msg.ask !== "tool") return false + try { + const tool = JSON.parse(msg.text || "{}") + return tool.tool === "readFile" && !tool.batchFiles // Don't re-batch already batched + } catch { + return false + } + } + + // Consolidate consecutive read_file ask messages into batches + const result: ClineMessage[] = [] + let i = 0 + while (i < filtered.length) { + const msg = filtered[i] + + // Check if this starts a sequence of read_file asks + if (isReadFileAsk(msg)) { + // Collect all consecutive read_file asks + const batch: ClineMessage[] = [msg] + let j = i + 1 + while (j < filtered.length && isReadFileAsk(filtered[j])) { + batch.push(filtered[j]) + j++ + } + + if (batch.length > 1) { + // Create a synthetic batch message + const batchFiles = batch.map((batchMsg) => { + try { + const tool = JSON.parse(batchMsg.text || "{}") + return { + path: tool.path || "", + lineSnippet: tool.reason || "", + isOutsideWorkspace: tool.isOutsideWorkspace || false, + key: `${tool.path}${tool.reason ? ` (${tool.reason})` : ""}`, + content: tool.content || "", + } + } catch { + return { path: "", lineSnippet: "", key: "", content: "" } + } + }) + + // Use the first message as the base, but add batchFiles + const firstTool = JSON.parse(msg.text || "{}") + const syntheticMessage: ClineMessage = { + ...msg, + text: JSON.stringify({ + ...firstTool, + batchFiles, + }), + // Store original messages for response handling + _batchedMessages: batch, + } as ClineMessage & { _batchedMessages: ClineMessage[] } + + result.push(syntheticMessage) + i = j // Skip past all batched messages + } else { + // Single read_file ask, keep as-is + result.push(msg) + i++ + } + } else { + result.push(msg) + i++ + } + } if (isCondensing) { result.push({ diff --git a/webview-ui/src/components/settings/ContextManagementSettings.tsx b/webview-ui/src/components/settings/ContextManagementSettings.tsx index cef71534930..8663ea6e038 100644 --- a/webview-ui/src/components/settings/ContextManagementSettings.tsx +++ b/webview-ui/src/components/settings/ContextManagementSettings.tsx @@ -33,10 +33,8 @@ type ContextManagementSettingsProps = HTMLAttributes & { maxWorkspaceFiles: number showRooIgnoredFiles?: boolean enableSubfolderRules?: boolean - maxReadFileLine?: number maxImageFileSize?: number maxTotalImageSize?: number - maxConcurrentFileReads?: number profileThresholds?: Record includeDiagnosticMessages?: boolean maxDiagnosticMessages?: number @@ -53,10 +51,8 @@ type ContextManagementSettingsProps = HTMLAttributes & { | "maxWorkspaceFiles" | "showRooIgnoredFiles" | "enableSubfolderRules" - | "maxReadFileLine" | "maxImageFileSize" | "maxTotalImageSize" - | "maxConcurrentFileReads" | "profileThresholds" | "includeDiagnosticMessages" | "maxDiagnosticMessages" @@ -76,10 +72,8 @@ export const ContextManagementSettings = ({ showRooIgnoredFiles, enableSubfolderRules, setCachedStateField, - maxReadFileLine, maxImageFileSize, maxTotalImageSize, - maxConcurrentFileReads, profileThresholds = {}, includeDiagnosticMessages, maxDiagnosticMessages, @@ -218,29 +212,6 @@ export const ContextManagementSettings = ({ - - - {t("settings:contextManagement.maxConcurrentFileReads.label")} - -
- setCachedStateField("maxConcurrentFileReads", value)} - data-testid="max-concurrent-file-reads-slider" - /> - {Math.max(1, maxConcurrentFileReads ?? 5)} -
-
- {t("settings:contextManagement.maxConcurrentFileReads.description")} -
- - - -
- {t("settings:contextManagement.maxReadFile.label")} -
- { - const newValue = parseInt(e.target.value, 10) - if (!isNaN(newValue) && newValue >= -1) { - setCachedStateField("maxReadFileLine", newValue) - } - }} - onClick={(e) => e.currentTarget.select()} - data-testid="max-read-file-line-input" - disabled={maxReadFileLine === -1} - /> - {t("settings:contextManagement.maxReadFile.lines")} - - setCachedStateField("maxReadFileLine", e.target.checked ? -1 : 500) - } - data-testid="max-read-file-always-full-checkbox"> - {t("settings:contextManagement.maxReadFile.always_full_read")} - -
-
-
- {t("settings:contextManagement.maxReadFile.description")} -
- - (({ onDone, t showRooIgnoredFiles, enableSubfolderRules, remoteBrowserEnabled, - maxReadFileLine, maxImageFileSize, maxTotalImageSize, - maxConcurrentFileReads, customSupportPrompts, profileThresholds, alwaysAllowFollowupQuestions, @@ -403,10 +401,8 @@ const SettingsView = forwardRef(({ onDone, t maxWorkspaceFiles: Math.min(Math.max(0, maxWorkspaceFiles ?? 200), 500), showRooIgnoredFiles: showRooIgnoredFiles ?? true, enableSubfolderRules: enableSubfolderRules ?? false, - maxReadFileLine: maxReadFileLine ?? -1, maxImageFileSize: maxImageFileSize ?? 5, maxTotalImageSize: maxTotalImageSize ?? 20, - maxConcurrentFileReads: cachedState.maxConcurrentFileReads ?? 5, includeDiagnosticMessages: includeDiagnosticMessages !== undefined ? includeDiagnosticMessages : true, maxDiagnosticMessages: maxDiagnosticMessages ?? 50, @@ -848,10 +844,8 @@ const SettingsView = forwardRef(({ onDone, t maxWorkspaceFiles={maxWorkspaceFiles ?? 200} showRooIgnoredFiles={showRooIgnoredFiles} enableSubfolderRules={enableSubfolderRules} - maxReadFileLine={maxReadFileLine} maxImageFileSize={maxImageFileSize} maxTotalImageSize={maxTotalImageSize} - maxConcurrentFileReads={maxConcurrentFileReads} profileThresholds={profileThresholds} includeDiagnosticMessages={includeDiagnosticMessages} maxDiagnosticMessages={maxDiagnosticMessages} diff --git a/webview-ui/src/components/settings/__tests__/ContextManagementSettings.spec.tsx b/webview-ui/src/components/settings/__tests__/ContextManagementSettings.spec.tsx index b508d093404..2de2954c2b9 100644 --- a/webview-ui/src/components/settings/__tests__/ContextManagementSettings.spec.tsx +++ b/webview-ui/src/components/settings/__tests__/ContextManagementSettings.spec.tsx @@ -92,8 +92,6 @@ describe("ContextManagementSettings", () => { maxOpenTabsContext: 20, maxWorkspaceFiles: 200, showRooIgnoredFiles: false, - maxReadFileLine: -1, - maxConcurrentFileReads: 5, profileThresholds: {}, includeDiagnosticMessages: true, maxDiagnosticMessages: 50, @@ -199,7 +197,6 @@ describe("ContextManagementSettings", () => { // Check for other sliders expect(screen.getByTestId("open-tabs-limit-slider")).toBeInTheDocument() expect(screen.getByTestId("workspace-files-limit-slider")).toBeInTheDocument() - expect(screen.getByTestId("max-concurrent-file-reads-slider")).toBeInTheDocument() // Check for checkboxes expect(screen.getByTestId("show-rooignored-files-checkbox")).toBeInTheDocument() @@ -320,50 +317,6 @@ describe("ContextManagementSettings", () => { }) }) - it("renders max read file line controls", () => { - const propsWithMaxReadFileLine = { - ...defaultProps, - maxReadFileLine: 500, - } - render() - - // Max read file line input - const maxReadFileInput = screen.getByTestId("max-read-file-line-input") - expect(maxReadFileInput).toBeInTheDocument() - expect(maxReadFileInput).toHaveValue(500) - - // Always full read checkbox - const alwaysFullReadCheckbox = screen.getByTestId("max-read-file-always-full-checkbox") - expect(alwaysFullReadCheckbox).toBeInTheDocument() - expect(alwaysFullReadCheckbox).not.toBeChecked() - }) - - it("updates max read file line setting", () => { - const propsWithMaxReadFileLine = { - ...defaultProps, - maxReadFileLine: 500, - } - render() - - const input = screen.getByTestId("max-read-file-line-input") - fireEvent.change(input, { target: { value: "1000" } }) - - expect(defaultProps.setCachedStateField).toHaveBeenCalledWith("maxReadFileLine", 1000) - }) - - it("toggles always full read setting", () => { - const propsWithMaxReadFileLine = { - ...defaultProps, - maxReadFileLine: 500, - } - render() - - const checkbox = screen.getByTestId("max-read-file-always-full-checkbox") - fireEvent.click(checkbox) - - expect(defaultProps.setCachedStateField).toHaveBeenCalledWith("maxReadFileLine", -1) - }) - it("renders with autoCondenseContext enabled", () => { const propsWithAutoCondense = { ...defaultProps, @@ -440,18 +393,6 @@ describe("ContextManagementSettings", () => { }) }) - it("renders max read file line controls with -1 value", () => { - const propsWithMaxReadFileLine = { - ...defaultProps, - maxReadFileLine: -1, - } - render() - - const checkbox = screen.getByTestId("max-read-file-always-full-checkbox") - const input = checkbox.querySelector('input[type="checkbox"]') - expect(input).toBeChecked() - }) - it("handles boundary values for sliders", () => { const mockSetCachedStateField = vitest.fn() const props = { @@ -478,7 +419,6 @@ describe("ContextManagementSettings", () => { const propsWithUndefined = { ...defaultProps, showRooIgnoredFiles: undefined, - maxReadFileLine: undefined, } expect(() => { @@ -501,24 +441,6 @@ describe("ContextManagementSettings", () => { // When auto condense is false, threshold slider should not be visible expect(screen.queryByTestId("condense-threshold-slider")).not.toBeInTheDocument() }) - - it("renders max read file controls with default value when maxReadFileLine is undefined", () => { - const propsWithoutMaxReadFile = { - ...defaultProps, - maxReadFileLine: undefined, - } - render() - - // Controls should still be rendered with default value of -1 - const input = screen.getByTestId("max-read-file-line-input") - const checkbox = screen.getByTestId("max-read-file-always-full-checkbox") - - expect(input).toBeInTheDocument() - expect(input).toHaveValue(-1) - expect(input).not.toBeDisabled() // Input is not disabled when maxReadFileLine is undefined (only when explicitly set to -1) - expect(checkbox).toBeInTheDocument() - expect(checkbox).not.toBeChecked() // Checkbox is not checked when maxReadFileLine is undefined (only when explicitly set to -1) - }) }) describe("Accessibility", () => { @@ -537,17 +459,11 @@ describe("ContextManagementSettings", () => { }) it("has proper test ids for all interactive elements", () => { - const propsWithMaxReadFile = { - ...defaultProps, - maxReadFileLine: 500, - } - render() + render() expect(screen.getByTestId("open-tabs-limit-slider")).toBeInTheDocument() expect(screen.getByTestId("workspace-files-limit-slider")).toBeInTheDocument() expect(screen.getByTestId("show-rooignored-files-checkbox")).toBeInTheDocument() - expect(screen.getByTestId("max-read-file-line-input")).toBeInTheDocument() - expect(screen.getByTestId("max-read-file-always-full-checkbox")).toBeInTheDocument() }) }) diff --git a/webview-ui/src/components/settings/__tests__/SettingsView.change-detection.spec.tsx b/webview-ui/src/components/settings/__tests__/SettingsView.change-detection.spec.tsx index 89be961625b..07f8e0d30ea 100644 --- a/webview-ui/src/components/settings/__tests__/SettingsView.change-detection.spec.tsx +++ b/webview-ui/src/components/settings/__tests__/SettingsView.change-detection.spec.tsx @@ -193,7 +193,6 @@ describe("SettingsView - Change Detection Fix", () => { maxReadFileLine: -1, maxImageFileSize: 5, maxTotalImageSize: 20, - maxConcurrentFileReads: 5, customCondensingPrompt: "", customSupportPrompts: {}, profileThresholds: {}, diff --git a/webview-ui/src/components/settings/__tests__/SettingsView.unsaved-changes.spec.tsx b/webview-ui/src/components/settings/__tests__/SettingsView.unsaved-changes.spec.tsx index 996dad86399..7a9c947e0e4 100644 --- a/webview-ui/src/components/settings/__tests__/SettingsView.unsaved-changes.spec.tsx +++ b/webview-ui/src/components/settings/__tests__/SettingsView.unsaved-changes.spec.tsx @@ -198,7 +198,6 @@ describe("SettingsView - Unsaved Changes Detection", () => { maxReadFileLine: -1, maxImageFileSize: 5, maxTotalImageSize: 20, - maxConcurrentFileReads: 5, customCondensingPrompt: "", customSupportPrompts: {}, profileThresholds: {}, diff --git a/webview-ui/src/context/ExtensionStateContext.tsx b/webview-ui/src/context/ExtensionStateContext.tsx index 9594f83b86a..ef1aad86b95 100644 --- a/webview-ui/src/context/ExtensionStateContext.tsx +++ b/webview-ui/src/context/ExtensionStateContext.tsx @@ -48,7 +48,6 @@ export interface ExtensionStateContextType extends ExtensionState { cloudOrganizations?: CloudOrganizationMembership[] sharingEnabled: boolean publicSharingEnabled: boolean - maxConcurrentFileReads?: number mdmCompliant?: boolean hasOpenedModeSelector: boolean // New property to track if user has opened mode selector setHasOpenedModeSelector: (value: boolean) => void // Setter for the new property @@ -126,8 +125,6 @@ export interface ExtensionStateContextType extends ExtensionState { setRemoteBrowserEnabled: (value: boolean) => void awsUsePromptCache?: boolean setAwsUsePromptCache: (value: boolean) => void - maxReadFileLine: number - setMaxReadFileLine: (value: number) => void maxImageFileSize: number setMaxImageFileSize: (value: number) => void maxTotalImageSize: number @@ -230,12 +227,10 @@ export const ExtensionStateContextProvider: React.FC<{ children: React.ReactNode showRooIgnoredFiles: true, // Default to showing .rooignore'd files with lock symbol (current behavior). enableSubfolderRules: false, // Default to disabled - must be enabled to load rules from subdirectories renderContext: "sidebar", - maxReadFileLine: -1, // Default max read file line limit maxImageFileSize: 5, // Default max image file size in MB maxTotalImageSize: 20, // Default max total image size in MB pinnedApiConfigs: {}, // Empty object for pinned API configs terminalZshOhMy: false, // Default Oh My Zsh integration setting - maxConcurrentFileReads: 5, // Default concurrent file reads terminalZshP10k: false, // Default Powerlevel10k integration setting terminalZdotdir: false, // Default ZDOTDIR handling setting historyPreviewCollapsed: false, // Initialize the new state (default to expanded) @@ -563,7 +558,6 @@ export const ExtensionStateContextProvider: React.FC<{ children: React.ReactNode setEnableSubfolderRules: (value) => setState((prevState) => ({ ...prevState, enableSubfolderRules: value })), setRemoteBrowserEnabled: (value) => setState((prevState) => ({ ...prevState, remoteBrowserEnabled: value })), setAwsUsePromptCache: (value) => setState((prevState) => ({ ...prevState, awsUsePromptCache: value })), - setMaxReadFileLine: (value) => setState((prevState) => ({ ...prevState, maxReadFileLine: value })), setMaxImageFileSize: (value) => setState((prevState) => ({ ...prevState, maxImageFileSize: value })), setMaxTotalImageSize: (value) => setState((prevState) => ({ ...prevState, maxTotalImageSize: value })), setPinnedApiConfigs: (value) => setState((prevState) => ({ ...prevState, pinnedApiConfigs: value })), diff --git a/webview-ui/src/context/__tests__/ExtensionStateContext.spec.tsx b/webview-ui/src/context/__tests__/ExtensionStateContext.spec.tsx index 4d8be85728f..bfc60f1ace4 100644 --- a/webview-ui/src/context/__tests__/ExtensionStateContext.spec.tsx +++ b/webview-ui/src/context/__tests__/ExtensionStateContext.spec.tsx @@ -202,7 +202,6 @@ describe("mergeExtensionState", () => { showRooIgnoredFiles: true, enableSubfolderRules: false, renderContext: "sidebar", - maxReadFileLine: 500, cloudUserInfo: null, organizationAllowList: { allowAll: true, providers: {} }, autoCondenseContext: true, diff --git a/webview-ui/src/utils/formatPathTooltip.ts b/webview-ui/src/utils/formatPathTooltip.ts index cfe0b54a7fc..aeaafe6cb46 100644 --- a/webview-ui/src/utils/formatPathTooltip.ts +++ b/webview-ui/src/utils/formatPathTooltip.ts @@ -21,7 +21,7 @@ export function formatPathTooltip(path?: string, additionalContent?: string): st const formattedPath = removeLeadingNonAlphanumeric(path) + "\u200E" if (additionalContent) { - return formattedPath + additionalContent + return formattedPath + " " + additionalContent } return formattedPath