diff --git a/backend/src/main/java/run/webup/platform/AgentWorker.java b/backend/src/main/java/run/webup/platform/AgentWorker.java index 5e2b407..a19d242 100644 --- a/backend/src/main/java/run/webup/platform/AgentWorker.java +++ b/backend/src/main/java/run/webup/platform/AgentWorker.java @@ -278,11 +278,17 @@ public class AgentWorker { } tasks.heartbeat(id, lease); external = call.argumentError() == null; - String output = - call.argumentError() == null - ? sandbox.execute(project, call) - : json.writeValueAsString(Map.of("error", call.argumentError())); - earlierToolFailed = tasks.tool(id, lease, call, output); + boolean trackChanges = external && SandboxClient.tracksChanges(call); + try { + String output = + external + ? sandbox.execute(project, call) + : json.writeValueAsString(Map.of("error", call.argumentError())); + earlierToolFailed = tasks.tool(id, lease, call, output, trackChanges); + } catch (Exception failure) { + if (trackChanges) tasks.markFileChangesPartial(id, lease); + throw failure; + } } external = false; } diff --git a/backend/src/main/java/run/webup/platform/ProjectFilesController.java b/backend/src/main/java/run/webup/platform/ProjectFilesController.java index fe230d9..2482e4a 100644 --- a/backend/src/main/java/run/webup/platform/ProjectFilesController.java +++ b/backend/src/main/java/run/webup/platform/ProjectFilesController.java @@ -31,7 +31,7 @@ public class ProjectFilesController { projects.find(who, project); try { return json.readTree( - sandbox.execute( + sandbox.executeFileTool( project, new ModelGateway.Call(UUID.randomUUID().toString(), name, json.valueToTree(args)))); } catch (ApiError.Failure f) { diff --git a/backend/src/main/java/run/webup/platform/SandboxClient.java b/backend/src/main/java/run/webup/platform/SandboxClient.java index 4c976d1..9ec969e 100644 --- a/backend/src/main/java/run/webup/platform/SandboxClient.java +++ b/backend/src/main/java/run/webup/platform/SandboxClient.java @@ -74,6 +74,15 @@ public class SandboxClient { } public String execute(UUID project, ModelGateway.Call call) { + if (tracksChanges(call)) return executeBounded(project, call, 512000, false, true); + return executeFileTool(project, call); + } + + static boolean tracksChanges(ModelGateway.Call call) { + return Set.of("write_file", "delete_file", "run_command").contains(call.name()); + } + + String executeFileTool(UUID project, ModelGateway.Call call) { return executeBounded(project, call, 131072); } @@ -175,6 +184,11 @@ public class SandboxClient { } private String executeBounded(UUID project, ModelGateway.Call call, int limit, boolean review) { + return executeBounded(project, call, limit, review, false); + } + + private String executeBounded( + UUID project, ModelGateway.Call call, int limit, boolean review, boolean trackChanges) { check(); var allowed = review @@ -192,8 +206,9 @@ public class SandboxClient { "review_export"); if (!allowed.contains(call.name())) throw new ApiError.Failure(400, "不允许的工具"); try { - String payload = - json.writeValueAsString(Map.of("name", call.name(), "arguments", call.arguments())); + var body = json.createObjectNode().put("name", call.name()); + body.set("arguments", call.arguments()); + if (trackChanges) body.put("trackChanges", true); var request = HttpRequest.newBuilder( URI.create( @@ -204,7 +219,7 @@ public class SandboxClient { .timeout(Duration.ofSeconds(150)) .header("Authorization", "Bearer " + token) .header("Content-Type", "application/json") - .POST(HttpRequest.BodyPublishers.ofString(payload)) + .POST(HttpRequest.BodyPublishers.ofString(body.toString())) .build(); var response = client.send(request, HttpResponse.BodyHandlers.ofInputStream()); try (var input = response.body()) { diff --git a/backend/src/main/java/run/webup/platform/TaskController.java b/backend/src/main/java/run/webup/platform/TaskController.java index 5b37a0d..ec59030 100644 --- a/backend/src/main/java/run/webup/platform/TaskController.java +++ b/backend/src/main/java/run/webup/platform/TaskController.java @@ -81,6 +81,35 @@ public class TaskController { return tasks.history(id); } + @GetMapping("/{id}/changes") + public Map changes( + @AuthenticationPrincipal SecurityConfig.Identity who, + @PathVariable UUID id, + @RequestParam(defaultValue = "0") @Min(0) @Max(1000000) int page) { + var task = tasks.find(id, who.id()); + var totals = + db.queryForMap( + "SELECT COUNT(*) AS file_count,COALESCE(SUM(added_lines),0) AS added_lines," + + " COALESCE(SUM(removed_lines),0) AS removed_lines" + + " FROM ai_task_file_changes WHERE task_id=?", + id); + var files = + db.queryForList( + "SELECT file_path,last_status,added_lines,removed_lines,binary_change FROM" + + " ai_task_file_changes WHERE task_id=? ORDER BY file_path LIMIT 100 OFFSET ?", + id, + page * 100); + return Map.of( + "checks", + task.get("file_change_checks"), + "partial", + task.get("file_changes_partial"), + "totals", + totals, + "files", + files); + } + @PostMapping("/{id}/{action:pause|resume|cancel}") public Map action( @AuthenticationPrincipal SecurityConfig.Identity who, diff --git a/backend/src/main/java/run/webup/platform/TaskService.java b/backend/src/main/java/run/webup/platform/TaskService.java index ed5582b..603af9a 100644 --- a/backend/src/main/java/run/webup/platform/TaskService.java +++ b/backend/src/main/java/run/webup/platform/TaskService.java @@ -2,6 +2,7 @@ package run.webup.platform; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.*; +import com.fasterxml.jackson.databind.node.ObjectNode; import java.math.BigDecimal; import java.sql.Timestamp; import java.time.*; @@ -377,8 +378,20 @@ public class TaskService { @Transactional public boolean tool(UUID id, String lease, ModelGateway.Call call, String output) throws JsonProcessingException { + return tool(id, lease, call, output, false); + } + + @Transactional + public boolean tool( + UUID id, String lease, ModelGateway.Call call, String output, boolean trackChanges) + throws JsonProcessingException { locked(id, lease); var result = json.readTree(output); + if (trackChanges) { + var report = result instanceof ObjectNode object ? object.remove("fileChanges") : null; + recordFileChanges(id, report); + output = result.toString(); + } boolean failed = result.hasNonNull("error") || result.path("timedOut").asBoolean(false) @@ -395,6 +408,62 @@ public class TaskService { return failed; } + private void recordFileChanges(UUID task, JsonNode report) { + if (report == null || !report.path("files").isArray() || report.path("files").size() > 120) { + db.update("UPDATE ai_tasks SET file_changes_partial=TRUE WHERE id=?", task); + return; + } + boolean partial = !report.path("complete").asBoolean(false); + for (var file : report.path("files")) { + String path = file.path("path").asText(), status = file.path("status").asText(); + int added = file.path("added").asInt(-1), removed = file.path("removed").asInt(-1); + if (path.isEmpty() + || path.indexOf(0) >= 0 + || path.getBytes(java.nio.charset.StandardCharsets.UTF_8).length > 1000 + || !Set.of("ADDED", "MODIFIED", "DELETED").contains(status) + || added < 0 + || removed < 0) { + partial = true; + continue; + } + boolean binary = file.path("binary").asBoolean(false); + int updated = + db.update( + "UPDATE ai_task_file_changes SET last_status=?,added_lines=added_lines+?," + + " removed_lines=removed_lines+?,binary_change=(binary_change OR ?)" + + " WHERE task_id=? AND file_path=?", + status, + added, + removed, + binary, + task, + path); + if (updated == 0) { + db.update( + "INSERT INTO ai_task_file_changes" + + " (task_id,file_path,last_status,added_lines,removed_lines,binary_change)" + + " VALUES (?,?,?,?,?,?)", + task, + path, + status, + added, + removed, + binary); + } + } + db.update( + "UPDATE ai_tasks SET file_change_checks=file_change_checks+1," + + " file_changes_partial=(file_changes_partial OR ?) WHERE id=?", + partial, + task); + } + + @Transactional + public void markFileChangesPartial(UUID task, String lease) { + locked(task, lease); + recordFileChanges(task, null); + } + @Transactional public void requireSource(UUID id, String lease) { locked(id, lease); diff --git a/backend/src/main/resources/db/migration/V26__agent_file_changes.sql b/backend/src/main/resources/db/migration/V26__agent_file_changes.sql new file mode 100644 index 0000000..fdecc72 --- /dev/null +++ b/backend/src/main/resources/db/migration/V26__agent_file_changes.sql @@ -0,0 +1,12 @@ +ALTER TABLE ai_tasks ADD COLUMN file_change_checks INTEGER NOT NULL DEFAULT 0; +ALTER TABLE ai_tasks ADD COLUMN file_changes_partial BOOLEAN NOT NULL DEFAULT FALSE; + +CREATE TABLE ai_task_file_changes ( + task_id UUID NOT NULL REFERENCES ai_tasks(id) ON DELETE CASCADE, + file_path VARCHAR(1000) NOT NULL, + last_status VARCHAR(12) NOT NULL CHECK (last_status IN ('ADDED', 'MODIFIED', 'DELETED')), + added_lines BIGINT NOT NULL CHECK (added_lines >= 0), + removed_lines BIGINT NOT NULL CHECK (removed_lines >= 0), + binary_change BOOLEAN NOT NULL, + PRIMARY KEY (task_id, file_path) +); diff --git a/backend/src/test/java/run/webup/platform/AgentWorkerIntegrationTest.java b/backend/src/test/java/run/webup/platform/AgentWorkerIntegrationTest.java index 0ddd65a..0f7999e 100644 --- a/backend/src/test/java/run/webup/platform/AgentWorkerIntegrationTest.java +++ b/backend/src/test/java/run/webup/platform/AgentWorkerIntegrationTest.java @@ -745,6 +745,67 @@ class AgentWorkerIntegrationTest { return result; } + @Test + void toolFileChangesAreSummedAndVisibleOnlyToTheTaskOwner() throws Exception { + UUID id = + (UUID) + tasks + .create(user, project, model, "Build", new BigDecimal("20"), "file-changes") + .get("id"); + var claimed = tasks.claim(); + when(transport.request(anyString(), anyString(), anyString(), anyString(), anyInt(), any())) + .thenReturn( + response("Write", "write_file"), + response("Edit", "write_file"), + response("Done", null)); + when(sandbox.execute(eq(project), any())) + .thenReturn( + """ + {"written":12,"fileChanges":{"complete":true,"files":[{"path":"index.html","status":"ADDED","added":3,"removed":0,"binary":false}]}} + """, + """ + {"written":10,"fileChanges":{"complete":true,"files":[{"path":"index.html","status":"MODIFIED","added":1,"removed":2,"binary":false}]}} + """); + + new AgentWorker(tasks, gateway, sandbox, vault, json).run(claimed); + + assertEquals("COMPLETED", tasks.config(id).get("status")); + assertTrue( + tasks.history(id).stream() + .noneMatch(message -> message.toString().contains("fileChanges"))); + String token = + users + .login("worker@test.com", "StrongPassword123", "test", "test", 1) + .get("token") + .toString(); + mvc.perform(get("/api/tasks/" + id + "/changes").header("Authorization", "Bearer " + token)) + .andExpect(status().isOk()) + .andExpect( + org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath( + "$.totals.file_count") + .value(1)) + .andExpect( + org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath( + "$.totals.added_lines") + .value(4)) + .andExpect( + org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath( + "$.totals.removed_lines") + .value(2)) + .andExpect( + org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath( + "$.files[0].last_status") + .value("MODIFIED")); + users.create("other-owner@test.com", "Other", "StrongPassword123", "test"); + String other = + users + .login("other-owner@test.com", "StrongPassword123", "test", "test", 1) + .get("token") + .toString(); + mvc.perform(get("/api/tasks/" + id + "/changes").header("Authorization", "Bearer " + other)) + .andExpect(status().isNotFound()); + } + @Test void failedToolReturnsResultsForWholeBatchAndLetsAgentRecover() throws Exception { var batch = response("Build, then update the site", "run_command"); diff --git a/docs/API.md b/docs/API.md index 5c2582b..6db79f0 100644 --- a/docs/API.md +++ b/docs/API.md @@ -106,6 +106,7 @@ | GET | `/api/tasks/{id}` | 本人任务、预算与消费,不返回租约令牌 | | GET | `/api/tasks/{id}/events` | 本人最近 1000 条任务事件,按时间从早到晚返回 | | GET | `/api/tasks/{id}/messages` | 本人持久化消息与工具上下文 | +| GET | `/api/tasks/{id}/changes?page=0` | 本任务 Agent 文件变更,含累计增删行、最后一次操作和统计是否完整。每页 100 个文件 | | POST | `/api/tasks/{id}/pause` | 暂停新的调用,保留剩余预算 | | POST | `/api/tasks/{id}/resume` | PAUSED 任务重新入队 | | POST | `/api/tasks/{id}/cancel` | 结算当前调用,停止后续工具并释放未用预留 | diff --git a/frontend/src/views/Develop.vue b/frontend/src/views/Develop.vue index 76d3c0b..62d30b4 100644 --- a/frontend/src/views/Develop.vue +++ b/frontend/src/views/Develop.vue @@ -1,6 +1,7 @@ + + + + diff --git a/node-agent/Dockerfile b/node-agent/Dockerfile index aa59c83..47982f1 100644 --- a/node-agent/Dockerfile +++ b/node-agent/Dockerfile @@ -3,7 +3,7 @@ FROM ${BASE_IMAGE} WORKDIR /app COPY requirements.txt . RUN pip install --no-cache-dir -r requirements.txt -COPY app.py runtime.py domain_runtime.py review_tools.py ./ +COPY app.py runtime.py domain_runtime.py review_tools.py workspace_changes.py ./ USER 10001:10001 EXPOSE 8090 CMD ["uvicorn","app:app","--host","0.0.0.0","--port","8090","--workers","1","--no-access-log"] diff --git a/node-agent/app.py b/node-agent/app.py index 30268d3..4d61c2c 100644 --- a/node-agent/app.py +++ b/node-agent/app.py @@ -54,6 +54,7 @@ app.include_router(domain_router) class Tool(BaseModel): name: str = Field(max_length=40) arguments: dict + trackChanges: bool = False def authorize(value: str | None): @@ -201,6 +202,17 @@ import pathlib,os,json,base64,subprocess,signal,tempfile args=json.loads(base64.b64decode(PAYLOAD)) root=pathlib.Path('/workspace').resolve() name=args['name']; values=args['arguments'] +track_changes = args.get('trackChanges') and name in {'write_file', 'delete_file', 'run_command'} +temporary = None +before_complete = False +if track_changes: + try: + temporary = tempfile.TemporaryDirectory(prefix='webup-changes-', ignore_cleanup_errors=True) + change_dir = pathlib.Path(temporary.name) + snapshot_changes(root, change_dir / 'before') + before_complete = True + except (OSError, ValueError): + pass def path(value): if not isinstance(value,str) or len(value)>1000: raise ValueError('Invalid path') target=(root/value).resolve() @@ -288,11 +300,23 @@ try: if not target.is_file(): raise ValueError('Only individual files may be deleted') target.unlink();result={'deleted':True} else: raise ValueError('Unknown tool') - print(json.dumps(result,ensure_ascii=False)) except Exception as exc: - print(json.dumps({'error':str(exc)[:500]})) + result = {'error': str(exc)[:500]} +if track_changes: + result['fileChanges'] = {'complete': False, 'files': []} + try: + if before_complete: + snapshot_changes(root, change_dir / 'after') + result['fileChanges'] = summarize_changes(change_dir) + except (OSError, ValueError, subprocess.SubprocessError): + pass + finally: + if temporary is not None: + temporary.cleanup() +print(json.dumps(result, ensure_ascii=True if track_changes else False)) ''' -SCRIPT = Path(__file__).with_name('review_tools.py').read_text(encoding='utf-8') + '\n' + SCRIPT +SCRIPT = '\n'.join(Path(__file__).with_name(name).read_text(encoding='utf-8') + for name in ('review_tools.py', 'workspace_changes.py')) + '\n' + SCRIPT def read_exec_output(connection, limit): @@ -385,7 +409,10 @@ def execute_tool(project,request,authorization,review): except Exception: connection.close() raise - output = read_exec_output(connection, 16_000_000 if request.name in {'archive_source','review_export'} else MAX_OUTPUT) + output_limit = 16_000_000 if request.name in {'archive_source','review_export'} else MAX_OUTPUT + if request.trackChanges: + output_limit = 512_000 + output = read_exec_output(connection, output_limit) try: return json.loads(output) except ValueError: diff --git a/node-agent/test_gateway.py b/node-agent/test_gateway.py index 5e7e9b4..f5ef339 100644 --- a/node-agent/test_gateway.py +++ b/node-agent/test_gateway.py @@ -17,8 +17,10 @@ spec.loader.exec_module(gateway) class GatewayTests(unittest.TestCase): - def execute_file_tool(self, root, name, arguments): - payload = base64.b64encode(json.dumps({"name": name, "arguments": arguments}).encode()).decode() + def execute_file_tool(self, root, name, arguments, track_changes=False): + payload = base64.b64encode(json.dumps({ + "name": name, "arguments": arguments, "trackChanges": track_changes + }).encode()).decode() script = gateway.SCRIPT.replace("pathlib.Path('/workspace')", "pathlib.Path(" + repr(str(root)) + ")") result = subprocess.run([sys.executable, "-X", "utf8", "-c", "PAYLOAD=" + repr(payload) + "\n" + script], capture_output=True, timeout=10) self.assertEqual(0, result.returncode, result.stderr.decode()) @@ -39,6 +41,33 @@ class GatewayTests(unittest.TestCase): (root / "large.txt").write_bytes(b"x" * 64001) self.assertIn("error", self.execute_file_tool(root, "read_file", {"path": "large.txt"})) + def test_agent_file_change_counts_and_manual_edits_are_separate(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.assertNotIn("fileChanges", self.execute_file_tool(root, "write_file", { + "path": "manual.txt", "content": "user\n" + })) + results = [] + for tool, args in [ + ("write_file", {"path": "src/code.js", "content": "one\ntwo\n"}), + ("write_file", {"path": "src/code.js", "content": "one\nthree\nfour\n"}), + ("delete_file", {"path": "src/code.js"}), + ]: + result = self.execute_file_tool(root, tool, args, track_changes=True) + self.assertTrue(result["fileChanges"]["complete"]) + files = result["fileChanges"]["files"] + self.assertEqual(["src/code.js"], [file["path"] for file in files]) + results.append(tuple(files[0][key] for key in ("status", "added", "removed"))) + self.assertEqual([("ADDED", 2, 0), ("MODIFIED", 2, 1), ("DELETED", 0, 3)], results) + + def test_source_scan_failure_reports_partial_and_keeps_successful_write(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "large.dat").write_bytes(b"x" * 8_000_001) + result = self.execute_file_tool(root, "write_file", {"path": "index.js", "content": "ok"}, True) + self.assertEqual(2, result["written"]) + self.assertEqual({"complete": False, "files": []}, result["fileChanges"]) + def test_workspace_inspection_requires_nonempty_source_and_excludes_dependencies(self): with tempfile.TemporaryDirectory() as directory: root=Path(directory) diff --git a/node-agent/workspace_changes.py b/node-agent/workspace_changes.py new file mode 100644 index 0000000..0ff7df7 --- /dev/null +++ b/node-agent/workspace_changes.py @@ -0,0 +1,84 @@ +"""Bounded, private snapshots used for Agent tool change counts inside a workspace Pod.""" + +import os +import pathlib +import stat +import subprocess +import time + +CHANGE_EXCLUDED = {".npm", ".cache", ".pytest_cache", ".output", ".nuxt", ".turbo", + "vendor", "target", "venv", ".mypy_cache"} + + +def snapshot_changes(root, destination): + destination.mkdir() + deadline = time.monotonic() + 2 + total = visited = 0 + def fail(error): + raise error + + for directory, dirs, files in os.walk(root, followlinks=False, onerror=fail): + visited += 1 + if visited > 5000 or time.monotonic() >= deadline: + raise ValueError("Workspace change scan limit reached") + current = pathlib.Path(directory) + dirs[:] = sorted(d for d in dirs if d not in CHANGE_EXCLUDED + and not review_secret((current / d).relative_to(root)) + and not (current / d).is_symlink()) + for name in sorted(files): + visited += 1 + if visited > 5000 or time.monotonic() >= deadline: + raise ValueError("Workspace change scan limit reached") + source = current / name + relative = source.relative_to(root) + if review_secret(relative) or source.is_symlink() or not source.is_file(): + continue + if not source.resolve().is_relative_to(root): + raise ValueError("Workspace file moved during scan") + # Nonblocking open and fstat guard against paths replaced by special files. + flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0) + with os.fdopen(os.open(source, flags), "rb") as stream: + if not stat.S_ISREG(os.fstat(stream.fileno()).st_mode): + raise ValueError("Workspace file changed during scan") + data = stream.read(8_000_000 - total + 1) + total += len(data) + if total > 8_000_000: + raise ValueError("Workspace change snapshot exceeds 8 MB") + target = destination / relative + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(data) + + +def summarize_changes(directory): + result = subprocess.run( + ["git", "-c", "core.autocrlf=false", "diff", "--no-index", "--no-ext-diff", + "--no-textconv", "--no-renames", "--numstat", "-z", "--", "before", "after"], + cwd=directory, capture_output=True, timeout=4, + env={**os.environ, "GIT_CONFIG_NOSYSTEM": "1", "GIT_CONFIG_GLOBAL": os.devnull}, + ) + if result.returncode not in (0, 1): + raise ValueError("Unable to compare workspace snapshots") + records = iter(result.stdout.split(b"\0")) + files, size = [], 0 + for counts in records: + if not counts: + break + added, removed, path = counts.split(b"\t", 2) + if path: + raise ValueError("Unexpected Git snapshot path format") + old, new = next(records, b""), next(records, b"") + if new == b"/dev/null" and old.startswith(b"before/"): + path, status = old[len(b"before/"):], "DELETED" + elif new.startswith(b"after/") and (old == b"/dev/null" or old.startswith(b"before/")): + path = new[len(b"after/"):] + status = "ADDED" if old == b"/dev/null" else "MODIFIED" + else: + raise ValueError("Invalid Git snapshot path") + size += len(path) + if len(files) >= 120 or size > 12000 or len(path) > 1000: + return {"complete": False, "files": files} + binary = added == b"-" + files.append({"path": path.decode("utf-8"), "status": status, + "added": 0 if binary else int(added), "removed": 0 if binary else int(removed), + "binary": binary}) + return {"complete": True, "files": files}