This commit is contained in:
commit
db82325ed7
216 files changed
+33718
No files matched your search
@@ -0,0 +1,439 @@
|
||||
package run.webup.platform;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
import static org.mockito.Mockito.*;
|
||||
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.*;
|
||||
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import java.math.BigDecimal;
|
||||
import java.sql.Timestamp;
|
||||
import java.time.Instant;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.*;
|
||||
import org.junit.jupiter.api.*;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.test.context.ActiveProfiles;
|
||||
import org.springframework.test.context.bean.override.mockito.MockitoBean;
|
||||
import org.springframework.test.web.servlet.MockMvc;
|
||||
|
||||
@SpringBootTest
|
||||
@AutoConfigureMockMvc
|
||||
@ActiveProfiles("test")
|
||||
class QueueIntegrationTest {
|
||||
@Autowired IdentityService users;
|
||||
@Autowired TaskService tasks;
|
||||
@Autowired QueueService queue;
|
||||
@Autowired WalletService wallets;
|
||||
@Autowired ObjectMapper json;
|
||||
@Autowired ReconciliationService reconciliation;
|
||||
@Autowired MockMvc mvc;
|
||||
@MockitoBean EphemeralStore ephemeral;
|
||||
UUID user, model;
|
||||
int counter;
|
||||
|
||||
BigDecimal n(String value) {
|
||||
return new BigDecimal(value);
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
void setup() {
|
||||
TestDatabase.clear(users.db());
|
||||
counter = 0;
|
||||
user = users.create("queue@test.com", "Queue", "StrongPassword123", "test");
|
||||
model = UUID.randomUUID();
|
||||
UUID provider = UUID.randomUUID(), price = UUID.randomUUID();
|
||||
users
|
||||
.db()
|
||||
.update(
|
||||
"INSERT INTO"
|
||||
+ " ai_providers(id,name,base_url,api_type,encrypted_key,enabled,timeout_seconds)"
|
||||
+ " VALUES (?,'Fixture','https://api.openai.com/v1','OPENAI','fixture',TRUE,30)",
|
||||
provider);
|
||||
users
|
||||
.db()
|
||||
.update(
|
||||
"INSERT INTO"
|
||||
+ " ai_models(id,provider_id,name,model_id,context_length,max_output,tools_enabled,vision_enabled,enabled)"
|
||||
+ " VALUES (?,?,'Fixture','fixture',8192,256,TRUE,FALSE,TRUE)",
|
||||
model,
|
||||
provider);
|
||||
users
|
||||
.db()
|
||||
.update(
|
||||
"INSERT INTO"
|
||||
+ " ai_model_price_versions(id,model_id,input_price,output_price,cache_read_price,cache_write_price)"
|
||||
+ " VALUES (?,?,1,2,0,0)",
|
||||
price,
|
||||
model);
|
||||
users.db().update("UPDATE ai_models SET current_price_version=? WHERE id=?", price, model);
|
||||
wallets.adjust(user, n("100"), "credit", "test", user, "test");
|
||||
users
|
||||
.db()
|
||||
.update(
|
||||
"INSERT INTO worker_presence(instance_id,kind,status,capacity) VALUES"
|
||||
+ " (?,'AI','READY',8)",
|
||||
UUID.randomUUID());
|
||||
}
|
||||
|
||||
UUID task() {
|
||||
UUID project = UUID.randomUUID();
|
||||
users
|
||||
.db()
|
||||
.update(
|
||||
"INSERT INTO projects(id,owner_id,name,description,stack,requirements) VALUES"
|
||||
+ " (?,?,'Queue','','HTML','')",
|
||||
project,
|
||||
user);
|
||||
UUID id =
|
||||
(UUID) tasks.create(user, project, model, "Build", n("2"), "task-" + counter).get("id");
|
||||
users
|
||||
.db()
|
||||
.update(
|
||||
"UPDATE ai_tasks SET created_at=? WHERE id=?",
|
||||
Timestamp.from(Instant.parse("2026-01-01T00:00:00Z").plusSeconds(counter++)),
|
||||
id);
|
||||
return id;
|
||||
}
|
||||
|
||||
void enable(int max) {
|
||||
queue.configure(user, max, true, n("2"), "test", "test");
|
||||
}
|
||||
|
||||
void finish(Map<String, Object> task) {
|
||||
tasks.finish(
|
||||
(UUID) task.get("id"), task.get("lease_token").toString(), "COMPLETED", "test", true);
|
||||
}
|
||||
|
||||
@Test
|
||||
void simultaneousClaimsNeverExceedGlobalConcurrency() throws Exception {
|
||||
for (int i = 0; i < 8; i++) task();
|
||||
enable(2);
|
||||
var pool = Executors.newFixedThreadPool(8);
|
||||
try {
|
||||
var start = new CountDownLatch(1);
|
||||
List<Future<Map<String, Object>>> calls = new ArrayList<>();
|
||||
for (int i = 0; i < 8; i++)
|
||||
calls.add(
|
||||
pool.submit(
|
||||
() -> {
|
||||
start.await();
|
||||
return tasks.claim();
|
||||
}));
|
||||
start.countDown();
|
||||
Set<UUID> claimed = new HashSet<>();
|
||||
for (var call : calls) {
|
||||
var value = call.get(15, TimeUnit.SECONDS);
|
||||
if (value != null) claimed.add((UUID) value.get("id"));
|
||||
}
|
||||
assertEquals(2, claimed.size());
|
||||
assertEquals(2L, queue.occupied());
|
||||
} finally {
|
||||
pool.shutdownNow();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void administratorWeightPaidPriorityAndFifoUseOneOrder() {
|
||||
enable(1);
|
||||
UUID ordinary = task(), paid = task(), manual = task();
|
||||
queue.purchase(user, paid, n("2"), "paid");
|
||||
queue.prioritize(user, manual, 10, false, "test", "test");
|
||||
var first = tasks.claim();
|
||||
assertEquals(manual, first.get("id"));
|
||||
finish(first);
|
||||
var second = tasks.claim();
|
||||
assertEquals(paid, second.get("id"));
|
||||
tasks.beginExecution(paid, second.get("lease_token").toString());
|
||||
finish(second);
|
||||
assertEquals(ordinary, tasks.claim().get("id"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void runNextDoesNotInterruptRunningOrBypassCap() {
|
||||
enable(1);
|
||||
UUID running = task(), older = task(), chosen = task();
|
||||
var first = tasks.claim();
|
||||
assertEquals(running, first.get("id"));
|
||||
queue.prioritize(user, chosen, 0, true, "urgent", "test");
|
||||
assertNull(tasks.claim());
|
||||
assertEquals("RUNNING", tasks.find(running, user).get("status"));
|
||||
finish(first);
|
||||
assertEquals(chosen, tasks.claim().get("id"));
|
||||
assertEquals("QUEUED", tasks.find(older, user).get("status"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void reducedConcurrencyAppliesWithoutKillingExistingTasks() {
|
||||
enable(2);
|
||||
task();
|
||||
task();
|
||||
UUID waiting = task();
|
||||
var a = tasks.claim();
|
||||
var b = tasks.claim();
|
||||
enable(1);
|
||||
assertNull(tasks.claim());
|
||||
finish(a);
|
||||
assertNull(tasks.claim());
|
||||
finish(b);
|
||||
assertEquals(waiting, tasks.claim().get("id"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void priorityPurchaseIsConcurrentIdempotentAndIndependentOfBudget() throws Exception {
|
||||
enable(2);
|
||||
UUID id = task();
|
||||
var pool = Executors.newFixedThreadPool(6);
|
||||
try {
|
||||
List<Future<Object>> calls = new ArrayList<>();
|
||||
for (int i = 0; i < 6; i++)
|
||||
calls.add(pool.submit(() -> queue.purchase(user, id, n("2"), "same")));
|
||||
for (var call : calls) call.get(15, TimeUnit.SECONDS);
|
||||
} finally {
|
||||
pool.shutdownNow();
|
||||
}
|
||||
assertEquals(
|
||||
1, users.db().queryForObject("SELECT COUNT(*) FROM queue_priority_orders", Integer.class));
|
||||
assertEquals(
|
||||
1,
|
||||
users
|
||||
.db()
|
||||
.queryForObject(
|
||||
"SELECT COUNT(*) FROM wallet_ledger WHERE kind='PRIORITY_PURCHASE'",
|
||||
Integer.class));
|
||||
assertEquals(0, n("98").compareTo((BigDecimal) wallets.get(user).get("balance")));
|
||||
assertEquals(0, n("2").compareTo((BigDecimal) wallets.get(user).get("reserved")));
|
||||
}
|
||||
|
||||
@Test
|
||||
void cancellationBeforeFirstExecutionRefundsExactlyOnce() {
|
||||
enable(1);
|
||||
UUID id = task();
|
||||
queue.purchase(user, id, n("2"), "paid");
|
||||
tasks.action(id, user, "pause");
|
||||
tasks.action(id, user, "cancel");
|
||||
queue.refundUnused(id);
|
||||
assertEquals(0, n("100").compareTo((BigDecimal) wallets.get(user).get("balance")));
|
||||
assertEquals(0, BigDecimal.ZERO.compareTo((BigDecimal) wallets.get(user).get("reserved")));
|
||||
assertEquals(
|
||||
1,
|
||||
users
|
||||
.db()
|
||||
.queryForObject(
|
||||
"SELECT COUNT(*) FROM wallet_ledger WHERE kind='PRIORITY_REFUND'", Integer.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void firstExecutionConsumesPriorityAndLaterCancellationDoesNotRefund() {
|
||||
enable(1);
|
||||
UUID id = task();
|
||||
queue.purchase(user, id, n("2"), "paid");
|
||||
var claim = tasks.claim();
|
||||
assertEquals("", tasks.beginExecution(id, claim.get("lease_token").toString()));
|
||||
tasks.action(id, user, "cancel");
|
||||
tasks.finish(id, claim.get("lease_token").toString(), "CANCELLED", "test", true);
|
||||
assertEquals(
|
||||
"CONSUMED",
|
||||
users.db().queryForObject("SELECT status FROM queue_priority_orders", String.class));
|
||||
assertEquals(0, n("98").compareTo((BigDecimal) wallets.get(user).get("balance")));
|
||||
}
|
||||
|
||||
@Test
|
||||
void cancellationBetweenClaimAndStartStillRefunds() {
|
||||
enable(1);
|
||||
UUID id = task();
|
||||
queue.purchase(user, id, n("2"), "paid");
|
||||
var claim = tasks.claim();
|
||||
tasks.action(id, user, "cancel");
|
||||
assertEquals("CANCEL", tasks.beginExecution(id, claim.get("lease_token").toString()));
|
||||
tasks.finish(id, claim.get("lease_token").toString(), "CANCELLED", "test", true);
|
||||
assertEquals(
|
||||
"REFUNDED",
|
||||
users.db().queryForObject("SELECT status FROM queue_priority_orders", String.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void stalePriceOfflineWorkerAndWrongOwnerCannotCharge() {
|
||||
enable(1);
|
||||
UUID id = task();
|
||||
UUID other = users.create("other-queue@test.com", "Other", "StrongPassword123", "test");
|
||||
assertThrows(ApiError.Failure.class, () -> queue.purchase(other, id, n("2"), "foreign"));
|
||||
assertThrows(ApiError.Failure.class, () -> queue.purchase(user, id, n("1"), "old-price"));
|
||||
users.db().update("DELETE FROM worker_presence");
|
||||
assertThrows(ApiError.Failure.class, () -> queue.purchase(user, id, n("2"), "offline"));
|
||||
assertThrows(
|
||||
ApiError.Failure.class, () -> queue.prioritize(user, id, 100, true, "test", "test"));
|
||||
assertEquals(0, n("100").compareTo((BigDecimal) wallets.get(user).get("balance")));
|
||||
assertEquals(
|
||||
0, users.db().queryForObject("SELECT COUNT(*) FROM queue_priority_orders", Integer.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void ambiguousExpiredTaskKeepsSlotUntilReconciled() {
|
||||
enable(1);
|
||||
UUID first = task(), second = task();
|
||||
tasks.claim();
|
||||
users
|
||||
.db()
|
||||
.update(
|
||||
"UPDATE ai_tasks SET lease_expires_at=? WHERE id=?",
|
||||
Timestamp.from(Instant.now().minusSeconds(1)),
|
||||
first);
|
||||
assertNull(tasks.claim());
|
||||
assertEquals("REVIEW_REQUIRED", tasks.find(first, user).get("status"));
|
||||
reconciliation.resolve(first, user, BigDecimal.ZERO, "confirmed no extra charge", "test");
|
||||
assertEquals(second, tasks.claim().get("id"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void settingsAndDispatchArePermissionProtectedAndPositionIsOwnOnly() throws Exception {
|
||||
UUID id = task();
|
||||
String token =
|
||||
users
|
||||
.login("queue@test.com", "StrongPassword123", "test", "test", 1)
|
||||
.get("token")
|
||||
.toString();
|
||||
UUID owner = user,
|
||||
other = users.create("other-position@test.com", "Other", "StrongPassword123", "test");
|
||||
wallets.adjust(other, n("10"), "credit", "test", owner, "test");
|
||||
user = other;
|
||||
task();
|
||||
user = owner;
|
||||
mvc.perform(
|
||||
put("/api/admin/queue/config")
|
||||
.header("Authorization", "Bearer " + token)
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.content(
|
||||
"{\"maxConcurrent\":3,\"priorityEnabled\":true,\"priorityPrice\":2,\"reason\":\"test\"}"))
|
||||
.andExpect(status().isForbidden());
|
||||
mvc.perform(get("/api/queue").header("Authorization", "Bearer " + token))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.positions[0].position").value(1))
|
||||
.andExpect(jsonPath("$.positions.length()").value(1))
|
||||
.andExpect(jsonPath("$.queued").value(2));
|
||||
mvc.perform(
|
||||
post("/api/admin/tasks/" + id + "/priority")
|
||||
.header("Authorization", "Bearer " + token)
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.content("{\"priority\":10,\"runNext\":false,\"reason\":\"test\"}"))
|
||||
.andExpect(status().isForbidden());
|
||||
users.db().update("INSERT INTO user_roles VALUES (?,'OPS')", user);
|
||||
mvc.perform(
|
||||
put("/api/admin/queue/config")
|
||||
.header("Authorization", "Bearer " + token)
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.content(
|
||||
"{\"maxConcurrent\":3,\"priorityEnabled\":true,\"priorityPrice\":2,\"reason\":\"test\"}"))
|
||||
.andExpect(status().isOk());
|
||||
assertEquals(
|
||||
1,
|
||||
users
|
||||
.db()
|
||||
.queryForObject(
|
||||
"SELECT COUNT(*) FROM admin_operation_logs WHERE action='QUEUE_CONFIG'",
|
||||
Integer.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void oneWorkerActuallyExecutesTwoProjectsInParallel() throws Exception {
|
||||
enable(2);
|
||||
task();
|
||||
task();
|
||||
var gateway = mock(ModelGateway.class);
|
||||
var sandbox = mock(SandboxClient.class);
|
||||
when(sandbox.inspect(any()))
|
||||
.thenReturn(json.createObjectNode().put("sourceFiles", 1).put("complete", true));
|
||||
var vault = mock(SecretVault.class);
|
||||
when(vault.decrypt(anyString())).thenReturn("fixture-token");
|
||||
var entered = new CountDownLatch(2);
|
||||
var release = new CountDownLatch(1);
|
||||
when(gateway.call(
|
||||
anyString(),
|
||||
anyString(),
|
||||
anyString(),
|
||||
anyString(),
|
||||
anyInt(),
|
||||
anyInt(),
|
||||
anyList(),
|
||||
anyString()))
|
||||
.thenAnswer(
|
||||
call -> {
|
||||
entered.countDown();
|
||||
if (!release.await(10, TimeUnit.SECONDS))
|
||||
throw new IllegalStateException("test timeout");
|
||||
return new ModelGateway.Result(
|
||||
json.createObjectNode().put("role", "assistant").put("content", "done"),
|
||||
List.of(),
|
||||
new ModelGateway.Usage(10, 2, 0, 0, true, json.createObjectNode()),
|
||||
UUID.randomUUID().toString());
|
||||
});
|
||||
var worker = new AgentWorker(tasks, gateway, sandbox, vault, json);
|
||||
try {
|
||||
worker.poll();
|
||||
assertTrue(entered.await(5, TimeUnit.SECONDS), "Both model calls must overlap");
|
||||
assertEquals(2, worker.activeTasks());
|
||||
assertEquals(2L, queue.occupied());
|
||||
release.countDown();
|
||||
long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10);
|
||||
while (worker.activeTasks() > 0 && System.nanoTime() < deadline) Thread.sleep(10);
|
||||
assertEquals(0, worker.activeTasks());
|
||||
assertEquals(
|
||||
2,
|
||||
users
|
||||
.db()
|
||||
.queryForObject(
|
||||
"SELECT COUNT(*) FROM ai_tasks WHERE status='COMPLETED'", Integer.class));
|
||||
} finally {
|
||||
release.countDown();
|
||||
worker.stop();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void buyingAndClaimingSameTaskCannotDoubleChargeOrLoseRefund() throws Exception {
|
||||
enable(1);
|
||||
UUID id = task();
|
||||
var pool = Executors.newFixedThreadPool(2);
|
||||
var start = new CountDownLatch(1);
|
||||
try {
|
||||
var buy =
|
||||
pool.submit(
|
||||
() -> {
|
||||
start.await();
|
||||
try {
|
||||
queue.purchase(user, id, n("2"), "raced");
|
||||
return true;
|
||||
} catch (ApiError.Failure conflict) {
|
||||
assertEquals(409, conflict.status);
|
||||
return false;
|
||||
}
|
||||
});
|
||||
var claim =
|
||||
pool.submit(
|
||||
() -> {
|
||||
start.await();
|
||||
return tasks.claim();
|
||||
});
|
||||
start.countDown();
|
||||
buy.get(10, TimeUnit.SECONDS);
|
||||
var received = claim.get(10, TimeUnit.SECONDS);
|
||||
assertNotNull(received);
|
||||
tasks.action(id, user, "cancel");
|
||||
assertEquals("CANCEL", tasks.beginExecution(id, received.get("lease_token").toString()));
|
||||
tasks.finish(id, received.get("lease_token").toString(), "CANCELLED", "test", true);
|
||||
assertEquals(0, n("100").compareTo((BigDecimal) wallets.get(user).get("balance")));
|
||||
assertEquals(
|
||||
0,
|
||||
users
|
||||
.db()
|
||||
.queryForObject(
|
||||
"SELECT COUNT(*) FROM queue_priority_orders WHERE status<>'REFUNDED'",
|
||||
Integer.class));
|
||||
} finally {
|
||||
pool.shutdownNow();
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user