440 lines
16 KiB
Java
440 lines
16 KiB
Java
package run.webup.platform;
|
|
|
|
import static org.junit.jupiter.api.Assertions.*;
|
|
import static org.mockito.Mockito.*;
|
|
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.*;
|
|
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*;
|
|
|
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
import java.math.BigDecimal;
|
|
import java.sql.Timestamp;
|
|
import java.time.Instant;
|
|
import java.util.*;
|
|
import java.util.concurrent.*;
|
|
import org.junit.jupiter.api.*;
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc;
|
|
import org.springframework.boot.test.context.SpringBootTest;
|
|
import org.springframework.http.MediaType;
|
|
import org.springframework.test.context.ActiveProfiles;
|
|
import org.springframework.test.context.bean.override.mockito.MockitoBean;
|
|
import org.springframework.test.web.servlet.MockMvc;
|
|
|
|
@SpringBootTest
|
|
@AutoConfigureMockMvc
|
|
@ActiveProfiles("test")
|
|
class QueueIntegrationTest {
|
|
@Autowired IdentityService users;
|
|
@Autowired TaskService tasks;
|
|
@Autowired QueueService queue;
|
|
@Autowired WalletService wallets;
|
|
@Autowired ObjectMapper json;
|
|
@Autowired ReconciliationService reconciliation;
|
|
@Autowired MockMvc mvc;
|
|
@MockitoBean EphemeralStore ephemeral;
|
|
UUID user, model;
|
|
int counter;
|
|
|
|
BigDecimal n(String value) {
|
|
return new BigDecimal(value);
|
|
}
|
|
|
|
@BeforeEach
|
|
void setup() {
|
|
TestDatabase.clear(users.db());
|
|
counter = 0;
|
|
user = users.create("queue@test.com", "Queue", "StrongPassword123", "test");
|
|
model = UUID.randomUUID();
|
|
UUID provider = UUID.randomUUID(), price = UUID.randomUUID();
|
|
users
|
|
.db()
|
|
.update(
|
|
"INSERT INTO"
|
|
+ " ai_providers(id,name,base_url,api_type,encrypted_key,enabled,timeout_seconds)"
|
|
+ " VALUES (?,'Fixture','https://api.openai.com/v1','OPENAI','fixture',TRUE,30)",
|
|
provider);
|
|
users
|
|
.db()
|
|
.update(
|
|
"INSERT INTO"
|
|
+ " ai_models(id,provider_id,name,model_id,context_length,max_output,tools_enabled,vision_enabled,enabled)"
|
|
+ " VALUES (?,?,'Fixture','fixture',8192,256,TRUE,FALSE,TRUE)",
|
|
model,
|
|
provider);
|
|
users
|
|
.db()
|
|
.update(
|
|
"INSERT INTO"
|
|
+ " ai_model_price_versions(id,model_id,input_price,output_price,cache_read_price,cache_write_price)"
|
|
+ " VALUES (?,?,1,2,0,0)",
|
|
price,
|
|
model);
|
|
users.db().update("UPDATE ai_models SET current_price_version=? WHERE id=?", price, model);
|
|
wallets.adjust(user, n("100"), "credit", "test", user, "test");
|
|
users
|
|
.db()
|
|
.update(
|
|
"INSERT INTO worker_presence(instance_id,kind,status,capacity) VALUES"
|
|
+ " (?,'AI','READY',8)",
|
|
UUID.randomUUID());
|
|
}
|
|
|
|
UUID task() {
|
|
UUID project = UUID.randomUUID();
|
|
users
|
|
.db()
|
|
.update(
|
|
"INSERT INTO projects(id,owner_id,name,description,stack,requirements) VALUES"
|
|
+ " (?,?,'Queue','','HTML','')",
|
|
project,
|
|
user);
|
|
UUID id =
|
|
(UUID) tasks.create(user, project, model, "Build", n("2"), "task-" + counter).get("id");
|
|
users
|
|
.db()
|
|
.update(
|
|
"UPDATE ai_tasks SET created_at=? WHERE id=?",
|
|
Timestamp.from(Instant.parse("2026-01-01T00:00:00Z").plusSeconds(counter++)),
|
|
id);
|
|
return id;
|
|
}
|
|
|
|
void enable(int max) {
|
|
queue.configure(user, max, true, n("2"), "test", "test");
|
|
}
|
|
|
|
void finish(Map<String, Object> task) {
|
|
tasks.finish(
|
|
(UUID) task.get("id"), task.get("lease_token").toString(), "COMPLETED", "test", true);
|
|
}
|
|
|
|
@Test
|
|
void simultaneousClaimsNeverExceedGlobalConcurrency() throws Exception {
|
|
for (int i = 0; i < 8; i++) task();
|
|
enable(2);
|
|
var pool = Executors.newFixedThreadPool(8);
|
|
try {
|
|
var start = new CountDownLatch(1);
|
|
List<Future<Map<String, Object>>> calls = new ArrayList<>();
|
|
for (int i = 0; i < 8; i++)
|
|
calls.add(
|
|
pool.submit(
|
|
() -> {
|
|
start.await();
|
|
return tasks.claim();
|
|
}));
|
|
start.countDown();
|
|
Set<UUID> claimed = new HashSet<>();
|
|
for (var call : calls) {
|
|
var value = call.get(15, TimeUnit.SECONDS);
|
|
if (value != null) claimed.add((UUID) value.get("id"));
|
|
}
|
|
assertEquals(2, claimed.size());
|
|
assertEquals(2L, queue.occupied());
|
|
} finally {
|
|
pool.shutdownNow();
|
|
}
|
|
}
|
|
|
|
@Test
|
|
void administratorWeightPaidPriorityAndFifoUseOneOrder() {
|
|
enable(1);
|
|
UUID ordinary = task(), paid = task(), manual = task();
|
|
queue.purchase(user, paid, n("2"), "paid");
|
|
queue.prioritize(user, manual, 10, false, "test", "test");
|
|
var first = tasks.claim();
|
|
assertEquals(manual, first.get("id"));
|
|
finish(first);
|
|
var second = tasks.claim();
|
|
assertEquals(paid, second.get("id"));
|
|
tasks.beginExecution(paid, second.get("lease_token").toString());
|
|
finish(second);
|
|
assertEquals(ordinary, tasks.claim().get("id"));
|
|
}
|
|
|
|
@Test
|
|
void runNextDoesNotInterruptRunningOrBypassCap() {
|
|
enable(1);
|
|
UUID running = task(), older = task(), chosen = task();
|
|
var first = tasks.claim();
|
|
assertEquals(running, first.get("id"));
|
|
queue.prioritize(user, chosen, 0, true, "urgent", "test");
|
|
assertNull(tasks.claim());
|
|
assertEquals("RUNNING", tasks.find(running, user).get("status"));
|
|
finish(first);
|
|
assertEquals(chosen, tasks.claim().get("id"));
|
|
assertEquals("QUEUED", tasks.find(older, user).get("status"));
|
|
}
|
|
|
|
@Test
|
|
void reducedConcurrencyAppliesWithoutKillingExistingTasks() {
|
|
enable(2);
|
|
task();
|
|
task();
|
|
UUID waiting = task();
|
|
var a = tasks.claim();
|
|
var b = tasks.claim();
|
|
enable(1);
|
|
assertNull(tasks.claim());
|
|
finish(a);
|
|
assertNull(tasks.claim());
|
|
finish(b);
|
|
assertEquals(waiting, tasks.claim().get("id"));
|
|
}
|
|
|
|
@Test
|
|
void priorityPurchaseIsConcurrentIdempotentAndIndependentOfBudget() throws Exception {
|
|
enable(2);
|
|
UUID id = task();
|
|
var pool = Executors.newFixedThreadPool(6);
|
|
try {
|
|
List<Future<Object>> calls = new ArrayList<>();
|
|
for (int i = 0; i < 6; i++)
|
|
calls.add(pool.submit(() -> queue.purchase(user, id, n("2"), "same")));
|
|
for (var call : calls) call.get(15, TimeUnit.SECONDS);
|
|
} finally {
|
|
pool.shutdownNow();
|
|
}
|
|
assertEquals(
|
|
1, users.db().queryForObject("SELECT COUNT(*) FROM queue_priority_orders", Integer.class));
|
|
assertEquals(
|
|
1,
|
|
users
|
|
.db()
|
|
.queryForObject(
|
|
"SELECT COUNT(*) FROM wallet_ledger WHERE kind='PRIORITY_PURCHASE'",
|
|
Integer.class));
|
|
assertEquals(0, n("98").compareTo((BigDecimal) wallets.get(user).get("balance")));
|
|
assertEquals(0, n("2").compareTo((BigDecimal) wallets.get(user).get("reserved")));
|
|
}
|
|
|
|
@Test
|
|
void cancellationBeforeFirstExecutionRefundsExactlyOnce() {
|
|
enable(1);
|
|
UUID id = task();
|
|
queue.purchase(user, id, n("2"), "paid");
|
|
tasks.action(id, user, "pause");
|
|
tasks.action(id, user, "cancel");
|
|
queue.refundUnused(id);
|
|
assertEquals(0, n("100").compareTo((BigDecimal) wallets.get(user).get("balance")));
|
|
assertEquals(0, BigDecimal.ZERO.compareTo((BigDecimal) wallets.get(user).get("reserved")));
|
|
assertEquals(
|
|
1,
|
|
users
|
|
.db()
|
|
.queryForObject(
|
|
"SELECT COUNT(*) FROM wallet_ledger WHERE kind='PRIORITY_REFUND'", Integer.class));
|
|
}
|
|
|
|
@Test
|
|
void firstExecutionConsumesPriorityAndLaterCancellationDoesNotRefund() {
|
|
enable(1);
|
|
UUID id = task();
|
|
queue.purchase(user, id, n("2"), "paid");
|
|
var claim = tasks.claim();
|
|
assertEquals("", tasks.beginExecution(id, claim.get("lease_token").toString()));
|
|
tasks.action(id, user, "cancel");
|
|
tasks.finish(id, claim.get("lease_token").toString(), "CANCELLED", "test", true);
|
|
assertEquals(
|
|
"CONSUMED",
|
|
users.db().queryForObject("SELECT status FROM queue_priority_orders", String.class));
|
|
assertEquals(0, n("98").compareTo((BigDecimal) wallets.get(user).get("balance")));
|
|
}
|
|
|
|
@Test
|
|
void cancellationBetweenClaimAndStartStillRefunds() {
|
|
enable(1);
|
|
UUID id = task();
|
|
queue.purchase(user, id, n("2"), "paid");
|
|
var claim = tasks.claim();
|
|
tasks.action(id, user, "cancel");
|
|
assertEquals("CANCEL", tasks.beginExecution(id, claim.get("lease_token").toString()));
|
|
tasks.finish(id, claim.get("lease_token").toString(), "CANCELLED", "test", true);
|
|
assertEquals(
|
|
"REFUNDED",
|
|
users.db().queryForObject("SELECT status FROM queue_priority_orders", String.class));
|
|
}
|
|
|
|
@Test
|
|
void stalePriceOfflineWorkerAndWrongOwnerCannotCharge() {
|
|
enable(1);
|
|
UUID id = task();
|
|
UUID other = users.create("other-queue@test.com", "Other", "StrongPassword123", "test");
|
|
assertThrows(ApiError.Failure.class, () -> queue.purchase(other, id, n("2"), "foreign"));
|
|
assertThrows(ApiError.Failure.class, () -> queue.purchase(user, id, n("1"), "old-price"));
|
|
users.db().update("DELETE FROM worker_presence");
|
|
assertThrows(ApiError.Failure.class, () -> queue.purchase(user, id, n("2"), "offline"));
|
|
assertThrows(
|
|
ApiError.Failure.class, () -> queue.prioritize(user, id, 100, true, "test", "test"));
|
|
assertEquals(0, n("100").compareTo((BigDecimal) wallets.get(user).get("balance")));
|
|
assertEquals(
|
|
0, users.db().queryForObject("SELECT COUNT(*) FROM queue_priority_orders", Integer.class));
|
|
}
|
|
|
|
@Test
|
|
void ambiguousExpiredTaskKeepsSlotUntilReconciled() {
|
|
enable(1);
|
|
UUID first = task(), second = task();
|
|
tasks.claim();
|
|
users
|
|
.db()
|
|
.update(
|
|
"UPDATE ai_tasks SET lease_expires_at=? WHERE id=?",
|
|
Timestamp.from(Instant.now().minusSeconds(1)),
|
|
first);
|
|
assertNull(tasks.claim());
|
|
assertEquals("REVIEW_REQUIRED", tasks.find(first, user).get("status"));
|
|
reconciliation.resolve(first, user, BigDecimal.ZERO, "confirmed no extra charge", "test");
|
|
assertEquals(second, tasks.claim().get("id"));
|
|
}
|
|
|
|
@Test
|
|
void settingsAndDispatchArePermissionProtectedAndPositionIsOwnOnly() throws Exception {
|
|
UUID id = task();
|
|
String token =
|
|
users
|
|
.login("queue@test.com", "StrongPassword123", "test", "test", 1)
|
|
.get("token")
|
|
.toString();
|
|
UUID owner = user,
|
|
other = users.create("other-position@test.com", "Other", "StrongPassword123", "test");
|
|
wallets.adjust(other, n("10"), "credit", "test", owner, "test");
|
|
user = other;
|
|
task();
|
|
user = owner;
|
|
mvc.perform(
|
|
put("/api/admin/queue/config")
|
|
.header("Authorization", "Bearer " + token)
|
|
.contentType(MediaType.APPLICATION_JSON)
|
|
.content(
|
|
"{\"maxConcurrent\":3,\"priorityEnabled\":true,\"priorityPrice\":2,\"reason\":\"test\"}"))
|
|
.andExpect(status().isForbidden());
|
|
mvc.perform(get("/api/queue").header("Authorization", "Bearer " + token))
|
|
.andExpect(status().isOk())
|
|
.andExpect(jsonPath("$.positions[0].position").value(1))
|
|
.andExpect(jsonPath("$.positions.length()").value(1))
|
|
.andExpect(jsonPath("$.queued").value(2));
|
|
mvc.perform(
|
|
post("/api/admin/tasks/" + id + "/priority")
|
|
.header("Authorization", "Bearer " + token)
|
|
.contentType(MediaType.APPLICATION_JSON)
|
|
.content("{\"priority\":10,\"runNext\":false,\"reason\":\"test\"}"))
|
|
.andExpect(status().isForbidden());
|
|
users.db().update("INSERT INTO user_roles VALUES (?,'OPS')", user);
|
|
mvc.perform(
|
|
put("/api/admin/queue/config")
|
|
.header("Authorization", "Bearer " + token)
|
|
.contentType(MediaType.APPLICATION_JSON)
|
|
.content(
|
|
"{\"maxConcurrent\":3,\"priorityEnabled\":true,\"priorityPrice\":2,\"reason\":\"test\"}"))
|
|
.andExpect(status().isOk());
|
|
assertEquals(
|
|
1,
|
|
users
|
|
.db()
|
|
.queryForObject(
|
|
"SELECT COUNT(*) FROM admin_operation_logs WHERE action='QUEUE_CONFIG'",
|
|
Integer.class));
|
|
}
|
|
|
|
@Test
|
|
void oneWorkerActuallyExecutesTwoProjectsInParallel() throws Exception {
|
|
enable(2);
|
|
task();
|
|
task();
|
|
var gateway = mock(ModelGateway.class);
|
|
var sandbox = mock(SandboxClient.class);
|
|
when(sandbox.inspect(any()))
|
|
.thenReturn(json.createObjectNode().put("sourceFiles", 1).put("complete", true));
|
|
var vault = mock(SecretVault.class);
|
|
when(vault.decrypt(anyString())).thenReturn("fixture-token");
|
|
var entered = new CountDownLatch(2);
|
|
var release = new CountDownLatch(1);
|
|
when(gateway.call(
|
|
anyString(),
|
|
anyString(),
|
|
anyString(),
|
|
anyString(),
|
|
anyInt(),
|
|
anyInt(),
|
|
anyList(),
|
|
anyString()))
|
|
.thenAnswer(
|
|
call -> {
|
|
entered.countDown();
|
|
if (!release.await(10, TimeUnit.SECONDS))
|
|
throw new IllegalStateException("test timeout");
|
|
return new ModelGateway.Result(
|
|
json.createObjectNode().put("role", "assistant").put("content", "done"),
|
|
List.of(),
|
|
new ModelGateway.Usage(10, 2, 0, 0, true, json.createObjectNode()),
|
|
UUID.randomUUID().toString());
|
|
});
|
|
var worker = new AgentWorker(tasks, gateway, sandbox, vault, json);
|
|
try {
|
|
worker.poll();
|
|
assertTrue(entered.await(5, TimeUnit.SECONDS), "Both model calls must overlap");
|
|
assertEquals(2, worker.activeTasks());
|
|
assertEquals(2L, queue.occupied());
|
|
release.countDown();
|
|
long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10);
|
|
while (worker.activeTasks() > 0 && System.nanoTime() < deadline) Thread.sleep(10);
|
|
assertEquals(0, worker.activeTasks());
|
|
assertEquals(
|
|
2,
|
|
users
|
|
.db()
|
|
.queryForObject(
|
|
"SELECT COUNT(*) FROM ai_tasks WHERE status='COMPLETED'", Integer.class));
|
|
} finally {
|
|
release.countDown();
|
|
worker.stop();
|
|
}
|
|
}
|
|
|
|
@Test
|
|
void buyingAndClaimingSameTaskCannotDoubleChargeOrLoseRefund() throws Exception {
|
|
enable(1);
|
|
UUID id = task();
|
|
var pool = Executors.newFixedThreadPool(2);
|
|
var start = new CountDownLatch(1);
|
|
try {
|
|
var buy =
|
|
pool.submit(
|
|
() -> {
|
|
start.await();
|
|
try {
|
|
queue.purchase(user, id, n("2"), "raced");
|
|
return true;
|
|
} catch (ApiError.Failure conflict) {
|
|
assertEquals(409, conflict.status);
|
|
return false;
|
|
}
|
|
});
|
|
var claim =
|
|
pool.submit(
|
|
() -> {
|
|
start.await();
|
|
return tasks.claim();
|
|
});
|
|
start.countDown();
|
|
buy.get(10, TimeUnit.SECONDS);
|
|
var received = claim.get(10, TimeUnit.SECONDS);
|
|
assertNotNull(received);
|
|
tasks.action(id, user, "cancel");
|
|
assertEquals("CANCEL", tasks.beginExecution(id, received.get("lease_token").toString()));
|
|
tasks.finish(id, received.get("lease_token").toString(), "CANCELLED", "test", true);
|
|
assertEquals(0, n("100").compareTo((BigDecimal) wallets.get(user).get("balance")));
|
|
assertEquals(
|
|
0,
|
|
users
|
|
.db()
|
|
.queryForObject(
|
|
"SELECT COUNT(*) FROM queue_priority_orders WHERE status<>'REFUNDED'",
|
|
Integer.class));
|
|
} finally {
|
|
pool.shutdownNow();
|
|
}
|
|
}
|
|
}
|