Try to increase throughput

deno_server.ts

@@ -23,21 +23,35 @@ for (let i = 0; i < numWorkers; i++) {
   workerPool.push({ worker, inUse: false });
 }
 
+const requestQueue: ((
+  value:
+    | { worker: Worker; inUse: boolean }
+    | PromiseLike<{ worker: Worker; inUse: boolean }>
+) => void)[] = [];
+
 function getAvailableWorker() {
   return new Promise((resolve) => {
-    const check = () => {
-      const availableWorker = workerPool.find((w) => !w.inUse);
-      if (availableWorker) {
-        availableWorker.inUse = true;
-        resolve(availableWorker);
-      } else {
-        setTimeout(check, 100); // Check again in 100ms.
-      }
-    };
-    check();
+    const availableWorker = workerPool.find((w) => !w.inUse);
+    if (availableWorker) {
+      availableWorker.inUse = true;
+      resolve(availableWorker);
+    } else {
+      requestQueue.push(resolve);
+    }
   });
 }
 
+function releaseWorker(worker) {
+  if (requestQueue.length > 0) {
+    const nextRequest = requestQueue.shift();
+    if (nextRequest) {
+      nextRequest(worker);
+    }
+  } else {
+    worker.inUse = false;
+  }
+}
+
 async function handler(req: Request): Promise<Response> {
   if (req.method !== "POST" || new URL(req.url).pathname !== "/execute") {
     return new Response("Not Found", { status: 404 });
@@ -95,7 +109,7 @@ async function handler(req: Request): Promise<Response> {
   } catch (e) {
     return new Response(`Bad Request: ${e.message}`, { status: 400 });
   } finally {
-    availableWorker.inUse = false; // Release the worker.
+    releaseWorker(availableWorker); // Release the worker.
   }
 }