Skip to content

Commit ab9cdd5

Browse files
KevithMannageKevithMannage
committed
modification to compile engine
1 parent 8c30e85 commit ab9cdd5

File tree

6 files changed

+323
-88
lines changed

6 files changed

+323
-88
lines changed

‎Compiler_Engine/compose.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,3 +16,7 @@ services:
1616
depends_on:
1717
- redis
1818

19+
worker-py:
20+
build: ./worker-py
21+
depends_on:
22+
- redis

‎Compiler_Engine/producer/producer.js‎

Lines changed: 134 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -1,65 +1,164 @@
1+
// import express from 'express';
2+
// import bodyParser from 'body-parser';
3+
// import { Queue, QueueEvents } from 'bullmq';
4+
// import IORedis from 'ioredis';
5+
// import dotenv from 'dotenv';
6+
7+
// dotenv.config();
8+
9+
// const PORT = process.env.PORT | 4000;
10+
// const SUPPORTED_LANGUAGES = ['javascript', 'python'];
11+
12+
// const app = express();
13+
// app.use(bodyParser.json());
14+
15+
16+
// const redisHost = process.env.REDIS_HOST || 'redis';
17+
// const redisPort = process.env.REDIS_PORT || 6379;
18+
// const connection = new IORedis(`redis://${redisHost}:${redisPort}`, {
19+
// maxRetriesPerRequest: null
20+
// });
21+
22+
// const queue = new Queue('code-execution', { connection });
23+
// const queueEvents = new QueueEvents('code-execution', { connection });
24+
// await queueEvents.waitUntilReady();
25+
26+
// //HAVE TO THINK ABOUT THIS BECAUSE PYTHON INTEGRATION IS HARD
27+
28+
// // Dynamically get supported languges from Redis
29+
// // const keys = await connection.keys('lang:*');
30+
// // const activeLanguages = keys.map(k => k.split(':')[1]);
31+
32+
// app.post('/run', async (req, res) => {
33+
// const { language, code, input } = req.body;
34+
35+
// if (!SUPPORTED_LANGUAGES.includes(language)) {
36+
// return res.status(400).json({
37+
// success: false,
38+
// error: `Language "${language}" not supported currently. Available: ${SUPPORTED_LANGUAGES.join(', ')}`
39+
// });
40+
// }
41+
42+
// let jobName;
43+
// switch (language) {
44+
// case 'python':
45+
// jobName = 'runpy';
46+
// break;
47+
// case 'javascript':
48+
// jobName = 'runjs';
49+
// break;
50+
// default:
51+
// break;
52+
// }
53+
54+
// const job = await queue.add(jobName, { language, code, input });
55+
56+
// try {
57+
// const result = await job.waitUntilFinished(queueEvents, 10000);
58+
// res.send(result);
59+
// }
60+
// catch (error) {
61+
// res.status(500).json({ success: false, error: error.message || 'Job failed' });
62+
// }
63+
// });
64+
65+
// app.listen(PORT, () => console.log(`Producer running on port ${PORT}`));
66+
67+
68+
69+
// import express from 'express';
70+
// import { Queue, QueueEvents } from 'bullmq';
71+
// import IORedis from 'ioredis';
72+
// import dotenv from 'dotenv';
73+
// dotenv.config();
74+
75+
// const app = express();
76+
// app.use(express.json());
77+
78+
// // Redis connection for BullMQ (maxRetriesPerRequest must be null)
79+
// const connection = new IORedis('redis://redis:6379', {
80+
// maxRetriesPerRequest: null
81+
// });
82+
83+
// const queue = new Queue('code-execution', { connection });
84+
// const queueEvents = new QueueEvents('code-execution', { connection });
85+
86+
// // API endpoint
87+
// app.post('/run', async (req, res) => {
88+
// const { code, input, language } = req.body;
89+
// try {
90+
// const job = await queue.add('runjs', { code, input, language });
91+
// const result = await job.waitUntilFinished(queueEvents);
92+
// res.json({ jobId: job.id, result });
93+
// } catch (err) {
94+
// res.status(500).json({ error: err.message });
95+
// }
96+
// });
97+
98+
// app.listen(process.env.PORT || 4000, () => {
99+
// console.log('Producer running on port 4000');
100+
// });
101+
102+
1103
import express from 'express';
2-
import bodyParser from 'body-parser';
3104
import { Queue, QueueEvents } from 'bullmq';
4105
import IORedis from 'ioredis';
5106
import dotenv from 'dotenv';
6-
7107
dotenv.config();
8108

9-
const PORT = process.env.PORT | 4000;
10-
const SUPPORTED_LANGUAGES = ['javascript', 'python'];
11-
12109
const app = express();
13-
app.use(bodyParser.json());
110+
app.use(express.json());
14111

112+
const PORT = process.env.PORT || 4000;
15113

16-
const redisHost = process.env.REDIS_HOST || 'localhost';
17-
const redisPort = process.env.REDIS_PORT || 6379;
18-
const connection = new IORedis(`redis://${redisHost}:${redisPort}`, {
114+
// Supported languages
115+
const SUPPORTED_LANGUAGES = ['javascript', 'python'];
116+
117+
// Redis connection
118+
const connection = new IORedis('redis://redis:6379', {
19119
maxRetriesPerRequest: null
20120
});
21121

122+
// BullMQ queue & events
22123
const queue = new Queue('code-execution', { connection });
23124
const queueEvents = new QueueEvents('code-execution', { connection });
24125
await queueEvents.waitUntilReady();
25126

26-
//HAVE TO THINK ABOUT THIS BECAUSE PYTHON INTEGRATION IS HARD
27-
28-
// Dynamically get supported languges from Redis
29-
// const keys = await connection.keys('lang:*');
30-
// const activeLanguages = keys.map(k => k.split(':')[1]);
31-
127+
// POST /run endpoint
32128
app.post('/run', async (req, res) => {
33129
const { language, code, input } = req.body;
34130

131+
// Validate language
35132
if (!SUPPORTED_LANGUAGES.includes(language)) {
36133
return res.status(400).json({
37134
success: false,
38-
error: `Language "${language}" not supported currently. Available: ${SUPPORTED_LANGUAGES.join(', ')}`
135+
error: `Language "${language}" not supported. Available: ${SUPPORTED_LANGUAGES.join(', ')}`
39136
});
40137
}
41138

42-
let jobName;
43-
switch (language) {
44-
case 'python':
45-
jobName = 'runpy';
46-
break;
47-
case 'javascript':
48-
jobName = 'runjs';
49-
break;
50-
default:
51-
break;
52-
}
53-
54-
const job = await queue.add(jobName, { language, code, input });
139+
// Determine job name based on language
140+
const jobName = language === 'python' ? 'runpy' : 'runjs';
55141

56142
try {
57-
const result = await job.waitUntilFinished(queueEvents, 10000);
58-
res.send(result);
59-
}
60-
catch (error) {
61-
res.status(500).json({ success: false, error: error.message || 'Job failed' });
143+
// Add job to queue
144+
const job = await queue.add(jobName, { language, code, input });
145+
146+
// Wait for worker to finish job
147+
const result = await job.waitUntilFinished(queueEvents, 10000); // 10 sec timeout
148+
149+
res.json({
150+
success: true,
151+
jobId: job.id,
152+
result
153+
});
154+
} catch (err) {
155+
res.status(500).json({
156+
success: false,
157+
error: err.message || 'Job execution failed'
158+
});
62159
}
63160
});
64161

65-
app.listen(PORT, () => console.log(`Producer running on port ${PORT}`));
162+
app.listen(PORT, () => {
163+
console.log(`Producer running on port ${PORT}`);
164+
});

‎Compiler_Engine/worker-js/dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,4 +4,4 @@ COPY . .
44
COPY package*.json ./
55

66
RUN npm install
7-
CMD ["node", "worker.js"]
7+
CMD ["node", "index.js"]

‎Compiler_Engine/worker-js/index.js‎

Lines changed: 111 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,41 +1,128 @@
1+
// import { Worker } from 'bullmq';
2+
// import IORedis from 'ioredis';
3+
4+
// const LANGUAGE = 'javascript';
5+
// const QUEUE_NAME = 'code-execution';
6+
7+
// const connection = new IORedis('redis://redis:6379', {
8+
// maxRetriesPerRequest: null,
9+
// });
10+
11+
// //HAVE TO THINK ABOUT THIS BECAUSE PYTHON INTEGRATION IS HARD
12+
13+
// // Register this worker's language in Redis with TTL expires in 30s, rerun every 20s
14+
// // This is used by the producer to know which workers are available for this language
15+
16+
// // const registerLanguage = async () => {
17+
// // await connection.set(`lang:${LANGUAGE}`, '1', 'EX', 30);
18+
// // };
19+
// // setInterval(registerLanguage, 20000);
20+
// // await registerLanguage();
21+
22+
// new Worker(
23+
// QUEUE_NAME,
24+
// {
25+
// "runjs": async job => {
26+
// console.log(`Processing job ${job.data.id} for language: ${job.data.language}`);
27+
// if (job.data.language !== LANGUAGE) return;
28+
29+
// return new Promise((resolve) => {
30+
// try {
31+
// const fn = new Function('input', job.data.code);
32+
// const output = fn(job.data.input);
33+
// resolve({success: true, output: output.toString()});
34+
// } catch (err) {
35+
// resolve({ success: false, error: err.message });
36+
// }
37+
// });
38+
// }
39+
// },
40+
// { connection }
41+
// );
42+
43+
// import { Worker } from 'bullmq';
44+
// import IORedis from 'ioredis';
45+
46+
// const LANGUAGE = 'javascript,python';
47+
// const QUEUE_NAME = 'code-execution';
48+
49+
// const connection = new IORedis('redis://redis:6379', {
50+
// maxRetriesPerRequest: null,
51+
// });
52+
53+
// new Worker(
54+
// QUEUE_NAME,
55+
// async (job) => {
56+
// console.log(`Processing job ${job.id} for language: ${job.data.language}`);
57+
58+
// if (job.data.language !== LANGUAGE) {
59+
// return { success: false, error: `Unsupported language: ${job.data.language}` };
60+
// }
61+
62+
// try {
63+
// const fn = new Function('input', job.data.code);
64+
// const output = fn(job.data.input);
65+
// return { success: true, output: output?.toString() };
66+
// } catch (err) {
67+
// return { success: false, error: err.message };
68+
// }
69+
// },
70+
// { connection }
71+
// );
72+
73+
174
import { Worker } from 'bullmq';
275
import IORedis from 'ioredis';
76+
import fs from 'fs';
77+
import { spawnSync } from 'child_process';
78+
import { randomUUID } from 'crypto';
379

4-
const LANGUAGE = 'javascript';
80+
const SUPPORTED_LANGUAGES = ['javascript', 'python'];
581
const QUEUE_NAME = 'code-execution';
682

783
const connection = new IORedis('redis://redis:6379', {
884
maxRetriesPerRequest: null,
985
});
1086

11-
//HAVE TO THINK ABOUT THIS BECAUSE PYTHON INTEGRATION IS HARD
87+
// Helper: Run Python code
88+
function runPython(code, input) {
89+
const filename = `temp_${randomUUID()}.py`;
90+
fs.writeFileSync(filename, code);
91+
const result = spawnSync('python3', [filename], {
92+
input: input || '',
93+
encoding: 'utf-8',
94+
timeout: 10000
95+
});
96+
fs.unlinkSync(filename);
1297

13-
// Register this worker's language in Redis with TTL expires in 30s, rerun every 20s
14-
// This is used by the producer to know which workers are available for this language
98+
if (result.error) return { success: false, error: result.error.message };
99+
if (result.status !== 0) return { success: false, error: result.stderr.trim() || 'Non-zero exit code' };
100+
return { success: true, output: result.stdout.trim() };
101+
}
15102

16-
// const registerLanguage = async () => {
17-
// await connection.set(`lang:${LANGUAGE}`, '1', 'EX', 30);
18-
// };
19-
// setInterval(registerLanguage, 20000);
20-
// await registerLanguage();
103+
// Helper: Run JS code
104+
function runJS(code, input) {
105+
try {
106+
const fn = new Function('input', code);
107+
const output = fn(input);
108+
return { success: true, output: output?.toString() };
109+
} catch (err) {
110+
return { success: false, error: err.message };
111+
}
112+
}
21113

22114
new Worker(
23115
QUEUE_NAME,
24-
{
25-
"runjs": async job => {
26-
console.log(`Processing job ${job.data.id} for language: ${job.data.language}`);
27-
if (job.data.language !== LANGUAGE) return;
28-
29-
return new Promise((resolve) => {
30-
try {
31-
const fn = new Function('input', job.data.code);
32-
const output = fn(job.data.input);
33-
resolve({success: true, output: output.toString()});
34-
} catch (err) {
35-
resolve({ success: false, error: err.message });
36-
}
37-
});
116+
async (job) => {
117+
const { language, code, input } = job.data;
118+
console.log(`Processing job ${job.id} for language: ${language}`);
119+
120+
if (!SUPPORTED_LANGUAGES.includes(language)) {
121+
return { success: false, error: `Unsupported language: ${language}` };
38122
}
123+
124+
if (language === 'javascript') return runJS(code, input);
125+
if (language === 'python') return runPython(code, input);
39126
},
40127
{ connection }
41-
);
128+
);

‎Compiler_Engine/worker-py/Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,4 +14,4 @@ RUN npm install
1414

1515
COPY . .
1616

17-
CMD [ "npm", "start" ]
17+
CMD [ "node", "workerpy.js" ]

0 commit comments

Comments
 (0)