Port Comp_iler to Myriad (#8)
This commit is contained in:
1Computer1
parent
0dd7f992be
commit
c72376656c
GPG key ID:
4AEE18F83AFDEB23
91 changed files with 156 additions and 1140 deletions
|
|
@ -1,5 +1,5 @@
|
|||
const { AkairoClient, CommandHandler, ListenerHandler } = require('discord-akairo');
|
||||
const LanguageHandler = require('./LanguageHandler');
|
||||
const Myriad = require('./Myriad');
|
||||
const path = require('path');
|
||||
|
||||
class CompilerClient extends AkairoClient {
|
||||
|
|
@ -24,25 +24,19 @@ class CompilerClient extends AkairoClient {
|
|||
directory: path.join(__dirname, '../listeners')
|
||||
});
|
||||
|
||||
this.languageHandler = new LanguageHandler(this, {
|
||||
directory: path.join(__dirname, '../languages')
|
||||
});
|
||||
|
||||
this.myriad = new Myriad(config.myriad);
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
async start() {
|
||||
start() {
|
||||
this.commandHandler.useListenerHandler(this.listenerHandler);
|
||||
this.listenerHandler.setEmitters({
|
||||
commandHandler: this.commandHandler,
|
||||
listenerHandler: this.listenerHandler,
|
||||
languageHandler: this.languageHandler
|
||||
listenerHandler: this.listenerHandler
|
||||
});
|
||||
|
||||
this.commandHandler.loadAll();
|
||||
this.listenerHandler.loadAll();
|
||||
this.languageHandler.loadAll();
|
||||
await this.languageHandler.buildDocker();
|
||||
return this.login(this.config.token);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,24 +0,0 @@
|
|||
const { AkairoModule } = require('discord-akairo');
|
||||
|
||||
class Language extends AkairoModule {
|
||||
constructor(id, {
|
||||
category,
|
||||
name,
|
||||
aliases,
|
||||
loads = [id],
|
||||
options = {}
|
||||
} = {}) {
|
||||
super(id, { category });
|
||||
|
||||
this.name = name;
|
||||
this.aliases = aliases;
|
||||
this.loads = loads;
|
||||
this.options = options;
|
||||
}
|
||||
|
||||
runWith() {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = Language;
|
||||
|
|
@ -1,227 +0,0 @@
|
|||
const { AkairoHandler } = require('discord-akairo');
|
||||
const { Collection } = require('discord.js');
|
||||
const Language = require('./Language');
|
||||
const Queue = require('./Queue');
|
||||
const childProcess = require('child_process');
|
||||
const util = require('util');
|
||||
const path = require('path');
|
||||
|
||||
const exec = util.promisify(childProcess.exec);
|
||||
|
||||
class LanguageHandler extends AkairoHandler {
|
||||
constructor(client, {
|
||||
directory,
|
||||
classToHandle = Language,
|
||||
extensions = ['.js', '.ts'],
|
||||
automateCategories,
|
||||
loadFilter = filepath =>
|
||||
!this.client.config.languages || this.client.config.languages.includes(path.parse(filepath).name)
|
||||
}) {
|
||||
super(client, {
|
||||
directory,
|
||||
classToHandle,
|
||||
extensions,
|
||||
automateCategories,
|
||||
loadFilter
|
||||
});
|
||||
|
||||
this.aliases = new Collection();
|
||||
this.containers = new Collection();
|
||||
this.queues = new Collection();
|
||||
}
|
||||
|
||||
register(language, filepath) {
|
||||
super.register(language, filepath);
|
||||
|
||||
for (let alias of language.aliases) {
|
||||
const conflict = this.aliases.get(alias.toLowerCase());
|
||||
if (conflict) {
|
||||
throw new TypeError(`Alias conflict of ${alias} between ${language.id} and ${conflict}`);
|
||||
}
|
||||
|
||||
alias = alias.toLowerCase();
|
||||
this.aliases.set(alias, language.id);
|
||||
}
|
||||
}
|
||||
|
||||
deregister(language) {
|
||||
for (let alias of language.aliases) {
|
||||
alias = alias.toLowerCase();
|
||||
this.aliases.delete(alias);
|
||||
}
|
||||
|
||||
super.deregister(language);
|
||||
}
|
||||
|
||||
findLanguage(alias) {
|
||||
return this.modules.get(this.aliases.get(alias.toLowerCase()));
|
||||
}
|
||||
|
||||
async buildDocker() {
|
||||
if (this.client.config.parallel) {
|
||||
await Promise.all(this.modules.map(({ loads }) => Promise.all(loads.map(dockerID => this.buildImage(dockerID)))));
|
||||
return;
|
||||
}
|
||||
|
||||
for (const { loads } of this.modules.values()) {
|
||||
for (const dockerID of loads) {
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
await this.buildImage(dockerID);
|
||||
}
|
||||
}
|
||||
|
||||
if (this.client.config.cleanup > 0) {
|
||||
setInterval(() => this.cleanup().catch(() => null), this.client.config.cleanup * 60 * 1000);
|
||||
}
|
||||
}
|
||||
|
||||
async buildImage(dockerID) {
|
||||
const folder = path.join(__dirname, '../../docker', dockerID);
|
||||
await util.promisify(childProcess.exec)(`docker build -t "1computer1/comp_iler:${dockerID}" ${folder}`);
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(`Built image 1computer1/comp_iler:${dockerID}.`);
|
||||
const concurrent = this.getCompilerConfig(dockerID, 'concurrent', 'number');
|
||||
this.queues.set(dockerID, { evalQueue: new Queue(concurrent), setupQueue: new Queue(1) });
|
||||
if (this.client.config.prepare) {
|
||||
await this.setupContainer(dockerID);
|
||||
}
|
||||
}
|
||||
|
||||
async setupContainer(dockerID) {
|
||||
if (this.containers.has(dockerID)) {
|
||||
return this.containers.get(dockerID);
|
||||
}
|
||||
|
||||
const cpus = this.getCompilerConfig(dockerID, 'cpus', 'string');
|
||||
const memory = this.getCompilerConfig(dockerID, 'memory', 'string');
|
||||
const name = `comp_iler-${dockerID}-${Date.now()}`;
|
||||
try {
|
||||
await exec([
|
||||
`docker run --rm --name=${name}`,
|
||||
'-u1000:1000 -w/tmp/ -dt',
|
||||
`--net=none --cpus=${cpus} -m=${memory} --memory-swap=${memory}`,
|
||||
`1computer1/comp_iler:${dockerID} /bin/sh`
|
||||
].join(' '));
|
||||
|
||||
await exec(`docker exec ${name} mkdir eval`);
|
||||
await exec(`docker exec ${name} chmod 711 eval`);
|
||||
this.containers.set(dockerID, { name });
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(`Started container ${name}.`);
|
||||
return this.containers.get(dockerID);
|
||||
} catch (err) {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
evalCode({ language, code, options, retries = 0 }) {
|
||||
const { id: dockerID = language.id, env = {} } = language.runWith(options);
|
||||
const { evalQueue, setupQueue } = this.queues.get(dockerID);
|
||||
return evalQueue.enqueue(async () => {
|
||||
const { name } = await setupQueue.enqueue(() => this.setupContainer(dockerID));
|
||||
const now = Date.now();
|
||||
try {
|
||||
await exec(`docker exec ${name} mkdir eval/${now}`);
|
||||
await exec(`docker exec ${name} chmod 777 eval/${now}`);
|
||||
const proc = childProcess.spawn('docker', [
|
||||
'exec', '-u1001:1001', `-w/tmp/eval/${now}`,
|
||||
...Object.entries(env).map(([k, v]) => `-e${k}=${v}`),
|
||||
name, '/bin/sh', '/var/run/run.sh', code
|
||||
]);
|
||||
|
||||
const timeout = this.getCompilerConfig(dockerID, 'timeout', 'number');
|
||||
const result = await this.handleSpawn(proc, timeout);
|
||||
await exec(`docker exec ${name} rm -rf eval/${now}`);
|
||||
return result;
|
||||
} catch (err) {
|
||||
this.containers.delete(dockerID);
|
||||
try {
|
||||
await this.kill(name);
|
||||
} catch (err2) {
|
||||
// The container was not alive to be killed, i.e. multiple evals were occuring,
|
||||
// one of them caused the container to be killed, the remaining evals all fail.
|
||||
// Retry those remaining ones until they work!
|
||||
if (retries < this.getCompilerConfig(dockerID, 'retries', 'number')) {
|
||||
return this.evalCode({ language, code, options, retries: retries + 1 });
|
||||
}
|
||||
}
|
||||
|
||||
throw err;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
handleSpawn(proc, timeout = null) {
|
||||
return new Promise((resolve, reject) => {
|
||||
let handled = false;
|
||||
if (timeout !== null) {
|
||||
setTimeout(() => {
|
||||
handled = true;
|
||||
reject(new Error('Timed out'));
|
||||
}, timeout);
|
||||
}
|
||||
|
||||
let output = '';
|
||||
let hasStderr = false;
|
||||
let error;
|
||||
proc.stdout.on('data', chunk => {
|
||||
output += chunk;
|
||||
});
|
||||
|
||||
proc.stderr.on('data', chunk => {
|
||||
hasStderr = true;
|
||||
output += chunk;
|
||||
});
|
||||
|
||||
proc.on('error', e => {
|
||||
error = e;
|
||||
});
|
||||
|
||||
proc.on('close', status => {
|
||||
if (!handled) {
|
||||
handled = true;
|
||||
if (status !== 0 || error) {
|
||||
if (!error) {
|
||||
error = new Error(output || 'Something went wrong');
|
||||
}
|
||||
|
||||
reject(error);
|
||||
} else {
|
||||
resolve({ output, hasStderr });
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async kill(name) {
|
||||
let cmd;
|
||||
if (process.platform === 'win32') {
|
||||
cmd = `docker kill --signal=9 ${name} >nul 2>nul`;
|
||||
} else {
|
||||
cmd = `docker kill --signal=9 ${name} >/dev/null 2>/dev/null`;
|
||||
}
|
||||
|
||||
await exec(cmd);
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(`Killed container ${name}.`);
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
return Promise.all(this.containers.map(({ name }, dockerID) => {
|
||||
this.containers.delete(dockerID);
|
||||
return this.kill(name);
|
||||
}));
|
||||
}
|
||||
|
||||
getCompilerConfig(dockerID, key, type) {
|
||||
const o = this.client.config[key];
|
||||
return typeof o === type
|
||||
? o
|
||||
: o[dockerID] !== undefined
|
||||
? o[dockerID]
|
||||
: o.default;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = LanguageHandler;
|
||||
87
src/struct/Myriad.js
Normal file
87
src/struct/Myriad.js
Normal file
|
|
@ -0,0 +1,87 @@
|
|||
const fetch = require('node-fetch');
|
||||
|
||||
class Myriad {
|
||||
constructor(port) {
|
||||
this.port = port;
|
||||
}
|
||||
|
||||
url(k) {
|
||||
return `http://localhost:${this.port}/${k}`;
|
||||
}
|
||||
|
||||
getLanguages() {
|
||||
return fetch(this.url('languages'), { method: 'GET' }).then(x => x.json());
|
||||
}
|
||||
|
||||
async postEval(language, code) {
|
||||
const response = await fetch(this.url('eval'), {
|
||||
method: 'POST',
|
||||
body: JSON.stringify({ language, code }),
|
||||
headers: { 'Content-Type': 'application/json' }
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
if (response.status === 404 && /^Language .+? was not found$/.test(response.statusText)) {
|
||||
return [false, 'Invalid language'];
|
||||
}
|
||||
|
||||
if (response.status === 500 && response.statusText === 'Evaluation failed') {
|
||||
return [false, 'Evaluation failed'];
|
||||
}
|
||||
|
||||
if (response.status === 504 && response.statusText === 'Evaluation timed out') {
|
||||
return [false, 'Evaluation timed out'];
|
||||
}
|
||||
|
||||
throw new Error(`Unexpected ${response.status} response from ['Myriad', ${response.statusText}`);
|
||||
}
|
||||
|
||||
const body = await response.json();
|
||||
return [true, body.result || '\n'];
|
||||
}
|
||||
|
||||
getContainers() {
|
||||
return fetch(this.url('containers'), { method: 'GET' }).then(x => x.json());
|
||||
}
|
||||
|
||||
postCleanup() {
|
||||
return fetch(this.url('cleanup'), { method: 'POST' }).then(x => x.json());
|
||||
}
|
||||
}
|
||||
|
||||
const entries = [
|
||||
['apl', ['apl']],
|
||||
['bash', ['bash', 'sh']],
|
||||
['brainfuck', ['brainfuck', 'bf']],
|
||||
['c', ['c']],
|
||||
['clojure', ['clojure', 'clj']],
|
||||
['cpp', ['cpp']],
|
||||
['csharp', ['csharp', 'cs']],
|
||||
['elixir', ['elixir']],
|
||||
['fsharp', ['fsharp', 'fs']],
|
||||
['go', ['golang', 'go']],
|
||||
['haskell', ['haskell', 'hs']],
|
||||
['java', ['java']],
|
||||
['javascript', ['javascript', 'js']],
|
||||
['julia', ['julia']],
|
||||
['lua', ['lua']],
|
||||
['ocaml', ['ocaml', 'ml']],
|
||||
['pascal', ['pascal', 'pas', 'freepascal']],
|
||||
['perl', ['perl', 'pl']],
|
||||
['php', ['php']],
|
||||
['prolog', ['prolog']],
|
||||
['python', ['python', 'py']],
|
||||
['racket', ['lisp']],
|
||||
['ruby', ['ruby', 'rb']],
|
||||
['rust', ['rust', 'rs']]
|
||||
];
|
||||
|
||||
Myriad.Languages = new Map(entries);
|
||||
Myriad.Aliases = new Map();
|
||||
for (const [l, xs] of entries) {
|
||||
for (const x of xs) {
|
||||
Myriad.Aliases.set(x, l);
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = Myriad;
|
||||
|
|
@ -1,38 +0,0 @@
|
|||
class Queue {
|
||||
constructor(limit) {
|
||||
this.limit = limit;
|
||||
this.tasks = [];
|
||||
this.ongoing = 0;
|
||||
}
|
||||
|
||||
get length() {
|
||||
return this.tasks.length;
|
||||
}
|
||||
|
||||
enqueue(task) {
|
||||
return new Promise((resolve, reject) => {
|
||||
this.tasks.push({ task, resolve, reject });
|
||||
if (this.ongoing < this.limit) {
|
||||
this.process();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async process() {
|
||||
this.ongoing++;
|
||||
const { task, resolve, reject } = this.tasks.shift();
|
||||
try {
|
||||
const x = await task();
|
||||
resolve(x);
|
||||
} catch (e) {
|
||||
reject(e);
|
||||
}
|
||||
|
||||
this.ongoing--;
|
||||
while (this.ongoing < this.limit && this.tasks.length > 0) {
|
||||
this.process();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = Queue;
|
||||
Loading…
Add table
Add a link
Reference in a new issue