Initial nix installable packages

Author: Brikaa

api/src/api/v2.js

@@ -59,6 +59,7 @@ function get_job(job_info, available_runtimes) {
         run_memory_limit,
         run_timeout,
         compile_timeout,
+        packages,
     } = job_info;
 
     return new Promise((resolve, reject) => {
@@ -133,6 +134,8 @@ function get_job(job_info, available_runtimes) {
             }
         }
 
+        // TODO: packages check
+
         compile_timeout = compile_timeout || rt.timeouts.compile;
         run_timeout = run_timeout || rt.timeouts.run;
         compile_memory_limit = compile_memory_limit || rt.memory_limits.compile;
@@ -151,6 +154,7 @@ function get_job(job_info, available_runtimes) {
                     run: run_memory_limit,
                     compile: compile_memory_limit,
                 },
+                packages: packages || [],
             })
         );
     });
@@ -283,6 +287,7 @@ router.post('/execute', async (req, res) => {
 
         return res.status(200).send(result);
     } catch (error) {
+        console.error(error);
         return res.status(400).json(error);
     }
 });

api/src/config.js

@@ -72,9 +72,10 @@ const options = {
         parser: parse_int,
         validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`],
     },
+    // TODO: handle max install file size
     max_file_size: {
         desc: 'Max file size in bytes for a file',
-        default: 10000000, //10MB
+        default: 100000000, //10MB
         parser: parse_int,
         validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`],
     },

api/src/job.js

@@ -31,7 +31,15 @@ class Job {
     #active_timeouts;
     #active_parent_processes;
 
-    constructor({ runtime, files, args, stdin, timeouts, memory_limits }) {
+    constructor({
+        runtime,
+        files,
+        args,
+        stdin,
+        timeouts,
+        memory_limits,
+        packages,
+    }) {
         this.uuid = uuidv4();
 
         this.logger = logplease.create(`job/${this.uuid}`);
@@ -52,6 +60,8 @@ class Job {
             this.stdin += '\n';
         }
 
+        this.packages = packages;
+
         this.#active_timeouts = [];
         this.#active_parent_processes = [];
 
@@ -141,9 +151,12 @@ class Job {
         this.logger.debug(`Finished exit cleanup`);
     }
 
-    async safe_call(file, args, timeout, memory_limit, eventBus = null) {
+    async safe_call(file, args, timeout, memory_limit, options) {
         return new Promise((resolve, reject) => {
-            const nonetwork = config.disable_networking ? ['nosocket'] : [];
+            const nonetwork =
+                config.disable_networking && !options.elevated_permissions
+                    ? ['nosocket']
+                    : [];
 
             const prlimit = [
                 'prlimit',
@@ -169,26 +182,33 @@ class Job {
             var stderr = '';
             var output = '';
 
+            const proc_info = {};
+            if (!options.elevated_permissions) {
+                proc_info.uid = this.uid;
+                proc_info.gid = this.gid;
+            }
+            if (options.env) {
+                proc_info.env = options.env;
+            }
             const proc = cp.spawn(proc_call[0], proc_call.splice(1), {
                 stdio: 'pipe',
                 cwd: this.dir,
-                uid: this.uid,
-                gid: this.gid,
                 detached: true, //give this process its own process group
+                ...proc_info,
             });
 
             this.#active_parent_processes.push(proc);
 
-            if (eventBus === null) {
+            if (!options.eventBus) {
                 proc.stdin.write(this.stdin);
                 proc.stdin.end();
                 proc.stdin.destroy();
             } else {
-                eventBus.on('stdin', data => {
+                options.eventBus.on('stdin', data => {
                     proc.stdin.write(data);
                 });
 
-                eventBus.on('kill', signal => {
+                options.eventBus.on('kill', signal => {
                     proc.kill(signal);
                 });
             }
@@ -203,8 +223,8 @@ class Job {
             this.#active_timeouts.push(kill_timeout);
 
             proc.stderr.on('data', async data => {
-                if (eventBus !== null) {
-                    eventBus.emit('stderr', data);
+                if (options.eventBus) {
+                    options.eventBus.emit('stderr', data);
                 } else if (stderr.length > this.runtime.output_max_size) {
                     this.logger.info(`stderr length exceeded`);
                     process.kill(proc.pid, 'SIGKILL');
@@ -215,8 +235,8 @@ class Job {
             });
 
             proc.stdout.on('data', async data => {
-                if (eventBus !== null) {
-                    eventBus.emit('stdout', data);
+                if (options.eventBus) {
+                    options.eventBus.emit('stdout', data);
                 } else if (stdout.length > this.runtime.output_max_size) {
                     this.logger.info(`stdout length exceeded`);
                     process.kill(proc.pid, 'SIGKILL');
@@ -241,6 +261,7 @@ class Job {
     }
 
     async execute() {
+        // TODO: group with execute_interactively
         if (this.state !== job_states.PRIMED) {
             throw new Error(
                 'Job must be in primed state, current state: ' +
@@ -250,6 +271,29 @@ class Job {
 
         this.logger.info(`Executing job runtime=${this.runtime.toString()}`);
 
+        let install;
+
+        // TODO: install timeouts, install memory limits, install errored
+        if (this.runtime.package_support && this.packages.length > 0) {
+            install = await this.safe_call(
+                this.runtime.installPackages,
+                this.packages.map(
+                    p => `${this.runtime.flake_path}.packages.${p}`
+                ),
+                -1,
+                -1,
+                {
+                    elevated_permissions: true,
+                }
+            );
+        }
+
+        const env = {};
+        if (install && install.stdout !== '' && install.code === 0) {
+            const install_json = JSON.parse(install.stdout);
+            env.PISTON_PACKAGES_PATH = install_json.map(i => i.outputs.out).join(':');
+        }
+
         const code_files = this.files.filter(file => file.encoding == 'utf8');
 
         this.logger.debug('Compiling');
@@ -262,7 +306,8 @@ class Job {
                 this.runtime.compile,
                 code_files.map(x => x.name),
                 this.timeouts.compile,
-                this.memory_limits.compile
+                this.memory_limits.compile,
+                { env }
             );
             compile_errored = compile.code !== 0;
         }
@@ -275,13 +320,15 @@ class Job {
                 this.runtime.run,
                 [code_files[0].name, ...this.args],
                 this.timeouts.run,
-                this.memory_limits.run
+                this.memory_limits.run,
+                { env }
             );
         }
 
         this.state = job_states.EXECUTED;
 
         return {
+            install,
             compile,
             run,
             language: this.runtime.language,
@@ -311,7 +358,7 @@ class Job {
                 code_files.map(x => x.name),
                 this.timeouts.compile,
                 this.memory_limits.compile,
-                eventBus
+                { eventBus }
             );
 
             eventBus.emit('exit', 'compile', { error, code, signal });
@@ -326,7 +373,7 @@ class Job {
                 [code_files[0].name, ...this.args],
                 this.timeouts.run,
                 this.memory_limits.run,
-                eventBus
+                { eventBus }
             );
 
             eventBus.emit('exit', 'run', { error, code, signal });

api/src/runtime.js

@@ -10,7 +10,7 @@ class Runtime {
         runtime,
         run,
         compile,
-        packageSupport,
+        installPackages,
         flake_path,
         timeouts,
         memory_limits,
@@ -34,9 +34,9 @@ class Runtime {
 
         this.run = run;
         this.compile = compile;
+        this.installPackages = installPackages;
 
         this.flake_path = flake_path;
-        this.package_support = packageSupport;
     }
 
     ensure_built() {
@@ -50,6 +50,7 @@ class Runtime {
         }
 
         _ensure_built('run');
+        if (this.package_support) _ensure_built('installPackages');
         if (this.compiled) _ensure_built('compile');
 
         logger.debug(`Finished ensuring ${this} is installed`);
@@ -59,6 +60,10 @@ class Runtime {
         return this.compile !== null;
     }
 
+    get package_support() {
+        return this.installPackages !== null;
+    }
+
     toString() {
         return `${this.language}-${this.version}`;
     }
@@ -72,8 +77,7 @@ function compute_single_limit(
     return (
         (config.limit_overrides[language_name] &&
             config.limit_overrides[language_name][limit_name]) ||
-        (language_limit_overrides &&
-            language_limit_overrides[limit_name]) ||
+        (language_limit_overrides && language_limit_overrides[limit_name]) ||
         config[limit_name]
     );
 }
@@ -134,14 +138,11 @@ function get_runtime_from_flakes(runtime_name) {
 
     const this_runtime = new Runtime({
         ...metadata,
-        ...compute_all_limits(
-            metadata.language,
-            metadata.limitOverrides
-        ),
+        ...compute_all_limits(metadata.language, metadata.limitOverrides),
         flake_path,
     });
 
-    return this_runtime
+    return this_runtime;
 }
 
 module.exports.Runtime = Runtime;

flake.nix

@@ -29,7 +29,9 @@
             inherit language version runtime aliases limitOverrides;
             run = runFile;
             compile = compileFile;
-            packageSupport = packages != null;
+            installPackages = if packages != null then pkgs.writeShellScript "install-packages" ''
+            nix build --json "$@"
+            '' else null;
           };
         in {
           inherit packages metadata;

runtimes/python3.nix

@@ -12,9 +12,11 @@ in piston.mkRuntime {
     ];
 
     run = ''
-    ${pkg}/bin/python3 "$@"
+    PYTHONPATH=$PISTON_PACKAGES_PATH ${pkg}/bin/python3 "$@"
     '';
 
+    packages = pkg.pkgs;
+
     tests = [
         (piston.mkTest {
             files = {