Merge branch 'code.pyret.org'

Author: jpolitz

code.pyret.org/package-lock.json

@@ -55,6 +55,7 @@
     "react-dom": "^15.7.0",
     "redis": "^0.10.3",
     "request": "^2.88.2",
+    "request-filtering-agent": "^3.2.0",
     "requirejs": "2.1.14",
     "s-expression": "~2.2.0",
     "script-loader": "^0.7.2",
@@ -88,7 +89,7 @@
   "author": "Joe Politz",
   "license": "Apache-2.0",
   "devDependencies": {
-    "chromedriver": "^146.0.1",
+    "chromedriver": "^146.0.4",
     "selenium-webdriver": "^3.6.0",
     "webpack-cli": "^5.1.4"
   }

code.pyret.org/package.json

@@ -4,12 +4,14 @@ var OAuth2 = gapi.auth.OAuth2;
 
 var DEFAULT_OAUTH_SCOPES = [
                     "email",
+                    "profile",
                     "https://www.googleapis.com/auth/drive.file",
                     "https://www.googleapis.com/auth/drive.install",
                   ];
 
 var FULL_OAUTH_SCOPES = [
                     "email",
+                    "profile",
                     "https://www.googleapis.com/auth/spreadsheets",
                     "https://www.googleapis.com/auth/drive.file",
                     "https://www.googleapis.com/auth/drive",

code.pyret.org/src/google-auth.js

@@ -23,6 +23,8 @@ var res = Q.fcall(function(db) {
     development: process.env["NODE_ENV"] !== "production",
     baseUrl: process.env["BASE_URL"],
     logURL: process.env["LOG_URL"],
+    logUser: process.env["LOG_USER"],
+    logPassword: process.env["LOG_PASSWORD"],
     gitRev:    process.env["GIT_REV"]    || git.short(),
     gitBranch: process.env["GIT_BRANCH"] || git.branch(),
     port: process.env["PORT"],

code.pyret.org/src/run.js

@@ -7,14 +7,39 @@ const { drive } = require("googleapis/build/src/apis/drive/index.js");
 
 var BACKREF_KEY = "originalProgram";
 
+// Limits for the streaming proxy. /downloadImg gets larger/looser caps because
+// images can legitimately be tens of MB; also we've seen e.g. Drive ?export=
+// take a while to get going. SHAREURL is intended to always be program
+// plaintext.
+// NOTE(joe + claude): really the timeout maybe should be on idleness at
+// startup/between bytes, not overall per completed request, but that's work to
+// plumb into `request`
+var IMAGE_PROXY_MAX_BYTES    = 20 * 1024 * 1024; // 20 MB
+var IMAGE_PROXY_TIMEOUT_MS   = 30 * 1000;        // 30 s
+var SHAREURL_PROXY_MAX_BYTES  = 1 * 1024 * 1024;  // 1 MB
+var SHAREURL_PROXY_TIMEOUT_MS = 10 * 1000;        // 10 s
+
 function start(config, onServerReady) {
+  var defaultOpts = {
+      PYRET: process.env.PYRET,
+      BASE_URL: config.baseUrl,
+      GOOGLE_API_KEY: config.google.apiKey,
+      GOOGLE_APP_ID: config.google.appId,
+      LOG_URL: config.logURL,
+      LOG_PASSWORD: config.logPassword,
+      LOG_USER: config.logUser,
+      GIT_REV : config.gitRev,
+      GIT_BRANCH: config.gitBranch,
+      POSTMESSAGE_ORIGIN: process.env.POSTMESSAGE_ORIGIN
+    };
   var express = require('express');
   var cookieSession = require('cookie-session');
   var cookieParser = require('cookie-parser');
   var bodyParser = require('body-parser');
   var csrf = require('csurf');
   var googleAuth = require('./google-auth.js');
   var request = require('request');
+  var requestFilteringAgent = require('request-filtering-agent');
   var mustache = require('mustache-express');
   var url = require('url');
   var fs = require('fs');
@@ -107,14 +132,8 @@ function start(config, onServerReady) {
 
   app.get("/", function(req, res) {
     var content = loggedIn(req) ? "My Programs" : "Log In";
-    res.render("index.html", {
-      PYRET: process.env.PYRET,
+    res.render("index.html", { ...defaultOpts,
       LEFT_LINK: content,
-      GOOGLE_API_KEY: config.google.apiKey,
-      BASE_URL: config.baseUrl,
-      LOG_URL: config.logURL,
-      GIT_REV : config.gitRev,
-      GIT_BRANCH: config.gitBranch
     });
   });
 
@@ -180,24 +199,74 @@ function start(config, onServerReady) {
     });
   }
 
-  app.get("/downloadImg", function(req, response) {
-    var parsed = url.parse(req.url);
-    var googleLink = decodeURIComponent(parsed.query.slice(0));
-    var googleParsed = url.parse(googleLink);
-    var gReq = request({url: googleLink, encoding: 'binary'}, function(error, imgResponse, body) {
-      if(error) {
-        response.status(400).send({type: "image-load-failure", error: "Unable to load image " + String(error)});
+  function proxyStreamFetch(opts) {
+    var res = opts.res;
+    res.set('X-Content-Type-Options', 'nosniff');
+    res.set('Content-Security-Policy', 'sandbox');
+
+    var parsed;
+    try { parsed = new URL(opts.url); }
+    catch (e) { return res.status(400).send({ error: 'invalid-url' }); }
+    if (opts.allowedHosts && !opts.allowedHosts(parsed.hostname)) {
+      return res.status(400).send({ error: 'host-not-allowed' });
+    }
+
+    var bytes = 0;
+    var upstream = request({
+      url: opts.url,
+      timeout: opts.timeoutMs,
+      agent: requestFilteringAgent.useAgent(opts.url),
+      followRedirect: function(resp) {
+        if (!opts.allowedHosts) return true;
+        try {
+          var next = new URL(resp.headers.location, opts.url);
+          return opts.allowedHosts(next.hostname);
+        } catch (_) { return false; }
+      },
+    });
+    // If the client disconnects (e.g. the browser aborts /load-shareurl after
+    // direct succeeded), tear down the upstream connection too — otherwise
+    // we'd keep streaming bytes from raw.githubusercontent.com to nowhere.
+    res.on('close', function() { upstream.destroy(); });
+    upstream.on('error', function(err) {
+      if (!res.headersSent) opts.onError(res, err);
+    });
+    upstream.on('response', function(upRes) {
+      if (opts.contentTypeOk && !opts.contentTypeOk(upRes.headers['content-type'])) {
+        upstream.destroy();
+        return res.status(400).send({ error: 'content-type-not-allowed', detail: upRes.headers['content-type'] });
       }
-      else {
-        var h = imgResponse.headers;
-        var ct = h['content-type'];
-        if((!ct) || (ct.indexOf('image/') !== 0)) {
-          response.status(400).send({type: "non-image", error: "Invalid image type " + ct});
-          return;
-        }
-        response.set('content-type', ct);
-        response.end(body, 'binary');
+      res.status(upRes.statusCode);
+      if (upRes.headers['content-type']) {
+        res.set('content-type', upRes.headers['content-type']);
       }
+      upRes.on('data', function(chunk) {
+        bytes += chunk.length;
+        if (bytes > opts.maxBytes) {
+          upstream.destroy();
+          if (!res.headersSent) res.status(502).send({ error: 'too-large' });
+          else res.destroy();
+        }
+      });
+      // Pipe upRes (IncomingMessage), not upstream (request object). The
+      // request library's .pipe copies upstream headers verbatim, which
+      // would overwrite the security headers set above.
+      upRes.pipe(res);
+    });
+  }
+
+  app.get("/downloadImg", function(req, response) {
+    var googleLink = decodeURIComponent(url.parse(req.url).query.slice(0));
+    proxyStreamFetch({
+      res: response,
+      url: googleLink,
+      allowedHosts: null,
+      maxBytes: IMAGE_PROXY_MAX_BYTES,
+      timeoutMs: IMAGE_PROXY_TIMEOUT_MS,
+      contentTypeOk: function(ct) { return ct && ct.indexOf('image/') === 0; },
+      onError: function(res, err) {
+        res.status(400).send({ type: 'image-load-failure', error: 'Unable to load image ' + String(err) });
+      },
     });
   });
 
@@ -529,30 +598,14 @@ function start(config, onServerReady) {
   });
 
   app.get("/editor", function(req, res) {
-    res.render("editor.html", {
-      PYRET: process.env.PYRET,
-      BASE_URL: config.baseUrl,
-      GOOGLE_API_KEY: config.google.apiKey,
-      GOOGLE_APP_ID: config.google.appId,
+    res.render("editor.html", { ...defaultOpts,
       CSRF_TOKEN: req.csrfToken(),
-      LOG_URL: config.logURL,
-      GIT_REV : config.gitRev,
-      GIT_BRANCH: config.gitBranch,
-      POSTMESSAGE_ORIGIN: process.env.POSTMESSAGE_ORIGIN
     });
   });
 
   app.get("/blocks", function(req, res) {
-    res.render("blocks.html", {
-      PYRET: process.env.PYRET,
-      BASE_URL: config.baseUrl,
-      GOOGLE_API_KEY: config.google.apiKey,
-      GOOGLE_APP_ID: config.google.appId,
+    res.render("blocks.html", { ...defaultOpts,
       CSRF_TOKEN: req.csrfToken(),
-      LOG_URL: config.logURL,
-      GIT_REV : config.gitRev,
-      GIT_BRANCH: config.gitBranch,
-      POSTMESSAGE_ORIGIN: process.env.POSTMESSAGE_ORIGIN
     });
   });
 
@@ -575,6 +628,26 @@ function start(config, onServerReady) {
 
   });
 
+  // Server-side proxy for #shareurl loads from hosts that some school networks
+  // block or will likely block (notably raw.githubusercontent.com).
+  // Eager-proxied client-side for any URL whose host is in
+  // SHAREURL_ALLOWED_HOSTS. We can expand this list as needed.
+  var SHAREURL_ALLOWED_HOSTS = new Set(['raw.githubusercontent.com']);
+
+  app.get("/load-shareurl", function(req, res) {
+    proxyStreamFetch({
+      res: res,
+      url: req.query.url,
+      allowedHosts: function(h) { return SHAREURL_ALLOWED_HOSTS.has(h); },
+      maxBytes: SHAREURL_PROXY_MAX_BYTES,
+      timeoutMs: SHAREURL_PROXY_TIMEOUT_MS,
+      contentTypeOk: null,
+      onError: function(res, err) {
+        res.status(502).send({ error: 'upstream-error' });
+      },
+    });
+  });
+
 
   app.post("/share-image", function(req, res) {
     var driveFileId = req.body.fileId;

code.pyret.org/src/server.js

@@ -486,7 +486,7 @@ div.repl-animation {
 
 div.trace {
   overflow-x: auto;
-  padding-top: 1px;
+  margin-top: 1px;
 }
 div.trace.error {
    background-color: var(--trace-err-bg);
@@ -2304,23 +2304,42 @@ table.pyret-table.pyret-matrix {
 }
 
 
-table.pyret-table { width: 98%; overflow: scroll; }
+table.pyret-table { width: 98%; overflow: scroll; border: none; }
+/* make sure rotated elts in the table (like roughnums!) don't punch through */
 table.pyret-table thead {
   box-shadow: 0px 2px 2px var(--shadow-color);
   position: sticky;
   top: 0;
+  z-index: 1; 
 }
 
-table.pyret-table tr:last-child td:first-child {
- border-bottom-left-radius: var(--table-radius); 
+/* top/bottom borders on the first and last rows */
+table.pyret-table th { border-top: solid 1px black; }
+table.pyret-table > :nth-last-child(1 of :has(tr)) > tr:last-child td {
+  border-bottom: solid 1px black;
 }
-table.pyret-table tr:last-child td:last-child {
- border-bottom-right-radius: var(--table-radius); 
+
+/* left/right borders on the first and last cells */
+table.pyret-table td:first-child, table.pyret-table th:first-child {
+  border-left: solid 1px black;
+}
+table.pyret-table td:last-child, table.pyret-table th:last-child {
+  border-right: solid 1px black;
 }
 
-/* style first and last th elements to use the table's rounded corners */
-table.pyret-table th:first-child { border-top-left-radius: var(--table-radius); }
-table.pyret-table th:last-child  { border-top-right-radius: var(--table-radius); }
+/* rounded borders for corners */
+table.pyret-table th:first-child {
+ border-top-left-radius: var(--table-radius);
+}
+table.pyret-table th:last-child {
+ border-top-right-radius: var(--table-radius);
+}
+table.pyret-table > :nth-last-child(1 of :has(tr)) > tr:last-child td:first-child {
+ border-bottom-left-radius: var(--table-radius);
+}
+table.pyret-table > :nth-last-child(1 of :has(tr)) > tr:last-child td:last-child {
+ border-bottom-right-radius: var(--table-radius);
+}