Skip to content

Commit 560a47f

Browse files
alexchan-vaalexchan-va
authored
Merge branch 'master' into dependabot/bundler/oj-3.16.10
2 parents 2e00450 + 5bac218 commit 560a47f

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

42 files changed

+492
-1289
lines changed

‎app/assets/javascripts/upload.js

Lines changed: 59 additions & 200 deletions
Original file line numberDiff line numberDiff line change
@@ -7,16 +7,19 @@ $(function() {
77
);
88
});
99

10-
// ASYNC UPLOAD LOGIC
10+
// SEQUENTIAL UPLOAD LOGIC
11+
// Simulates multiple file upload by breaking file down and submitting sequence of uploads
12+
13+
//
1114
const getPathPrefix = () => {
1215
const hostname = window.location.hostname
1316
const subdomain = hostname.split('.')[0];
1417
return subdomain === 'localhost' ? '' : '/gids'
1518
};
1619
const PATH_PREFIX = getPathPrefix();;
1720

18-
// Open dialog during initial async upload to disable page
19-
$("#async-upload-dialog").dialog({
21+
// Open dialog during sequential upload to disable page
22+
$("#sequential-upload-dialog").dialog({
2023
autoOpen: false,
2124
modal: true,
2225
width: "auto",
@@ -26,235 +29,91 @@ $(function() {
2629
}
2730
});
2831

29-
// POLL UPLOAD STATUS
30-
const ON_SCREEN_POLL_RATE = 5_000;
31-
const BACKGROUND_POLL_RATE = 10_000;
32-
33-
const capitalize = (str, titlecase) => titlecase ? str.charAt(0).toUpperCase() + str.slice(1) : str;
34-
35-
// CRUD methods for local storage
36-
const getUploadQueue = () => {
37-
try {
38-
const uploadQueue = JSON.parse(localStorage.getItem("uploadQueue"));
39-
if (Array.isArray(uploadQueue)) {
40-
return uploadQueue;
41-
}
42-
throw new TypeError()
43-
} catch(error) {
44-
localStorage.setItem("uploadQueue", "[]");
45-
return [];
46-
}
47-
};
48-
const addToQueue = (uploadId) => {
49-
const uploadQueue = getUploadQueue();
50-
uploadQueue.push(uploadId);
51-
localStorage.setItem("uploadQueue", JSON.stringify([...new Set(uploadQueue)]));
52-
};
53-
const removeFromQueue = (uploadId) => {
54-
const uploadQueue = getUploadQueue();
55-
const filteredQueue = uploadQueue.filter((id) => id !== uploadId);
56-
localStorage.setItem("uploadQueue", JSON.stringify([...new Set(filteredQueue)]));
57-
};
58-
59-
// Cancel upload if still active
60-
const cancelUpload = async (icon, uploadId) => {
61-
$(icon).off("click mouseleave");
62-
try {
63-
await $.ajax({
64-
url: `${PATH_PREFIX}/uploads/${uploadId}/cancel`,
65-
type: "PATCH",
66-
contentType: false,
67-
processData: false,
68-
});
69-
pollUploadStatus();
70-
} catch(error) {
71-
console.error(error);
72-
const uploadStatusDiv = $(`#async-upload-status-${uploadId}`)[0];
73-
const { titlecase } = uploadStatusDiv.dataset || false;
74-
$(uploadStatusDiv).html(capitalize("failed to cancel", titlecase));
75-
await new Promise((resolve) => setTimeout(resolve, ON_SCREEN_POLL_RATE));
76-
pollUploadStatus();
77-
}
78-
};
79-
80-
// Dialog and dialog display method for when async upload complete
81-
$("#async-upload-alert").dialog({
82-
autoOpen: false,
83-
modal: true,
84-
width: "auto",
85-
residable: false,
86-
open: function() {
87-
const { uploadId, csvType } = $(this).data();
88-
$(this).html(
89-
`<p>${csvType} file upload complete</p>` +
90-
'<p>Click ' + `<a href="${PATH_PREFIX}/uploads/${uploadId}">here</a>` +
91-
' for a more detailed report</p>'
92-
);
93-
}
94-
});
95-
const displayAlert = (uploadId, csvType) => {
96-
$("#async-upload-alert").data({"uploadId": uploadId, "csvType": csvType}).dialog("open");
97-
};
98-
99-
// Grab active client-side uploads from local storage and poll each upload for status
100-
let consecutiveFails = 0;
101-
const pollUploadStatus = async () => {
102-
const uploadQueue = getUploadQueue();
103-
uploadQueue.forEach(async (uploadId) => {
104-
const uploadStatusDiv = $(`#async-upload-status-${uploadId}`)[0];
105-
const onScreen = typeof uploadStatusDiv !== "undefined";
106-
const pollRate = onScreen ? ON_SCREEN_POLL_RATE : BACKGROUND_POLL_RATE;
107-
const { titlecase } = uploadStatusDiv?.dataset || false;
108-
try {
109-
const xhr = new XMLHttpRequest();
110-
const getUploadStatus = () => {
111-
xhr.open("GET", `${PATH_PREFIX}/uploads/${uploadId}/status`);
112-
xhr.send();
113-
};
114-
xhr.onload = function() {
115-
if (this.status === 200) {
116-
consecutiveFails = 0;
117-
const { message, active, ok, canceled, type } = JSON.parse(xhr.response).async_status;
118-
// If upload active and status currently visible on screen
119-
if (active) {
120-
if (onScreen) {
121-
// Update DOM
122-
$(uploadStatusDiv).html(
123-
'<i class="fa fa-gear fa-spin upload-icon" style="font-size:16px"></i>' +
124-
`<div>${capitalize(message, titlecase)}</div>`
125-
);
126-
const icon = $(uploadStatusDiv).find("i");
127-
// Enable cancel upload button
128-
$(icon).on({
129-
mouseover: function(_event) {
130-
clearInterval(pollingInterval);
131-
$(this).removeClass("fa-gear fa-spin").addClass("fa-solid fa-times").css({color: "red", fontSize: "20px"});
132-
$(this).on("click", (_event) => cancelUpload(this, uploadId));
133-
},
134-
mouseleave: function(_event) {
135-
pollingInterval = setInterval(getUploadStatus, pollRate);
136-
$(this).removeClass("fa-solid fa-times").addClass("fa-gear fa-spin").css({color: "#333", fontSize: "16px"});
137-
$(this).off("click");
138-
}
139-
});
140-
}
141-
// If upload completed or canceled
142-
} else{
143-
removeFromQueue(uploadId);
144-
clearInterval(pollingInterval);
145-
// If upload status currently visible on screen
146-
if (onScreen) {
147-
$(uploadStatusDiv).html(capitalize(ok ? "succeeded" : "failed", titlecase));
148-
}
149-
// If on upload#show page, reload page to render flash alerts
150-
if (window.location.pathname === `${PATH_PREFIX}/uploads/${uploadId}`) {
151-
window.location.reload();
152-
// Otherwise render link to alerts in pop dialog
153-
} else if (!canceled) {
154-
displayAlert(uploadId, type);
155-
}
156-
}
157-
} else {
158-
consecutiveFails++;
159-
if (consecutiveFails === 5) {
160-
removeFromQueue(uploadId);
161-
clearInterval(pollingInterval);
162-
}
163-
}
164-
};
165-
getUploadStatus();
166-
let pollingInterval = setInterval(getUploadStatus, pollRate);
167-
} catch(error) {
168-
console.error(error);
169-
}
170-
});
171-
};
172-
$(document).ready(() => pollUploadStatus());
173-
174-
// Reset active upload if for some reason stuck on "Loading . . ."
175-
// Not sure this is necessary, but technically someone could mess with local storage and
176-
// it would mess up queue
177-
$(".default-async-loading").on({
178-
mouseover: function(_event) {
179-
$(this).removeClass("fa-gear fa-spin").addClass("fa-solid fa-rotate").css({color: "green"});
180-
$(this).on("click", (_event) => {
181-
const { uploadId } = this.dataset;
182-
addToQueue(parseInt(uploadId));
183-
pollUploadStatus();
184-
});
185-
},
186-
mouseleave: function(_event) {
187-
$(this).removeClass("fa-solid fa-rotate").addClass("fa-gear fa-spin").css({color: "#333", fontSize: "16px"});
188-
$(this).off("click");
189-
}
190-
});
191-
192-
// ASYNC SUBMIT ACTION
193-
// Submit logic for new upload form when async upload enabled
194-
$("#async-submit-btn").on("click", async function(event) {
32+
// Submit logic for new upload form when sequential upload enabled
33+
$("#seq-submit-btn").on("click", async function(event) {
19534
event.preventDefault();
196-
// Grab form data and validate file extension
35+
// Grab form data and validate file selected
19736
const form = $("#new_upload")[0];
198-
const formData = new FormData(form);
199-
const file = formData.get("upload[upload_file]");
200-
const ext = file.name !== '' ? file.name.slice(file.name.lastIndexOf(".")) : null;
201-
const fileInput = $("#upload_upload_file");
202-
const validExts = $(fileInput).attr("accept").split(", ");
203-
$(fileInput)[0].setCustomValidity('');
204-
if (ext !== null && !validExts.includes(ext)) {
205-
$(fileInput)[0].setCustomValidity(`${ext} is not a valid file format.`);
206-
}
20737
if (!form.reportValidity()) {
20838
return;
20939
}
40+
const formData = new FormData(form);
41+
const file = formData.get("upload[upload_file]");
42+
21043
// Open dialog and disable page until client-side processing complete
211-
$("#async-upload-dialog").dialog("open");
44+
$("#sequential-upload-dialog").dialog("open");
21245
$(this).html(
213-
'<div id="async-submit-btn-div">' +
46+
'<div id="sequential-submit-btn-div">' +
21447
'<i class="fa fa-gear fa-spin" style="font-size:16px"></i>' +
21548
'Submitting . . .' +
21649
'</div>'
21750
);
218-
const csvType = formData.get("upload[csv_type]");
219-
let uploadId = null;
51+
22052
// Divide upload file into smaller files
22153
const blobs = [];
22254
const generateBlobs = async () => {
22355
const chunkSize = parseInt(this.dataset.chunkSize);
56+
const text = await file.text();
57+
const header = text.slice(0, text.indexOf('\n') + 1);
22458
for (let start = 0; start < file.size; start += chunkSize) {
225-
const blob = file.slice(start, start + chunkSize, "text/plain");
226-
blobs.push(blob);
59+
let end = start + chunkSize;
60+
let charsToEndOfRow = 0;
61+
// Ensure rows not divided between blobs
62+
if (text[end - 1] !== "\n") {
63+
charsToEndOfRow = text.slice(end).indexOf("\n") + 1;
64+
end += charsToEndOfRow;
65+
};
66+
const blob = file.slice(start, end, "text/plain");
67+
// Add header if not already present
68+
const fileBits = start === 0 ? [blob] : [header, blob]
69+
const newFile = new File(fileBits, { type: "text/plain" });
70+
blobs.push(newFile);
71+
start += charsToEndOfRow;
22772
}
22873
};
74+
22975
// Send individual POST request for each blob, simulating multiple file upload
76+
let uploadId;
23077
const submitBlobs = async () => {
78+
const total = blobs.length;
79+
const idx = file.name.lastIndexOf(".");
80+
const [basename, ext] = [file.name.slice(0, idx), file.name.slice(idx)];
23181
try {
23282
for (let i = 0; i < blobs.length; i++) {
233-
formData.set("upload[upload_file]", blobs[i], file.name);
83+
const current = i + 1;
84+
const filePosition = `${current.toString().padStart(2, '0')}_of_${total.toString().padStart(2, '0')}`;
85+
const fileName = `${basename}_${filePosition}${ext}`;
86+
formData.set("upload[upload_file]", blobs[i], fileName);
87+
// Set multiple_file_upload to true after first upload
88+
if (i > 0) {
89+
formData.set("upload[multiple_file_upload]", true);
90+
}
23491
// Include metadata in payload to track upload progress across multiple requests
235-
formData.set("upload[metadata][upload_id]", uploadId);
236-
formData.set("upload[metadata][count][current]", i + 1);
237-
formData.set("upload[metadata][count][total]", blobs.length);
238-
const response = await $.ajax({
239-
url: `${PATH_PREFIX}/uploads`,
240-
type: "POST",
241-
data: formData,
242-
dataType: "json",
243-
contentType: false,
244-
processData: false,
92+
formData.set("upload[sequence][current]", current);
93+
formData.set("upload[sequence][total]", total);
94+
const response = await fetch(`${PATH_PREFIX}/uploads`, {
95+
method: "POST",
96+
body: formData
24597
});
246-
uploadId = response.id;
98+
99+
if (!response.ok) {
100+
throw new Error(`Upload failed with status ${response.status}`);
101+
}
102+
103+
const data = await response.json();
104+
if (data.final) {
105+
uploadId = data.upload.id;
106+
}
247107
}
248-
// If successful, save upload ID in local storage to enable status polling
249-
addToQueue(uploadId);
250108
window.location.href = `${PATH_PREFIX}/uploads/${uploadId}`;
251109
} catch(error) {
252110
console.error(error);
111+
const csvType = formData.get("upload[csv_type]");
253112
window.location.href = `${PATH_PREFIX}/uploads/new/${csvType}`;
254113
}
255114
};
256115

257-
generateBlobs();
116+
await generateBlobs();
258117
submitBlobs();
259118
});
260119
});

‎app/assets/stylesheets/application.css

Lines changed: 6 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -154,32 +154,25 @@
154154
}
155155

156156
#preview_dialog,
157-
#async-upload-dialog {
157+
#sequential-upload-dialog {
158158
display: flex;
159159
justify-content: center;
160160
align-items: center;
161161
}
162162

163-
#async-upload-alert {
163+
#sequential-upload-alert {
164164
display: flex;
165165
flex-direction: column;
166166
justify-content: center;
167167
}
168168

169-
#async-upload-alert a {
170-
color: #337ab7;
171-
text-decoration: underline;
172-
font-weight: bold;
173-
}
174-
175169
#preview_dialog_div,
176-
#async-upload-dialog-div,
177-
#async-upload-alert {
170+
#sequential-upload-dialog-div {
178171
min-width: 250px;
179172
text-align: center;
180173
}
181174

182-
#async-upload-dialog-div {
175+
#sequential-upload-dialog-div {
183176
display: flex;
184177
justify-content: center;
185178
align-items: center;
@@ -203,15 +196,13 @@
203196
margin-bottom: 1em;
204197
}
205198

206-
.async-upload-status-div,
207-
#async-submit-btn-div {
199+
#sequential-submit-btn-div {
208200
display: flex;
209201
align-items: center;
210202
gap: 5px;
211203
}
212204

213-
.upload-icon:hover,
214-
.default-async-loading {
205+
.upload-icon:hover {
215206
cursor: pointer;
216207
}
217208

‎app/contraints/async_upload_constraint.rb

Lines changed: 0 additions & 32 deletions
This file was deleted.

0 commit comments

Comments
 (0)