Feature/import data

surveyadmin/src/app/admin/export/ExportSurveyData.tsx

@@ -0,0 +1,235 @@
+"use client";
+import React, { useReducer, useState } from "react";
+import { apiRoutes } from "~/lib/apiRoutes";
+
+/**
+ * Trigger a file download for any URL
+ *
+ * NOTE: if the download fails and the server returns a JSON,
+ * this will show the download file as a .json because of the response application type
+ * @param url
+ */
+const triggerDownload = (url: string, filename?: string) => {
+  const a = document.createElement("a");
+  a.href = url;
+  a.download = filename || "";
+  document.body.appendChild(a);
+  a.click();
+  a.remove();
+  /*
+         Alternative code with fetch
+         This is better to handle error messages, however it rely on
+         loading the file client-side as a BLOB, which can cost a lot of RAM
+
+         const response = await fetch(apiRoutes.admin.dataExport.href, {
+           method: "GET",
+           body: JSON.stringify(body),
+           headers: {
+             "Content-type": "application/json",
+           },
+         });
+
+         if (!response.ok) {
+           const body = await response.text();
+           console.error(body);
+           dispatch({
+             type: "error",
+             payload: new Error(body.slice(0, 500)),
+           });
+         } else {
+           dispatch({ type: "done" });
+           try {
+             await downloadFromResponse(response, "export.json");
+             dispatch({ type: "downloadStarted" });
+           } catch (error) {
+             dispatch({ type: "error", payload: error });
+           }
+         }*/
+};
+
+/**
+ * Useful when getting the file from a POST request or a fetch response
+ *
+ * However it's easier to generate a <a> tag and let it trigger the relevant GET request automatically,
+ * instead of using fetch
+ */
+const downloadFromResponse = async (response: Response, filename: string) => {
+  const blob = await response.blob();
+  var url = window.URL.createObjectURL(blob);
+  triggerDownload(url, filename);
+};
+
+const initialState = { loading: false, error: null, res: null, done: false };
+const exportReducer = (state, action) => {
+  switch (action.type) {
+    case "start": {
+      return { loading: true, error: null, done: false };
+    }
+    case "done": {
+      return { loading: false, error: null, done: true };
+    }
+    case "downloadStarted": {
+      return initialState;
+    }
+    case "error": {
+      const error = action.payload;
+      return { loading: false, error, res: null, done: false };
+    }
+    default: {
+      return { loading: false, res: null, error: null, done: false };
+    }
+  }
+};
+
+export const ExportSurveyData = ({
+  surveyId,
+  editionId,
+}: {
+  surveyId: string;
+  editionId: string;
+}) => {
+  const [state, dispatch] = useReducer(exportReducer, initialState);
+
+  // timestamp = the unique id for the latest export
+  // we can pass it back to the server to download or further process the generated file
+  const [timestamp, setTimestamp] = useState<null | string>(null);
+
+  /**
+   * Trigger the generation of an export server-side,
+   * that will be stored in a tmp folder
+   */
+  async function generateExport({
+    surveyId,
+    editionId,
+  }: {
+    surveyId: string;
+    editionId: string;
+  }) {
+    try {
+      dispatch({ type: "start" });
+      const url = apiRoutes.export.generate.href({ surveyId, editionId });
+      // TODO: error handling
+      const { timestamp } = await (await fetch(url)).json();
+      setTimestamp(timestamp as string);
+      dispatch("downloadStarted");
+    } catch (error) {
+      console.error(error);
+      dispatch({ type: "error", payload: error });
+    }
+  }
+  /**
+   * Download previously gnerated export
+   * @param param0
+   * @returns
+   */
+  async function downloadExport({
+    surveyId,
+    editionId,
+    timestamp,
+  }: {
+    surveyId: string;
+    editionId: string;
+    timestamp: string;
+  }) {
+    try {
+      if (!timestamp) {
+        return dispatch({
+          type: "error",
+          payload:
+            "No timestamp provided, please first generate an export for this survey",
+        });
+      }
+      dispatch({ type: "start" });
+      const url = apiRoutes.export.download.href({
+        surveyId,
+        editionId,
+        timestamp,
+      });
+
+      triggerDownload(url);
+      dispatch("downloadStarted");
+    } catch (error) {
+      console.error(error);
+      dispatch({ type: "error", payload: error });
+    }
+  }
+  return (
+    <div>
+      <h1>Export</h1>
+      <h2>Step 1: generate a CSV export for the survey</h2>
+      <p>This will create a file in your tmp folder</p>
+      <button
+        aria-busy={state.loading}
+        disabled={state.loading}
+        onClick={() => {
+          if (!(surveyId && editionId)) {
+            dispatch({
+              type: "error",
+              payload: new Error(
+                "Can't trigger download without selecting surveyId && editionId"
+              ),
+            });
+          }
+          generateExport({ surveyId, editionId });
+        }}
+      >
+        {!state.loading
+          ? timestamp
+            ? "Generate exports"
+            : "Regenerate exports"
+          : "Busy..."}
+        {state.error && <p>Error: {state.error.message}</p>}
+        {state.done && <p>Export done</p>}
+      </button>
+      <div>
+        <p>
+          Manually input a timestamp if you have already generated some exports,
+          or copy this value to later retrieve your export:
+        </p>
+        <input
+          type="text"
+          value={timestamp || ""}
+          onChange={(e) => {
+            setTimestamp(e.target.value);
+          }}
+        />
+      </div>
+      {timestamp && (
+        <>
+          <h2>Step 3: download the CSV if you need too</h2>
+          <button
+            aria-busy={state.loading}
+            disabled={state.loading}
+            onClick={() => {
+              if (!(surveyId && editionId)) {
+                return dispatch({
+                  type: "error",
+                  payload: new Error(
+                    "Can't trigger download without selecting surveyId && editionId"
+                  ),
+                });
+              }
+              if (!timestamp) {
+                return dispatch({
+                  type: "error",
+                  payload: new Error(
+                    "First generate an export before downloading it"
+                  ),
+                });
+              }
+              downloadExport({ surveyId, editionId, timestamp });
+            }}
+          >
+            {!state.loading ? "Download exports zip" : "Busy..."}
+            {state.error && <p>Error: {state.error.message}</p>}
+            {state.done && <p>Download will start shortly...</p>}
+          </button>
+          <p>
+            NOTE: if the downloaded file is a ".json" instead of ".zip" there
+            has been an error server-side.
+          </p>
+        </>
+      )}
+    </div>
+  );
+};

surveyadmin/src/app/admin/export/ImportData.tsx

@@ -0,0 +1,42 @@
+export const ImportData = ({
+  surveyId,
+  editionId,
+}: {
+  surveyId: string;
+  editionId: string;
+}) => {
+  return (
+    <div>
+      <h1>Import data from an external source</h1>
+      <p>
+        Advanced data analysis algorithm can be run on the exported/flattened
+        CSV data.{" "}
+      </p>
+      <p>This interface allows importing the result of this analysis.</p>
+      <ul>
+        <li>
+          The CSV file must have a `responseId` header, which will be used to
+          reconcile data. It corresponds to the _id of the original response
+          (NOT the _id of the exported normalized response)
+        </li>
+        <li>
+          Fields that already exist in the "normalized response" collection will
+          be ignored. We can only add fields, not update existing ones.
+        </li>
+      </ul>
+      <p>WORK IN PROGRESS</p>
+      {/**
+       *
+       * TODO:
+       * - allow to upload a CSV file
+       * - fill a new collection
+       * - make sure we use the right editionId/surveyId?
+       * they might not be in the normalizedResponse collection
+       * thus not in the exported data => we must be careful
+       * to select the right ids in this import interface
+       * - optionnaly add a button to allow merging those data
+       * in the normalizedResponse collection?
+       */}
+    </div>
+  );
+};

surveyadmin/src/app/admin/export/SurveyMarkdownOutline.tsx

@@ -0,0 +1,48 @@
+"use client";
+import { convertSurveyToMarkdown } from "~/lib/export/outlineExport";
+import React, { useState } from "react";
+import { EditionMetadata } from "@devographics/types";
+
+export const SurveyMarkdownOutline = ({
+  edition,
+}: {
+  edition: EditionMetadata;
+}) => {
+  const [showFieldName, setShowFieldName] = useState<boolean>(false);
+  // const intl = useIntlContext();
+  // TODO: filter for the current survey only, but we need a tag to do so
+  //const { data, loading, error } = useEntitiesQuery();
+
+  //if (loading) return <Loading />;
+  //if (error) return <span>Could not load entities</span>;
+  //if (!data) return <span>No entities found</span>;
+  //const { entities } = data;
+
+  return (
+    <div className="survey-section-wrapper">
+      <div>
+        <label htmlFor="fieldname">Show fieldName? (for CSV/JSON export)</label>
+        <input
+          type="checkbox"
+          id="fieldname"
+          name="fieldname"
+          onChange={(evt) => {
+            setShowFieldName(evt.target.checked);
+          }}
+        />
+      </div>
+      <textarea
+        style={{ width: 800, height: 600 }}
+        readOnly={true}
+        value={convertSurveyToMarkdown({
+          formatMessage: (key) => key, // intl.formatMessage,
+          edition,
+          entities: [],
+          options: {
+            showFieldName,
+          },
+        })}
+      />
+    </div>
+  );
+};