Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions AUTHORS
Original file line number Diff line number Diff line change
Expand Up @@ -8,3 +8,4 @@ Authors of Antragsgrün in chronological order of first contribution:
Antoine Tifine <antoine.tifine@posteo.net> (French translation)
Mick van Breukelen <mick.vanbreukelen@volteuropa.org> (Dutch translation)
Danilo Boskovic <secretary.general@fmura.me> (Montenegrin translation)
Nils (niStee) <52573120+niStee@users.noreply.github.com> (Development)
124 changes: 124 additions & 0 deletions components/UserGroupAdminMethods.php
Original file line number Diff line number Diff line change
Expand Up @@ -665,4 +665,128 @@ public function addUsersByEmail(): void
}
}
}

/**
* @param resource $fp
* @param array<string, int> $headerMap
* @return array{processedRows: int, errors: string[]}
*/
public function processCsvChunk($fp, array $headerMap, string $collisionBehavior, bool $sendEmail, string $emailText): array
{
$processedRows = 0;
$errors = [];
$maxRowsPerChunk = 50;

while ($processedRows < $maxRowsPerChunk && ($row = fgetcsv($fp, escape: '\\')) !== false) {
if (empty(array_filter($row))) {
continue; // Skip empty rows
}

$processedRows++;

try {
$email = isset($headerMap['email'], $row[$headerMap['email']]) ? trim($row[$headerMap['email']]) : '';
if ($email === '') {
$errors[] = 'Missing email on row.';
continue;
}

$firstName = isset($headerMap['first_name'], $row[$headerMap['first_name']]) ? trim($row[$headerMap['first_name']]) : '';
$lastName = isset($headerMap['last_name'], $row[$headerMap['last_name']]) ? trim($row[$headerMap['last_name']]) : '';
$organization = isset($headerMap['organization'], $row[$headerMap['organization']]) ? trim($row[$headerMap['organization']]) : '';

$name = trim($firstName . ' ' . $lastName);
if ($name === '') {
$name = $email;
}

/** @var ConsultationUserGroup[] $userGroups */
$userGroups = [];
if (isset($headerMap['groups']) && !empty($row[$headerMap['groups']])) {
$groupNames = array_map('trim', explode(',', $row[$headerMap['groups']]));
foreach ($groupNames as $groupName) {
if ($groupName === '') continue;
$group = ConsultationUserGroup::find()
->where(['title' => $groupName])
->orWhere(['externalId' => $groupName])
->one();
if ($group) {
$userGroups[] = $group;
} else {
$errors[] = "$email: Warning: Group '$groupName' not found.";
}
}
}
if (empty($userGroups)) {
$defaultGroup = $this->getDefaultUserGroup();
if ($defaultGroup !== null) {
$userGroups[] = $defaultGroup;
}
}

$user = User::findOne(['email' => $email]);

if ($user) {
if ($collisionBehavior === 'skip') {
continue;
}

if ($firstName !== '' || $lastName !== '') {
$user->nameGiven = $firstName;
$user->nameFamily = $lastName;
$user->name = $name;
}
if ($organization !== '') {
$user->organization = $organization;
}
$user->save(false);

if ($collisionBehavior === 'replace') {
$user->unlinkAll('userGroups', true);
foreach ($userGroups as $group) {
$user->link('userGroups', $group);
$this->logUserGroupAdd($user, $group);
}
} elseif ($collisionBehavior === 'merge') {
$existingGroupIds = array_map(fn($g) => $g->id, $user->userGroups);
foreach ($userGroups as $group) {
if (!in_array($group->id, $existingGroupIds)) {
$user->link('userGroups', $group);
$this->logUserGroupAdd($user, $group);
}
}
}
} else {
$auth = User::AUTH_EMAIL . ':' . mb_strtolower($email);
$user = new User();
$user->auth = $auth;
$user->email = mb_strtolower($email);
$user->name = $name;
if ($firstName !== '') $user->nameGiven = $firstName;
if ($lastName !== '') $user->nameFamily = $lastName;
if ($organization !== '') $user->organization = $organization;
$user->pwdEnc = password_hash(User::createPassword(), PASSWORD_DEFAULT);
$user->status = User::STATUS_CONFIRMED;
$user->emailConfirmed = 1;
$user->organizationIds = '';
$user->save(false);

foreach ($userGroups as $group) {
$user->link('userGroups', $group);
$this->logUserGroupAdd($user, $group);
}
if ($sendEmail) {
$this->sendWelcomeEmail($user, $emailText, null);
}
}
} catch (\Exception $e) {
$errors[] = $email . ': ' . $e->getMessage();
}
}

return [
'processedRows' => $processedRows,
'errors' => $errors
];
}
}
2 changes: 1 addition & 1 deletion config/urls.php
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@
$adminMotionListPaths = 'index|motion-excellist|motion-odslist|motion-pdfziplist';
$adminMotionListPaths .= '|motion-odtziplist|motion-odslistall|motion-odtall|motion-openslides|motion-comments-xlsx';
$adminAmendmentPaths = 'excellist|odslist|odslist-short|xlsx-list|pdflist|pdfziplist|odtziplist|openslides';
$adminUserPaths = 'save|poll|add-single-init|add-single|add-multiple-ww|add-multiple-email|search-groups';
$adminUserPaths = 'save|poll|add-single-init|add-single|add-multiple-ww|add-multiple-email|search-groups|upload-csv-init|process-csv-chunk';
$adminPaths = 'consultation|appearance|translation|translation-motion-type|siteaccess|siteconsultations|openslidesusers';
$adminPaths .= '|theming|files|proposed-procedure|ods-proposed-procedure|check-updates|goto-update';
$adminPpPaths = 'index-ajax|ods|save-motion-comment|save-amendment-comment|save-motion-visible|save-amendment-visible|save-responsibility|save-tags';
Expand Down
88 changes: 88 additions & 0 deletions controllers/admin/UsersController.php
Original file line number Diff line number Diff line change
Expand Up @@ -309,4 +309,92 @@ public function actionPoll(): RestApiResponse

return new RestApiResponse(200, $this->getUsersWidgetData($consultation));
}
public function actionUploadCsvInit(): JsonResponse
{
$this->getConsultationAndCheckAdminPermission();

if (empty($_FILES['csvFile']['tmp_name'])) {
return new JsonResponse(['success' => false, 'error' => 'No file uploaded.']);
}

$tmpName = $_FILES['csvFile']['tmp_name'];
$token = uniqid('csv_', true);
$filePath = AntragsgruenApp::getInstance()->getTmpDir() . '/' . $token . '.csv';

if (!move_uploaded_file($tmpName, $filePath)) {
return new JsonResponse(['success' => false, 'error' => 'Failed to save uploaded file.']);
}

$size = filesize($filePath);

return new JsonResponse([
'success' => true,
'token' => $token,
'totalSize' => $size,
'startOffset' => 0
]);
}

public function actionProcessCsvChunk(): JsonResponse
{
$this->getConsultationAndCheckAdminPermission();

$token = $this->getPostValue('token', '');
$offset = (int) $this->getPostValue('offset', 0);
$collisionBehavior = $this->getPostValue('collisionBehavior', 'skip');
$sendEmail = (bool) $this->getPostValue('sendEmail', false);
$emailText = $this->getPostValue('emailText', '');

if (!$token || !preg_match('/^csv_[a-zA-Z0-9.]+$/', $token)) {
return new JsonResponse(['success' => false, 'error' => 'Invalid token.']);
}

$filePath = AntragsgruenApp::getInstance()->getTmpDir() . '/' . $token . '.csv';
if (!file_exists($filePath)) {
return new JsonResponse(['success' => false, 'error' => 'File not found or expired.']);
}

$fp = fopen($filePath, 'r');
if ($fp === false) {
return new JsonResponse(['success' => false, 'error' => 'Could not open file.']);
}

// Always read the header first to establish mappings
$header = fgetcsv($fp, escape: '\\');
if ($header === false) {
return new JsonResponse(['success' => false, 'error' => 'File is empty.']);
}

// Strip BOM from first column name if present
$header[0] = preg_replace('/^\xEF\xBB\xBF/', '', (string) $header[0]);
$headerMap = array_flip(array_map('trim', array_map(function($v) { return strtolower((string) $v); }, $header)));

if (!isset($headerMap['email'])) {
fclose($fp);
@unlink($filePath);
return new JsonResponse(['success' => false, 'error' => 'CSV is missing the required "email" column.']);
}

if ($offset !== 0) {
fseek($fp, $offset);
}

$result = $this->userGroupAdminMethods->processCsvChunk($fp, $headerMap, $collisionBehavior, $sendEmail, $emailText);
$nextOffset = ftell($fp);
$finished = feof($fp) || $result['processedRows'] === 0;

fclose($fp);

if ($finished) {
@unlink($filePath);
}

return new JsonResponse([
'success' => true,
'nextOffset' => $nextOffset,
'processed' => $result['processedRows'],
'errors' => $result['errors'],
'finished' => $finished
]);
}
}
4 changes: 4 additions & 0 deletions docs/sample_user_import.csv
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
email,first_name,last_name,organization,groups
ada.lovelace@example.com,Ada,Lovelace,Volt Europa,"Delegates"
alan.turing@example.com,Alan,Turing,,
grace.hopper@example.com,Grace,Hopper,Volt Germany,"Admin, Delegates"
Comment thread
niStee marked this conversation as resolved.
50 changes: 50 additions & 0 deletions messages/de/admin.php
Original file line number Diff line number Diff line change
Expand Up @@ -892,4 +892,54 @@
'text' => 'Soll der Update-Modus aktiviert werden? Während der Update-Modus aktiv ist, ist diese Antragsgrün-Version nicht verfügbar.',
'js' => true,
],
'csv_uploading' => [
'text' => 'Datei wird hochgeladen...',
'js' => true,
],
'csv_init_failed' => [
'text' => 'Fehler bei der Initialisierung des Uploads',
'js' => true,
],
'csv_processing' => [
'text' => 'Verarbeitung... ({percent}%)',
'js' => true,
],
'csv_chunk_failed' => [
'text' => 'Fehler bei der Verarbeitung',
'js' => true,
],
'csv_success' => [
'text' => 'Erfolgreich {processedRows} Benutzer verarbeitet.',
'js' => true,
],
'csv_errors' => [
'text' => '<strong>Gefundene Fehler:</strong><br>',
'js' => true,
],
'csv_errors_count' => [
'text' => ' {errorCount} Fehler gefunden.',
'js' => true,
],
'csv_reload' => [
'text' => ' Du kannst die Seite neu laden, um sie zu sehen.',
'js' => true,
],
'csv_error_prefix' => [
'text' => 'Fehler: {message}',
'js' => true,
],
'csv_failed' => [
'text' => 'Import fehlgeschlagen.',
'js' => true,
],
'user_csv_upload_info' => 'Lade eine CSV-Datei mit den folgenden Spalten hoch: <code>email, first_name, last_name, organization, groups</code>. (Nur <code>email</code> ist zwingend erforderlich. <code>groups</code> kann eine kommagetrennte Liste von Gruppennamen oder externen IDs sein).',
'user_csv_label_file' => 'CSV-Datei:',
'user_csv_label_collision' => 'Bestehende Benutzer:',
'user_csv_collision_skip' => 'Bestehende Benutzer überspringen',
'user_csv_collision_merge' => 'Benutzer aktualisieren und neue Gruppen hinzufügen',
'user_csv_collision_replace' => 'Benutzer aktualisieren und Gruppen ersetzen',
'user_csv_label_email' => 'Willkommens-E-Mail:',
'user_csv_send_email' => 'Willkommens-E-Mail an NEUE Benutzer senden',
'user_csv_submit' => 'CSV hochladen und verarbeiten',
'user_csv_progress_init' => 'Verarbeitung...',
];
50 changes: 50 additions & 0 deletions messages/en/admin.php
Original file line number Diff line number Diff line change
Expand Up @@ -1084,4 +1084,54 @@
'text' => 'Do you want to enable the update mode? While the update mode is active, this installation of Antragsgrün will be unavailable for all users.',
'js' => true,
],
'csv_uploading' => [
'text' => 'Uploading file...',
'js' => true,
],
'csv_init_failed' => [
'text' => 'Failed to initialize upload',
'js' => true,
],
'csv_processing' => [
'text' => 'Processing... ({percent}%)',
'js' => true,
],
'csv_chunk_failed' => [
'text' => 'Failed to process chunk',
'js' => true,
],
'csv_success' => [
'text' => 'Successfully processed {processedRows} users.',
'js' => true,
],
'csv_errors' => [
'text' => '<strong>Errors encountered:</strong><br>',
'js' => true,
],
'csv_errors_count' => [
'text' => ' Encountered {errorCount} errors.',
'js' => true,
],
'csv_reload' => [
'text' => ' You can reload the page to see them.',
'js' => true,
],
'csv_error_prefix' => [
'text' => 'Error: {message}',
'js' => true,
],
'csv_failed' => [
'text' => 'Import failed.',
'js' => true,
],
'user_csv_upload_info' => 'Upload a CSV file with the following columns: <code>email, first_name, last_name, organization, groups</code>. (Only <code>email</code> is strictly required. <code>groups</code> can be a comma-separated list of group names or external IDs).',
'user_csv_label_file' => 'CSV File:',
'user_csv_label_collision' => 'Existing Users:',
'user_csv_collision_skip' => 'Skip existing users',
'user_csv_collision_merge' => 'Update user and merge new groups',
'user_csv_collision_replace' => 'Update user and replace groups',
'user_csv_label_email' => 'Welcome Email:',
'user_csv_send_email' => 'Send welcome email to NEW users',
'user_csv_submit' => 'Upload and Process CSV',
'user_csv_progress_init' => 'Processing...',
];
31 changes: 31 additions & 0 deletions tests/Acceptance/admin/UserAdminCsvImportCept.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
<?php

/** @var \Codeception\Scenario $scenario */
use Tests\Support\AcceptanceTester;

$I = new AcceptanceTester($scenario);
$I->populateDBData1();

$I->wantTo('Test CSV User Import JS Frontend Logic');
$I->loginAndGotoStdAdminPage()->gotoUserAdministration();

// Ensure the form is accessible via the opener
$I->clickJS('.addUsersOpener.csv');

// Form and elements should be visible
$I->seeElement('#csvImportForm');
$I->seeElement('#csvSubmitBtn');

// Progress container should be hidden initially
$I->dontSeeElement('#csvProgressContainer:not(.hidden)');

// Remove the required attribute so we can submit the form without an actual file,
// triggering the JS submit event instead of HTML5 validation
$I->executeJS("document.querySelector('input[name=\"csvFile\"]').removeAttribute('required');");

// Click the submit button
$I->clickJS('#csvSubmitBtn');

// The JS logic should catch the submit, prevent default, and show the progress container
$I->wait(0.5);
$I->seeElement('#csvProgressContainer:not(.hidden)');
Loading
Loading