Skip to content

Commit 822d8b7

Browse files
anvitanvit
authored
Add new CSV Validator tests
Add CSV Validator tests for: - Maximum information object and accession title lengths - Event Date format validation
1 parent a77c41c commit 822d8b7

File tree

8 files changed

+577
-1
lines changed

8 files changed

+577
-1
lines changed

‎lib/task/import/validate/csvAccessionTitleLengthValidator.php‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
<?php
2+
3+
/*
4+
* This file is part of the Access to Memory (AtoM) software.
5+
*
6+
* Access to Memory (AtoM) is free software: you can redistribute it and/or modify
7+
* it under the terms of the GNU Affero General Public License as published by
8+
* the Free Software Foundation, either version 3 of the License, or
9+
* (at your option) any later version.
10+
*
11+
* Access to Memory (AtoM) is distributed in the hope that it will be useful,
12+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
13+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14+
* GNU General Public License for more details.
15+
*
16+
* You should have received a copy of the GNU General Public License
17+
* along with Access to Memory (AtoM). If not, see <http://www.gnu.org/licenses/>.
18+
*/
19+
20+
/**
21+
* CSV length test for Accessions.
22+
*
23+
* @author Anvit Srivastav <asrivastav@artefactual.com>
24+
*/
25+
class CsvAccessionTitleLengthValidator extends CsvFieldLengthValidator
26+
{
27+
public const LIMIT_TO = ['QubitAccession'];
28+
29+
protected $fieldMaxSizes = [
30+
'title' => 255,
31+
];
32+
33+
protected $fieldAction = [
34+
'title' => CsvValidatorResult::RESULT_ERROR,
35+
];
36+
}

‎lib/task/import/validate/csvEventDateValidator.class.php‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
<?php
2+
3+
/*
4+
* This file is part of the Access to Memory (AtoM) software.
5+
*
6+
* Access to Memory (AtoM) is free software: you can redistribute it and/or modify
7+
* it under the terms of the GNU Affero General Public License as published by
8+
* the Free Software Foundation, either version 3 of the License, or
9+
* (at your option) any later version.
10+
*
11+
* Access to Memory (AtoM) is distributed in the hope that it will be useful,
12+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
13+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14+
* GNU General Public License for more details.
15+
*
16+
* You should have received a copy of the GNU General Public License
17+
* along with Access to Memory (AtoM). If not, see <http://www.gnu.org/licenses/>.
18+
*/
19+
20+
/**
21+
* CSV event date validity test. Checks if eventStartDates and eventEndDates
22+
* all contain a permissible date format.
23+
*
24+
* @author Anvit Srivastav <asrivastav@artefactual.com>
25+
*/
26+
class CsvEventDateValidator extends CsvBaseValidator
27+
{
28+
public const TITLE = 'Event Date Check';
29+
public const LIMIT_TO = ['QubitInformationObject'];
30+
31+
protected $invalidEventDates = [];
32+
protected $rowsWithInvalidDates = 0;
33+
34+
public function __construct(?array $options = null)
35+
{
36+
$this->setTitle(self::TITLE);
37+
parent::__construct($options);
38+
}
39+
40+
public function reset()
41+
{
42+
$this->invalidEventDates = [];
43+
$this->rowsWithInvalidDates = 0;
44+
45+
parent::reset();
46+
}
47+
48+
public function testRow(array $header, array $row)
49+
{
50+
$invalidDateFound = false;
51+
if (!parent::testRow($header, $row)) {
52+
return;
53+
}
54+
55+
$row = $this->combineRow($header, $row);
56+
57+
if (false !== strpos($row['eventStartDates'], '|')) {
58+
foreach (explode('|', $row['eventStartDates']) as $date) {
59+
if (!$this->checkDate($date)) {
60+
$this->invalidEventDates[] = $date;
61+
$invalidDateFound = true;
62+
}
63+
}
64+
} elseif (!$this->checkDate($row['eventStartDates'])) {
65+
$this->invalidEventDates[] = $row['eventStartDates'];
66+
$invalidDateFound = true;
67+
}
68+
69+
if (false !== strpos($row['eventEndDates'], '|')) {
70+
foreach (explode('|', $row['eventEndDates']) as $date) {
71+
if (!$this->checkDate($date)) {
72+
$this->invalidEventDates[] = $date;
73+
$invalidDateFound = true;
74+
}
75+
}
76+
} elseif (!$this->checkDate($row['eventEndDates'])) {
77+
$this->invalidEventDates[] = $row['eventEndDates'];
78+
$invalidDateFound = true;
79+
}
80+
81+
if ($invalidDateFound) {
82+
++$this->rowsWithInvalidDates;
83+
$this->appendToCsvRowList();
84+
}
85+
}
86+
87+
public function getTestResult()
88+
{
89+
if (0 < $this->rowsWithInvalidDates) {
90+
$this->testData->setStatusWarn();
91+
$this->testData->addResult(sprintf('Rows with invalid event date values: %s', $this->rowsWithInvalidDates));
92+
} else {
93+
$this->testData->addResult(sprintf("All ''eventStartDates' and 'eventEndDates' columns contain dates in a valid format."));
94+
}
95+
96+
if (!empty($this->getCsvRowList())) {
97+
$this->testData->addDetail(sprintf('CSV row numbers where issues were found: %s', implode(', ', $this->getCsvRowList())));
98+
$this->testData->addDetail(sprintf('Listing invalid date values: "%s"', implode(', ', $this->invalidEventDates)));
99+
}
100+
101+
return parent::getTestResult();
102+
}
103+
104+
protected function checkDate(?string $eventDate)
105+
{
106+
// Blank entries are valid
107+
if (empty($eventDate)) {
108+
return true;
109+
}
110+
111+
// Check for YYYY-MM-DD/YYYYMMDD format for dates
112+
$date = trim($eventDate);
113+
if (preg_match('/^[0-9]{4}-?(0[1-9]|1[0-2])-?(0[1-9]|[1-2][0-9]|3[0-1])(\s*)$/', $date)) {
114+
return true;
115+
}
116+
117+
// Check for YYYY-MM format for dates
118+
if (preg_match('/^[0-9]{4}-(0[1-9]|1[0-2])(\s*)$/', $date)) {
119+
return true;
120+
}
121+
122+
// Check for YYYY format for dates
123+
if (preg_match('/^[0-9]{4}(\s*)$/', $date)) {
124+
return true;
125+
}
126+
127+
return false;
128+
}
129+
}

‎lib/task/import/validate/csvFieldLengthValidator.php‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,9 @@ public function getTestResult()
113113

114114
foreach ($this->columnsFound as $columnName => $errorCount) {
115115
if (0 < $errorCount) {
116-
$this->testData->setStatusWarn();
116+
CsvValidatorResult::RESULT_WARN === $this->fieldAction[$columnName] ?
117+
$this->testData->setStatusWarn() :
118+
$this->testData->setStatusError();
117119
$this->testData->addResult(sprintf("'%s' column may have invalid values.", $columnName));
118120
}
119121

‎lib/task/import/validate/csvIOTitleLengthValidator.php‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
<?php
2+
3+
/*
4+
* This file is part of the Access to Memory (AtoM) software.
5+
*
6+
* Access to Memory (AtoM) is free software: you can redistribute it and/or modify
7+
* it under the terms of the GNU Affero General Public License as published by
8+
* the Free Software Foundation, either version 3 of the License, or
9+
* (at your option) any later version.
10+
*
11+
* Access to Memory (AtoM) is distributed in the hope that it will be useful,
12+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
13+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14+
* GNU General Public License for more details.
15+
*
16+
* You should have received a copy of the GNU General Public License
17+
* along with Access to Memory (AtoM). If not, see <http://www.gnu.org/licenses/>.
18+
*/
19+
20+
/**
21+
* CSV length test for Information Objects.
22+
*
23+
* @author Anvit Srivastav <asrivastav@artefactual.com>
24+
*/
25+
class CsvIOTitleLengthValidator extends CsvFieldLengthValidator
26+
{
27+
public const LIMIT_TO = ['QubitInformationObject'];
28+
29+
protected $fieldMaxSizes = [
30+
'title' => 1024,
31+
];
32+
33+
protected $fieldAction = [
34+
'title' => CsvValidatorResult::RESULT_ERROR,
35+
];
36+
}

‎lib/task/import/validate/csvValidatorCollection.php‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,9 @@ class CsvValidatorCollection
4242
'CsvDigitalObjectUriValidator' => CsvDigitalObjectUriValidator::class,
4343
'CsvScriptValidator' => CsvScriptValidator::class,
4444
'CsvRepoValidator' => CsvRepoValidator::class,
45+
'CsvIOTitleLengthValidator' => CsvIOTitleLengthValidator::class,
46+
'CsvAccessionTitleLengthValidator' => CsvAccessionTitleLengthValidator::class,
47+
'CsvEventDateValidator' => CsvEventDateValidator::class,
4548
];
4649

4750
protected $validators = [];

‎test/phpunit/csvImportValidator/CsvAccessionTitleLengthTest.php‎

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
<?php
2+
3+
use org\bovigo\vfs\vfsStream;
4+
5+
/**
6+
* @internal
7+
*
8+
* @covers \CsvAccessionTitleLengthValidator
9+
*/
10+
class CsvAccessionTitleLengthTest extends \PHPUnit\Framework\TestCase
11+
{
12+
protected $vdbcon;
13+
protected $context;
14+
protected $vfs;
15+
protected $csvData;
16+
protected $csvInvalidData;
17+
protected $csvHeader;
18+
19+
public function setUp(): void
20+
{
21+
$this->context = sfContext::getInstance();
22+
$this->vdbcon = $this->createMock(PropelPDO::class);
23+
24+
$this->csvHeader = 'accessionNumber,acquisitionDate,sourceOfAcquisition,title,culture';
25+
26+
$this->csvData = [
27+
'"","","","Valid Title","en"',
28+
'"","","","Another Valid Title","en"',
29+
];
30+
31+
$this->csvInvalidData = [
32+
'"","","","Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Aenean commodo ligula eget dolor. Aenean massa. Cum sociis natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Donec quam felis, ultricies nec, pellentesque eu, pretium quis s","en"',
33+
];
34+
35+
// define virtual file system
36+
$directory = [
37+
'unix_csv_valid.csv' => $this->csvHeader."\n".implode("\n", $this->csvData),
38+
'unix_csv_invalid.csv' => $this->csvHeader."\n".implode("\n", $this->csvInvalidData),
39+
];
40+
41+
$this->vfs = vfsStream::setup('root', null, $directory);
42+
}
43+
44+
/**
45+
* @dataProvider csvValidatorTestProvider
46+
*
47+
* Generic test - options and expected results from csvValidatorTestProvider()
48+
*
49+
* @param mixed $options
50+
*/
51+
public function testCsvValidator($options)
52+
{
53+
$filename = $this->vfs->url().$options['filename'];
54+
$validatorOptions = isset($options['validatorOptions']) ? $options['validatorOptions'] : null;
55+
56+
$validatorOptions['className'] = 'QubitAccession';
57+
58+
$csvValidator = new CsvImportValidator($this->context, null, $validatorOptions);
59+
$this->runValidator($csvValidator, $filename, $options['csvValidatorClasses']);
60+
$result = $csvValidator->getResultsByFilenameTestname($filename, $options['testname']);
61+
62+
$this->assertSame($options[CsvValidatorResult::TEST_TITLE], $result[CsvValidatorResult::TEST_TITLE]);
63+
$this->assertSame($options[CsvValidatorResult::TEST_STATUS], $result[CsvValidatorResult::TEST_STATUS]);
64+
$this->assertSame($options[CsvValidatorResult::TEST_RESULTS], $result[CsvValidatorResult::TEST_RESULTS]);
65+
$this->assertSame($options[CsvValidatorResult::TEST_DETAILS], $result[CsvValidatorResult::TEST_DETAILS]);
66+
}
67+
68+
public function csvValidatorTestProvider()
69+
{
70+
$vfsUrl = 'vfs://root';
71+
72+
return [
73+
/*
74+
* Test CsvAccessionTitleLengthValidator.class.php
75+
*
76+
* Tests:
77+
* - Valid title
78+
* - Title field with more than 255 characters
79+
*/
80+
[
81+
'CsvAccessionTitleLengthValidator-LengthCheckValid' => [
82+
'csvValidatorClasses' => 'CsvAccessionTitleLengthValidator',
83+
'filename' => '/unix_csv_valid.csv',
84+
'testname' => 'CsvAccessionTitleLengthValidator',
85+
CsvValidatorResult::TEST_TITLE => CsvAccessionTitleLengthValidator::TITLE,
86+
CsvValidatorResult::TEST_STATUS => CsvValidatorResult::RESULT_INFO,
87+
CsvValidatorResult::TEST_RESULTS => [
88+
'Checking columns: title',
89+
'\'title\' values that exceed 255 characters: 0',
90+
],
91+
CsvValidatorResult::TEST_DETAILS => [
92+
],
93+
],
94+
],
95+
[
96+
'CsvAccessionTitleLengthValidator-LengthCheckInvalid' => [
97+
'csvValidatorClasses' => 'CsvAccessionTitleLengthValidator',
98+
'filename' => '/unix_csv_invalid.csv',
99+
'testname' => 'CsvAccessionTitleLengthValidator',
100+
CsvValidatorResult::TEST_TITLE => CsvAccessionTitleLengthValidator::TITLE,
101+
CsvValidatorResult::TEST_STATUS => CsvValidatorResult::RESULT_ERROR,
102+
CsvValidatorResult::TEST_RESULTS => [
103+
'Checking columns: title',
104+
'\'title\' column may have invalid values.',
105+
'\'title\' values that exceed 255 characters: 1',
106+
],
107+
CsvValidatorResult::TEST_DETAILS => [
108+
'title column value: Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Aenean commodo ligula eget dolor. Aenean massa. Cum sociis natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Donec quam felis, ultricies nec, pellentesque eu, pretium quis s',
109+
],
110+
],
111+
],
112+
];
113+
}
114+
115+
// Generic Validation
116+
protected function runValidator($csvValidator, $filenames, $tests)
117+
{
118+
$csvValidator->setSpecificTests($tests);
119+
$csvValidator->setFilenames(explode(',', $filenames));
120+
121+
return $csvValidator->validate();
122+
}
123+
}

0 commit comments

Comments
 (0)