Skip to content

Commit 17166cc

Browse files
jtojnarjtojnar
committed
Use generators for database access
This will make it possible to GC memory when working with huge number of items.
1 parent 3d3df79 commit 17166cc

File tree

7 files changed

+38
-35
lines changed

7 files changed

+38
-35
lines changed

‎src/daos/ItemsInterface.php‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66

77
use DateTime;
88
use DateTimeImmutable;
9+
use Generator;
910

1011
/**
1112
* Interface describing concrete DAO for working with items.
@@ -184,9 +185,9 @@ public function bulkStatusUpdate(array $statuses): void;
184185
/**
185186
* returns raw items table contents
186187
*
187-
* @return array[] of all items
188+
* @return Generator<int, array, void, void> of all items
188189
*/
189-
public function getRaw(): array;
190+
public function getRaw(): Generator;
190191

191192
/**
192193
* inserts raw data into items table

‎src/daos/SourcesInterface.php‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44

55
namespace daos;
66

7+
use Generator;
8+
79
/**
810
* Interface describing concrete DAO for working with sources.
911
*/
@@ -110,9 +112,9 @@ public function checkIfExists(string $title, string $spout, array $params): int;
110112
/**
111113
* returns raw sources table contents
112114
*
113-
* @return array[] of all sources
115+
* @return Generator<int, array, void, void> of all sources
114116
*/
115-
public function getRaw(): array;
117+
public function getRaw(): Generator;
116118

117119
/**
118120
* inserts raw data into sources table

‎src/daos/StatementsInterface.php‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44

55
namespace daos;
66

7+
use Generator;
8+
79
/**
810
* Interface for class providing SQL helpers.
911
*/
@@ -100,16 +102,14 @@ public static function bool(bool $bool): string;
100102
public static function datetime(\DateTime $date): string;
101103

102104
/**
103-
* Ensure row values have the appropriate PHP type. This assumes we are
104-
* using buffered queries (sql results are in PHP memory);.
105+
* Ensure row values have the appropriate PHP type.
105106
*
106-
* @param array $rows array of associative array representing row results
107+
* @param iterable $rows array of associative array representing row results
107108
* @param array $expectedRowTypes associative array mapping columns to PDO types
108109
*
109-
* @return array of associative array representing row results having
110-
* expected types
110+
* @return Generator<int, array, void, void> of associative array representing row results having expected types
111111
*/
112-
public static function ensureRowTypes(array $rows, array $expectedRowTypes): array;
112+
public static function ensureRowTypes(iterable $rows, array $expectedRowTypes): Generator;
113113

114114
/**
115115
* convert string array to string for storage in table row

‎src/daos/mysql/Items.php‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
use daos\ItemOptions;
99
use DateTime;
1010
use DateTimeImmutable;
11+
use Generator;
1112
use helpers\Configuration;
1213
use function helpers\functions\map;
1314
use Monolog\Logger;
@@ -407,16 +408,16 @@ public function sync(int $sinceId, DateTime $notBefore, DateTime $since, int $ho
407408
* @return int lowest id of interest
408409
*/
409410
public function lowestIdOfInterest(): int {
410-
$lowest = static::$stmt::ensureRowTypes(
411+
$lowests = static::$stmt::ensureRowTypes(
411412
$this->database->exec(
412413
'SELECT id FROM ' . $this->configuration->dbPrefix . 'items AS items
413414
WHERE ' . static::$stmt::isTrue('unread') .
414415
' OR ' . static::$stmt::isTrue('starred') .
415416
' ORDER BY id LIMIT 1'),
416417
['id' => DatabaseInterface::PARAM_INT]
417418
);
418-
if ($lowest) {
419-
return $lowest[0]['id'];
419+
foreach ($lowests as $lowest) {
420+
return $lowest['id'];
420421
}
421422

422423
return 0;
@@ -667,7 +668,7 @@ public function bulkStatusUpdate(array $statuses): void {
667668
/**
668669
* {@inheritdoc}
669670
*/
670-
public function getRaw(): array {
671+
public function getRaw(): Generator {
671672
$stmt = static::$stmt;
672673
$items = $this->database->exec('SELECT * FROM ' . $this->configuration->dbPrefix . 'items');
673674

‎src/daos/mysql/Sources.php‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
namespace daos\mysql;
66

77
use daos\DatabaseInterface;
8+
use Generator;
89
use helpers\Configuration;
910
use function helpers\Functions\map;
1011

@@ -281,7 +282,7 @@ public function checkIfExists(string $title, string $spout, array $params): int
281282
/**
282283
* {@inheritdoc}
283284
*/
284-
public function getRaw(): array {
285+
public function getRaw(): Generator {
285286
$stmt = static::$stmt;
286287
$sources = $this->database->exec('SELECT * FROM ' . $this->configuration->dbPrefix . 'sources');
287288

‎src/daos/mysql/Statements.php‎

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
namespace daos\mysql;
66

77
use daos\DatabaseInterface;
8+
use Generator;
89

910
/**
1011
* MySQL specific statements
@@ -138,17 +139,15 @@ public static function datetime(\DateTime $date): string {
138139
}
139140

140141
/**
141-
* Ensure row values have the appropriate PHP type. This assumes we are
142-
* using buffered queries (sql results are in PHP memory).
142+
* Ensure row values have the appropriate PHP type.
143143
*
144-
* @param array $rows array of associative array representing row results
144+
* @param iterable $rows array of associative array representing row results
145145
* @param array $expectedRowTypes associative array mapping columns to PDO types
146146
*
147-
* @return array of associative array representing row results having
148-
* expected types
147+
* @return Generator<int, array, void, void> of associative array representing row results having expected types
149148
*/
150-
public static function ensureRowTypes(array $rows, array $expectedRowTypes): array {
151-
foreach ($rows as $rowIndex => $row) {
149+
public static function ensureRowTypes(iterable $rows, array $expectedRowTypes): Generator {
150+
foreach ($rows as $row) {
152151
foreach ($expectedRowTypes as $columnIndex => $type) {
153152
if (array_key_exists($columnIndex, $row)) {
154153
if ($type & DatabaseInterface::PARAM_NULL) {
@@ -185,13 +184,13 @@ public static function ensureRowTypes(array $rows, array $expectedRowTypes): arr
185184
$value = null;
186185
}
187186
if ($value !== null) {
188-
$rows[$rowIndex][$columnIndex] = $value;
187+
$row[$columnIndex] = $value;
189188
}
190189
}
191190
}
192-
}
193191

194-
return $rows;
192+
yield $row;
193+
}
195194
}
196195

197196
/**

‎src/daos/pgsql/Statements.php‎

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
namespace daos\pgsql;
66

77
use daos\DatabaseInterface;
8+
use Generator;
89

910
/**
1011
* PostgreSQL specific statements
@@ -74,17 +75,15 @@ public static function csvRowMatches(string $column, string $value): string {
7475
}
7576

7677
/**
77-
* Ensure row values have the appropriate PHP type. This assumes we are
78-
* using buffered queries (sql results are in PHP memory).
78+
* Ensure row values have the appropriate PHP type.
7979
*
80-
* @param array $rows array of associative array representing row results
80+
* @param iterable $rows array of associative array representing row results
8181
* @param array $expectedRowTypes associative array mapping columns to PDO types
8282
*
83-
* @return array of associative array representing row results having
84-
* expected types
83+
* @return Generator<int, array, void, void> of associative array representing row results having expected types
8584
*/
86-
public static function ensureRowTypes(array $rows, array $expectedRowTypes): array {
87-
foreach ($rows as $rowIndex => $row) {
85+
public static function ensureRowTypes(iterable $rows, array $expectedRowTypes): Generator {
86+
foreach ($rows as $row) {
8887
foreach ($expectedRowTypes as $columnIndex => $type) {
8988
if (array_key_exists($columnIndex, $row)) {
9089
if ($type & DatabaseInterface::PARAM_NULL) {
@@ -110,13 +109,13 @@ public static function ensureRowTypes(array $rows, array $expectedRowTypes): arr
110109
$value = null;
111110
}
112111
if ($value !== null) {
113-
$rows[$rowIndex][$columnIndex] = $value;
112+
$row[$columnIndex] = $value;
114113
}
115114
}
116115
}
117-
}
118116

119-
return $rows;
117+
yield $row;
118+
}
120119
}
121120

122121
/**

0 commit comments

Comments
 (0)