Skip to content

Commit 23d19d9

Browse files
author
Harry Bragg
authored
add a limit to the number of concurrent processes (#9)
* limit the number of concurrent processes * use 0.6.1 so table renders nicely * clear summary on completion
1 parent 33b2da2 commit 23d19d9

11 files changed

Lines changed: 400 additions & 82 deletions

File tree

‎composer.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@
3030
"squizlabs/php_codesniffer": "^2.9",
3131
"graze/standards": "^1.0",
3232
"symfony/console": "^3.1",
33-
"graze/console-diff-renderer": "^0.6",
33+
"graze/console-diff-renderer": "^0.6.1",
3434
"mockery/mockery": "^0.9.9"
3535
},
3636
"suggest": {

src/Exceptions/AlreadyRunningException.php renamed to src/Exceptions/NotRunningException.php

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,6 @@
44

55
use RuntimeException;
66

7-
class AlreadyRunningException extends RuntimeException
7+
class NotRunningException extends RuntimeException
88
{
99
}

‎src/Pool.php‎

Lines changed: 100 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -14,24 +14,29 @@
1414
namespace Graze\ParallelProcess;
1515

1616
use Graze\DataStructure\Collection\Collection;
17-
use Graze\ParallelProcess\Exceptions\AlreadyRunningException;
17+
use Graze\ParallelProcess\Exceptions\NotRunningException;
1818
use InvalidArgumentException;
1919
use Symfony\Component\Process\Process;
2020

2121
class Pool extends Collection implements RunInterface
2222
{
2323
const CHECK_INTERVAL = 0.1;
24+
const NO_MAX = -1;
2425

2526
/** @var RunInterface[] */
2627
protected $items = [];
2728
/** @var RunInterface[] */
2829
protected $running = [];
30+
/** @var RunInterface[] */
31+
protected $waiting = [];
2932
/** @var callable|null */
3033
protected $onSuccess;
3134
/** @var callable|null */
3235
protected $onFailure;
3336
/** @var callable|null */
3437
protected $onProgress;
38+
/** @var int */
39+
private $maxSimultaneous = -1;
3540

3641
/**
3742
* Pool constructor.
@@ -42,18 +47,21 @@ class Pool extends Collection implements RunInterface
4247
* @param callable|null $onSuccess function (Process $process, float $duration, string $last) : void
4348
* @param callable|null $onFailure function (Process $process, float $duration, string $last) : void
4449
* @param callable|null $onProgress function (Process $process, float $duration, string $last) : void
50+
* @param int $maxSimultaneous
4551
*/
4652
public function __construct(
4753
array $items = [],
4854
callable $onSuccess = null,
4955
callable $onFailure = null,
50-
callable $onProgress = null
56+
callable $onProgress = null,
57+
$maxSimultaneous = self::NO_MAX
5158
) {
5259
parent::__construct($items);
5360

5461
$this->onSuccess = $onSuccess;
5562
$this->onFailure = $onFailure;
5663
$this->onProgress = $onProgress;
64+
$this->maxSimultaneous = $maxSimultaneous;
5765
}
5866

5967
/**
@@ -106,15 +114,14 @@ public function add($item)
106114
throw new InvalidArgumentException("add: Can only add `RunInterface` to this collection");
107115
}
108116

109-
$itemRunning = $item->isRunning();
110-
if ((count($this->running) > 0) && !$itemRunning) {
111-
throw new AlreadyRunningException("add: unable to add an item when the pool is currently running");
117+
if (!$this->isRunning() && $item->isRunning()) {
118+
throw new NotRunningException("add: unable to add a running item when the pool has not started");
112119
}
113120

114121
parent::add($item);
115122

116-
if ($itemRunning) {
117-
$this->running[] = $item;
123+
if ($this->isRunning()) {
124+
$this->startRun($item);
118125
}
119126

120127
return $this;
@@ -145,14 +152,27 @@ protected function addProcess(Process $process)
145152
public function start()
146153
{
147154
foreach ($this->items as $run) {
148-
$run->start();
155+
$this->startRun($run);
149156
}
150157

151-
$this->running = $this->items;
152-
153158
return $this;
154159
}
155160

161+
/**
162+
* Start a run (or queue it if we are running the maximum number of processes already)
163+
*
164+
* @param RunInterface $run
165+
*/
166+
private function startRun(RunInterface $run)
167+
{
168+
if ($this->maxSimultaneous === static::NO_MAX || count($this->running) < $this->maxSimultaneous) {
169+
$run->start();
170+
$this->running[] = $run;
171+
} else {
172+
$this->waiting[] = $run;
173+
}
174+
}
175+
156176
/**
157177
* Blocking call to run processes;
158178
*
@@ -163,14 +183,31 @@ public function start()
163183
public function run($checkInterval = self::CHECK_INTERVAL)
164184
{
165185
$this->start();
186+
$interval = $checkInterval * 1000000;
166187

167-
while ($this->isRunning()) {
168-
usleep($checkInterval * 1000000);
188+
while ($this->poll()) {
189+
usleep($interval);
169190
}
170191

171192
return $this->isSuccessful();
172193
}
173194

195+
/**
196+
* Check when a run has finished, if there are processes waiting, start them
197+
*/
198+
private function checkFinished()
199+
{
200+
if ($this->maxSimultaneous !== static::NO_MAX
201+
&& count($this->waiting) > 0
202+
&& count($this->running) < $this->maxSimultaneous) {
203+
for ($i = count($this->running); $i < $this->maxSimultaneous && count($this->waiting) > 0; $i++) {
204+
$run = array_shift($this->waiting);
205+
$run->start();
206+
$this->running[] = $run;
207+
}
208+
}
209+
}
210+
174211
/**
175212
* Determine if any item has run
176213
*
@@ -191,13 +228,23 @@ public function hasStarted()
191228
*
192229
* @return bool
193230
*/
194-
public function isRunning()
231+
public function poll()
195232
{
196233
/** @var Run[] $running */
197234
$this->running = array_filter($this->running, function (RunInterface $run) {
198-
return $run->isRunning();
235+
return $run->poll();
199236
});
200237

238+
$this->checkFinished();
239+
240+
return $this->isRunning();
241+
}
242+
243+
/**
244+
* @return bool
245+
*/
246+
public function isRunning()
247+
{
201248
return count($this->running) > 0;
202249
}
203250

@@ -220,4 +267,43 @@ public function isSuccessful()
220267

221268
return true;
222269
}
270+
271+
/**
272+
* Get a list of all the currently running runs
273+
*
274+
* @return RunInterface[]
275+
*/
276+
public function getRunning()
277+
{
278+
return $this->running;
279+
}
280+
281+
/**
282+
* Get a list of all the current waiting runs
283+
*
284+
* @return RunInterface[]
285+
*/
286+
public function getWaiting()
287+
{
288+
return $this->waiting;
289+
}
290+
291+
/**
292+
* @return int
293+
*/
294+
public function getMaxSimultaneous()
295+
{
296+
return $this->maxSimultaneous;
297+
}
298+
299+
/**
300+
* @param int $maxSimultaneous
301+
*
302+
* @return $this
303+
*/
304+
public function setMaxSimultaneous($maxSimultaneous)
305+
{
306+
$this->maxSimultaneous = $maxSimultaneous;
307+
return $this;
308+
}
223309
}

‎src/Run.php‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22

33
namespace Graze\ParallelProcess;
44

5-
use Graze\ParallelProcess\Exceptions\AlreadyRunningException;
65
use Symfony\Component\Process\Process;
76

87
class Run implements RunInterface
@@ -61,8 +60,6 @@ public function start()
6160
});
6261
$this->started = microtime(true);
6362
$this->completed = false;
64-
} else {
65-
throw new AlreadyRunningException("start: this run is already running");
6663
}
6764

6865
return $this;
@@ -73,7 +70,7 @@ public function start()
7370
*
7471
* @return bool true if the process is currently running (started and not terminated)
7572
*/
76-
public function isRunning()
73+
public function poll()
7774
{
7875
if ($this->completed || !$this->hasStarted()) {
7976
return false;
@@ -95,6 +92,16 @@ public function isRunning()
9592
return false;
9693
}
9794

95+
/**
96+
* Return if the underlying process is running
97+
*
98+
* @return bool
99+
*/
100+
public function isRunning()
101+
{
102+
return $this->process->isRunning();
103+
}
104+
98105
/**
99106
* Call an event callback
100107
*

‎src/RunInterface.php‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ public function hasStarted();
1818
*
1919
* @return $this
2020
*
21-
* @throws \Graze\ParallelProcess\Exceptions\AlreadyRunningException
21+
* @throws \Graze\ParallelProcess\Exceptions\NotRunningException
2222
*/
2323
public function start();
2424

@@ -30,9 +30,16 @@ public function start();
3030
public function isSuccessful();
3131

3232
/**
33-
* Polls to see if this process is running
33+
* We think this is running
3434
*
3535
* @return bool
3636
*/
3737
public function isRunning();
38+
39+
/**
40+
* Pools to see if this process is running
41+
*
42+
* @return bool
43+
*/
44+
public function poll();
3845
}

‎src/Table.php‎

Lines changed: 46 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,8 @@ class Table
3838
private $terminal;
3939
/** @var bool */
4040
private $showOutput = true;
41+
/** @var bool */
42+
private $showSummary = true;
4143

4244
/**
4345
* Table constructor.
@@ -141,6 +143,27 @@ function ($process, $duration, $last) use ($index, $data) {
141143
$this->updateRowKeyLengths($data);
142144
}
143145

146+
/**
147+
* @return string
148+
*/
149+
private function getSummary()
150+
{
151+
if ($this->processPool->hasStarted()) {
152+
if ($this->processPool->isRunning()) {
153+
return sprintf(
154+
'<comment>Total</comment>: %2d, <comment>Running</comment>: %2d, <comment>Waiting</comment>: %2d',
155+
$this->processPool->count(),
156+
count($this->processPool->getRunning()),
157+
count($this->processPool->getWaiting())
158+
);
159+
} else {
160+
return '';
161+
}
162+
} else {
163+
return 'waiting...';
164+
}
165+
}
166+
144167
/**
145168
* Render a specific row
146169
*
@@ -149,7 +172,8 @@ function ($process, $duration, $last) use ($index, $data) {
149172
private function render($row = 0)
150173
{
151174
if ($this->output->getVerbosity() >= OutputInterface::VERBOSITY_VERBOSE) {
152-
$this->output->reWrite($this->rows, true);
175+
$rows = ($this->showSummary ? array_merge($this->rows, [$this->getSummary()]) : $this->rows);
176+
$this->output->reWrite($rows, !$this->showSummary);
153177
} else {
154178
$this->output->writeln($this->rows[$row]);
155179
}
@@ -167,6 +191,9 @@ public function run($checkInterval = Pool::CHECK_INTERVAL)
167191
$this->render();
168192
}
169193
$output = $this->processPool->run($checkInterval);
194+
if ($this->output->getVerbosity() >= OutputInterface::VERBOSITY_VERBOSE && $this->showSummary) {
195+
$this->render();
196+
}
170197

171198
if (count($this->exceptions) > 0) {
172199
foreach ($this->exceptions as $exception) {
@@ -195,7 +222,25 @@ public function isShowOutput()
195222
public function setShowOutput($showOutput)
196223
{
197224
$this->showOutput = $showOutput;
225+
return $this;
226+
}
198227

228+
/**
229+
* @return bool
230+
*/
231+
public function isShowSummary()
232+
{
233+
return $this->showSummary;
234+
}
235+
236+
/**
237+
* @param bool $showSummary
238+
*
239+
* @return $this
240+
*/
241+
public function setShowSummary($showSummary)
242+
{
243+
$this->showSummary = $showSummary;
199244
return $this;
200245
}
201246
}

‎tests/example/app.php‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,9 @@
2121

2222
$output = new ConsoleOutput(ConsoleOutput::VERBOSITY_VERY_VERBOSE);
2323

24-
$table = new Table($output);
24+
$pool = new \Graze\ParallelProcess\Pool();
25+
$pool->setMaxSimultaneous(3);
26+
$table = new Table($output, $pool);
2527
for ($i = 0; $i < 5; $i++) {
2628
$time = $i + 5;
2729
$table->add(new Process(sprintf('for i in `seq 1 %d` ; do date ; sleep 1 ; done', $time)), ['sleep' => $time]);

0 commit comments

Comments
 (0)