Async parallel

.env.example

@@ -46,7 +46,7 @@ VERSION_NUMBER="3.5.0"
 # Available options: "phpally", "equalaccess_local", "equalaccess_lambda"
 ACCESSIBILITY_CHECKER="equalaccess_lambda"
 
-# NOTE: When using a lambda function with equal access, 
+# NOTE: When using a lambda function with equal access,
 # you need to define the following in a separate .env.local:
 # EQUALACCESS_AWS_ACCESS_KEY_ID=<access_key_id>
 # EQUALACCESS_AWS_SECRET_ACCESS_KEY=<secret_access_key>
@@ -189,3 +189,9 @@ YOUTUBE_API_KEY=""
 ###> jwk base url ###
 JWK_BASE_URL="https://canvas.instructure.com/"
 ###> jwk base url ###
+
+###> Equal Access Local Num Browsers ###
+SCAN_POOL_SIZE=6
+
+###> Redis
+REDIS_URL=redis://udoit3-redis:6379

Dockerfile

@@ -44,6 +44,10 @@ COPY --from=composer:latest /usr/bin/composer /usr/local/bin/composer
 RUN wget https://get.symfony.com/cli/installer -O - | bash && \
     mv /root/.symfony/bin/symfony /usr/local/bin/symfony
 
+# ensures supervisor config is in place and supervisor is running
+COPY supervisor.conf /etc/supervisor/conf.d/messenger-worker.conf
+RUN service supervisor start
+
 #Copy over files
 COPY --chown=ssm-user:www-data . /var/www/html/
 
@@ -52,4 +56,5 @@ WORKDIR /var/www/html
 RUN chmod +x deploy/udoit-ng.sh
 RUN deploy/udoit-ng.sh
 
-CMD php-fpm
+#CMD php-fpm # runs supervisor which starts main application and messanger workers
+CMD ["/usr/bin/supervisord", "-n", "-c", "/etc/supervisor/supervisord.conf"]

assets/js/Components/Header.js

@@ -18,62 +18,62 @@ export default function Header({ t, navigation, handleNavigation }) {
           <ul>
             <li
               className={`flex-row ${navigation === 'summary' ? ' active-link' : ''}`}
-              onClick={()=>handleNavigation('summary')}
+              onClick={() => handleNavigation('summary')}
               onKeyPress={(e) => {
-                if(e.key === 'Enter' || e.key === ' ') {
+                if (e.key === 'Enter' || e.key === ' ') {
                   handleNavigation('summary')
                 }
               }}
               tabindex='0'>
               <div className='flex-column justify-content-center'>
-                <HomeIcon className='icon-md pr-1'/>
+                <HomeIcon className='icon-md pr-1' />
               </div>
               <div className='flex-column justify-content-center'>
                 {t('menu.summary')}
               </div></li>
-            <li 
+            <li
               className={`flex-row ${navigation === 'fixIssues' ? ' active-link' : ''}`}
-              onClick={()=>handleNavigation('fixIssues')}
+              onClick={() => handleNavigation('fixIssues')}
               onKeyPress={(e) => {
-                if(e.key === 'Enter' || e.key === ' ') {
+                if (e.key === 'Enter' || e.key === ' ') {
                   handleNavigation('fixIssues')
                 }
               }}
               tabindex='0'>
               <div className='flex-column justify-content-center'>
-                <UFIXITIcon className='icon-md pr-1'/> 
+                <UFIXITIcon className='icon-md pr-1' />
               </div>
               <div className='flex-column justify-content-center'>
                 {t('menu.fix_issues')}
               </div>
             </li>
             <li
               className={`flex-row ${navigation === 'reports' ? ' active-link' : ''}`}
-              onClick={()=>handleNavigation('reports')}
+              onClick={() => handleNavigation('reports')}
               onKeyPress={(e) => {
-                if(e.key === 'Enter' || e.key === ' ') {
+                if (e.key === 'Enter' || e.key === ' ') {
                   handleNavigation('reports')
                 }
               }}
               tabindex='0'>
               <div className='flex-column justify-content-center'>
-                <ReportIcon className='icon-md pr-1'/>
+                <ReportIcon className='icon-md pr-1' />
               </div>
               <div className='flex-column justify-content-center'>
                 {t('menu.reports')}
               </div>
             </li>
             <li
               className={`flex-row ${navigation === 'settings' ? ' active-link' : ''}`}
-              onClick={()=>handleNavigation('settings')}
+              onClick={() => handleNavigation('settings')}
               onKeyPress={(e) => {
-                if(e.key === 'Enter' || e.key === ' ') {
+                if (e.key === 'Enter' || e.key === ' ') {
                   handleNavigation('settings')
                 }
               }}
               tabindex='0'>
               <div className='flex-column justify-content-center'>
-                <SettingsIcon className='icon-md pr-1'/>
+                <SettingsIcon className='icon-md pr-1' />
               </div>
               <div className='flex-column justify-content-center'>
                 {t('menu.settings')}
@@ -84,4 +84,4 @@ export default function Header({ t, navigation, handleNavigation }) {
       </nav>
     </header>
   )
-}
+}

build/nginx/Dockerfile.php.pdo.mysql

@@ -1,5 +1,30 @@
 FROM php:8.4.2-fpm
 
-# PHP extensions
-RUN apt-get update && apt-get install -y libpng-dev zlib1g-dev git unzip
-RUN docker-php-ext-install gd pdo pdo_mysql
+# Install necessary packages and PHP extensions
+RUN apt-get update && apt-get install -y \
+    supervisor \
+    libpng-dev \
+    zlib1g-dev \
+    git \
+    unzip
+
+RUN docker-php-ext-install pdo pdo_mysql gd
+
+# Create log directories expected by Supervisor and your apps
+RUN mkdir -p /var/www/html/var/log/supervisor \
+    && mkdir -p /var/log/supervisor
+
+# Copy your code
+COPY . /var/www/html
+WORKDIR /var/www/html
+
+# Copy Supervisor configs into the image
+COPY build/supervisor/supervisord.conf /etc/supervisor/supervisord.conf
+COPY build/supervisor/php-fpm.conf /etc/supervisor/conf.d/php-fpm.conf
+COPY build/supervisor/ /etc/supervisor/conf.d/
+
+# Expose port if necessary (e.g., for Nginx, but with PHP-FPM you typically let Nginx handle this)
+EXPOSE 9000
+
+# Start Supervisor as the main process
+CMD ["/usr/bin/supervisord", "-n", "-c", "/etc/supervisor/supervisord.conf"]

build/supervisor/messenger-concurrent.conf

@@ -0,0 +1,12 @@
+[program:messenger-concurrent]
+directory     = /var/www/html
+# command       = php /var/www/html/bin/console messenger:consume async_concurrent --sleep=100000 -vv
+command       = php /var/www/html/bin/console messenger:consume async_concurrent -vv
+numprocs      = %(ENV_SCAN_POOL_SIZE)s
+process_name  = %(program_name)s_%(process_num)02d
+autostart     = true
+autorestart   = true
+stdout_logfile = /dev/fd/1
+stdout_logfile_maxbytes = 0
+stderr_logfile_maxbytes = 0
+stderr_logfile = /dev/fd/2

build/supervisor/messenger-high.conf

@@ -0,0 +1,11 @@
+[program:messenger-high]
+directory     = /var/www/html
+command       = php bin/console messenger:consume async_priority_high -vv
+numprocs      = 2                # maybe fewer, but always running
+process_name  = %(program_name)s_%(process_num)02d
+autostart     = true
+autorestart   = true
+stdout_logfile = /dev/fd/1
+stdout_logfile_maxbytes = 0
+stderr_logfile_maxbytes = 0
+stderr_logfile = /dev/fd/2

build/supervisor/messenger-low.conf

@@ -0,0 +1,11 @@
+[program:messenger-low]
+directory     = /var/www/html
+command       = php bin/console messenger:consume async_priority_low -vv
+numprocs      = 4
+process_name  = %(program_name)s_%(process_num)02d
+autostart     = true
+autorestart   = true
+stdout_logfile = /dev/fd/1
+stdout_logfile_maxbytes = 0
+stderr_logfile_maxbytes = 0
+stderr_logfile = /dev/fd/2

build/supervisor/php-fpm.conf

@@ -0,0 +1,13 @@
+[program:php-fpm]
+command=/usr/local/sbin/php-fpm --nodaemonize
+autostart=true
+autorestart=true
+startsecs=5
+priority=10
+
+# stdout_logfile=/var/www/html/var/log/supervisor/php-fpm.out.log
+stdout_logfile=/dev/fd/1
+stdout_logfile_maxbytes=0
+
+stderr_logfile=/dev/fd/2
+stderr_logfile_maxbytes=0

build/supervisor/supervisord.conf

@@ -0,0 +1,19 @@
+[supervisord]
+nodaemon=true
+user=root
+logfile=/var/log/supervisor/supervisord.log
+pidfile=/var/run/supervisord.pid
+childlogdir=/var/log/supervisor
+
+[unix_http_server]
+file=/var/run/supervisor.sock   #; path to the UNIX socket file
+chmod=0700
+
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
+
+[supervisorctl]
+serverurl=unix:///var/run/supervisor.sock
+
+[include]
+files = /etc/supervisor/conf.d/*.conf

composer.json

@@ -24,6 +24,7 @@
         "oro/doctrine-extensions": "^2.0",
         "phpdocumentor/reflection-docblock": "^5.2",
         "phpstan/phpdoc-parser": "^0.5.3",
+        "predis/predis": "^3.0",
         "sensio/framework-extra-bundle": "^6.2",
         "sunra/php-simple-html-dom-parser": "1.5.2",
         "symfony/apache-pack": "^1.0",

config/packages/framework.yaml

@@ -1,4 +1,8 @@
 framework:
+
+    messenger:
+        reset_on_message: true
+
     secret: '%env(APP_SECRET)%'
     #csrf_protection: true
     #http_method_override: true

config/packages/messenger.yaml

@@ -5,16 +5,23 @@ framework:
 
         transports:
             # https://symfony.com/doc/current/messenger.html#transport-configuration
-            async_priority_high: 
+            async_priority_high:
                 dsn: '%env(MESSENGER_TRANSPORT_DSN)%'
-                options: 
+                options:
                     table_name: 'queue'
                     queue_name: 'priority_high'
-            async_priority_low: 
+            async_priority_low:
                 dsn: '%env(MESSENGER_TRANSPORT_DSN)%'
-                options: 
+                options:
                     table_name: 'queue'
-                    queue_name: 'priority_low'   
+                    queue_name: 'priority_low'
+
+            async_concurrent:
+                dsn: '%env(MESSENGER_TRANSPORT_DSN)%'
+                options:
+                    table_name: 'queue'
+                    queue_name: 'concurrent'
+
             failed: 'doctrine://default?queue_name=failed'
             sync: 'sync://'
 
@@ -25,3 +32,6 @@ framework:
             #'App\Message\BackgroundQueueItem': async_priority_low
             'App\Message\PriorityQueueItem': sync
             'App\Message\BackgroundQueueItem': sync
+            'App\Message\ScanContentItem': async_concurrent
+            'App\Message\FullRescanMessage': async_priority_high
+            'App\Message\FinishRescanMessage': async_priority_high

config/services.yaml

@@ -26,6 +26,11 @@ services:
         resource: '../src/Controller'
         tags: ['controller.service_arguments']
 
+
+    Predis\Client:
+        class: Predis\Client            # autowiring id == class name
+        arguments:
+            - '%env(REDIS_URL)%'
+
     # add more service definitions when explicit configuration is needed
     # please note that last definitions always *replace* previous ones
-