Merge branch 'release/0.4.0'

Author: JC5

.env.example

@@ -18,6 +18,8 @@ FIREFLY_III_URL=
 # 1) Make sure you ADD http:// or https://
 # 2) Make sure you REMOVE any trailing slash from the end of the URL.
 #
+# IF YOU SET THIS VALUE, ALSO SET THE FIREFLY_III_URL
+#
 VANITY_URL=
 
 #
@@ -40,27 +42,50 @@ FIREFLY_III_ACCESS_TOKEN=
 FIREFLY_III_CLIENT_ID=
 
 #
-# Nordigen
+# Nordigen information.
 #
 NORDIGEN_ID=
 NORDIGEN_KEY=
 NORDIGEN_SANDBOX=false
 
 #
-# Spectre
+# Spectre information
 #
 SPECTRE_APP_ID=
 SPECTRE_SECRET=
 
 #
-# Use cache
+# Use cache. No need to do this.
 #
 USE_CACHE=false
 
 #
-# Post import directory white list
+# Auto import settings. Due to security constraints, you MUST enable each feature individually.
+# You must also set a secret. The secret is used for the web routes.
+#
+# The auto-import secret must be a string of at least 16 characters.
+# Visit this page for inspiration: https://www.random.org/passwords/?num=1&len=16&format=html&rnd=new
+#
+# Submit it using ?secret=X
+#
+AUTO_IMPORT_SECRET=
+
+#
+# Is the /autoimport even endpoint enabled?
+# By default it's disabled, and the secret alone will not enable it.
+#
+CAN_POST_AUTOIMPORT=false
+
+#
+# Is the /autoupload endpoint enabled?
+# By default it's disabled, and the secret alone will not enable it.
+#
+CAN_POST_FILES=false
+
+#
+# Import directory white list. You need to set this before the auto importer will accept a directory to import from.
 #
-POST_DIR_WHITELIST=
+IMPORT_DIR_WHITELIST=
 
 #
 # When you're running Firefly III under a (self-signed) certificate,

.github/mergify.yml

@@ -10,4 +10,4 @@ pull_request_rules:
       - base=main
     actions:
       close:
-        message: Do not open PR's on the main branch.
+        message: Please do not open PR's on the `main` branch, but on the `develop` branch only. Thank you!

app/Console/AutoImports.php

@@ -161,9 +161,19 @@ protected function importFiles(string $directory, array $files): void
      */
     private function importFile(string $directory, string $file): void
     {
+        app('log')->debug(sprintf('ImportFile: directory "%s"', $directory));
+        app('log')->debug(sprintf('ImportFile: file      "%s"', $file));
         $csvFile  = sprintf('%s/%s', $directory, $file);
         $jsonFile = sprintf('%s/%s.json', $directory, substr($file, 0, -5));
 
+        // TODO not yet sure why the distinction is necessary.
+        if ('csv' === $this->getExtension($file)) {
+            $jsonFile = sprintf('%s/%s.json', $directory, substr($file, 0, -4));
+        }
+
+        app('log')->debug(sprintf('ImportFile: CSV       "%s"', $csvFile));
+        app('log')->debug(sprintf('ImportFile: JSON      "%s"', $jsonFile));
+
         // do JSON check
         $jsonResult = $this->verifyJSON($jsonFile);
         if (false === $jsonResult) {
@@ -390,4 +400,43 @@ private function reportImport(): void
             }
         }
     }
+
+
+    /**
+     * @param string $csvFile
+     * @param string $jsonFile
+     * @throws ImporterErrorException
+     */
+    private function importUpload(string $csvFile, string $jsonFile): void
+    {
+        // do JSON check
+        $jsonResult = $this->verifyJSON($jsonFile);
+        if (false === $jsonResult) {
+            $message = sprintf('The importer can\'t import %s: could not decode the JSON in config file %s.', $csvFile, $jsonFile);
+            $this->error($message);
+
+            return;
+        }
+        $configuration = Configuration::fromArray(json_decode(file_get_contents($jsonFile), true));
+        $configuration->updateDateRange();
+
+
+        $this->line(sprintf('Going to convert from file %s using configuration %s and flow "%s".', $csvFile, $jsonFile, $configuration->getFlow()));
+
+        // this is it!
+        $this->startConversion($configuration, $csvFile);
+        $this->reportConversion();
+
+        $this->line(sprintf('Done converting from file %s using configuration %s.', $csvFile, $jsonFile));
+        $this->startImport($configuration);
+        $this->reportImport();
+
+        $this->line('Done!');
+        event(new ImportedTransactions(
+                  array_merge($this->conversionMessages, $this->importMessages),
+                  array_merge($this->conversionWarnings, $this->importWarnings),
+                  array_merge($this->conversionErrors, $this->importErrors)
+              )
+        );
+    }
 }

app/Console/Commands/AutoImport.php

@@ -66,6 +66,10 @@ public function handle(): int
 
         $argument  = (string) ($this->argument('directory') ?? './');
         $directory = realpath($argument);
+        if (!$this->isAllowedPath($directory)) {
+            $this->error(sprintf('Path "%s" is not in the list of allowed paths (IMPORT_DIR_WHITELIST).', $directory));
+            return 1;
+        }
         $this->line(sprintf('Going to automatically import everything found in %s (%s)', $directory, $argument));
 
         $files = $this->getFiles($directory);
@@ -86,5 +90,4 @@ public function handle(): int
 
         return 0;
     }
-
 }

app/Console/Commands/Import.php

@@ -75,6 +75,22 @@ public function handle(): int
         $file   = (string) $this->argument('file');
         $config = (string) $this->argument('config');
 
+        // validate config path:
+        if ('' !== $config) {
+            $directory = dirname($config);
+            if (!$this->isAllowedPath($directory)) {
+                $this->error(sprintf('Path "%s" is not in the list of allowed paths (IMPORT_DIR_WHITELIST).', $directory));
+                return 1;
+            }
+        }
+        if ('' !== $file) {
+            $directory = dirname($file);
+            if (!$this->isAllowedPath($directory)) {
+                $this->error(sprintf('Path "%s" is not in the list of allowed paths (IMPORT_DIR_WHITELIST).', $directory));
+                return 1;
+            }
+        }
+
         if (!file_exists($config) || (file_exists($config) && !is_file($config))) {
             $message = sprintf('The importer can\'t import: configuration file "%s" does not exist or could not be read.', $config);
             $this->error($message);

app/Console/HaveAccess.php

@@ -62,4 +62,27 @@ private function haveAccess(): bool
      * @return void
      */
     abstract public function error($string, $verbosity = null);
+
+    /**
+     * @param string $path
+     * @return bool
+     */
+    private function isAllowedPath(string $path): bool
+    {
+        $paths = config('importer.import_dir_whitelist');
+        if (null === $paths) {
+            $this->warn('No valid paths in IMPORT_DIR_WHITELIST, cannot continue.');
+            return false;
+        }
+        if (is_array($paths) && 0 === count($paths)) {
+            $this->warn('No valid paths in IMPORT_DIR_WHITELIST, cannot continue.');
+            return false;
+        }
+        if (is_array($paths) && 1 === count($paths) && '' === $paths[0]) {
+            $this->warn('No valid paths in IMPORT_DIR_WHITELIST, cannot continue.');
+            return false;
+        }
+
+        return in_array($path, $paths, true);
+    }
 }

app/Events/ImportedTransactions.php

@@ -1,5 +1,5 @@
 <?php
-declare(strict_types=1);
+
 /*
  * ImportedTransactions.php
  * Copyright (c) 2021 james@firefly-iii.org
@@ -21,6 +21,8 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
+declare(strict_types=1);
+
 namespace App\Events;
 
 use Illuminate\Queue\SerializesModels;

app/Handlers/Events/ImportedTransactionsEventHandler.php

@@ -1,5 +1,5 @@
 <?php
-declare(strict_types=1);
+
 /*
  * ImportedTransactionsEventHandler.php
  * Copyright (c) 2021 james@firefly-iii.org
@@ -21,6 +21,8 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
+declare(strict_types=1);
+
 namespace App\Handlers\Events;
 
 use App\Events\ImportedTransactions;
@@ -36,21 +38,21 @@ public function sendReportOverMail(ImportedTransactions $event): void
     {
         app('log')->debug('Now in sendReportOverMail');
 
-        $mailer = config('mail.default');
+        $mailer   = config('mail.default');
         $receiver = config('mail.destination');
-        if('' === $mailer) {
+        if ('' === $mailer) {
             app('log')->info('No mailer configured, will not mail.');
             return;
         }
-        if('' === $receiver) {
+        if ('' === $receiver) {
             app('log')->info('No mail receiver configured, will not mail.');
             return;
         }
 
-        $log  =[
+        $log = [
             'messages' => $event->messages,
             'warnings' => $event->warnings,
-            'errors' => $event->errors,
+            'errors'   => $event->errors,
         ];
         app('log')->info('Will send report message.');
         app('log')->debug('If no error below this line, mail was sent!');

app/Http/Controllers/AutoImportController.php

@@ -31,6 +31,7 @@
 use App\Console\VerifyJSON;
 use App\Exceptions\ImporterErrorException;
 use Illuminate\Http\Request;
+use Illuminate\Http\Response;
 use Log;
 
 /**
@@ -45,40 +46,45 @@ class AutoImportController extends Controller
     /**
      *
      */
-    public function index(Request $request)
+    public function index(Request $request): Response
     {
-        die('todo' . __METHOD__);
+        if (false === config('importer.can_post_autoimport')) {
+            throw new ImporterErrorException('Disabled, not allowed to import.');
+        }
+
+        $secret       = (string) ($request->get('secret') ?? '');
+        $systemSecret = (string) config('importer.auto_import_secret');
+        if ('' === $secret || '' === $systemSecret || $secret !== config('importer.auto_import_secret') || strlen($systemSecret) < 16) {
+            throw new ImporterErrorException('Bad secret, not allowed to import.');
+        }
+
+        $argument  = (string) ($request->get('directory') ?? './');
+        $directory = realpath($argument);
+
+        if (!$this->isAllowedPath($directory)) {
+            throw new ImporterErrorException('Not allowed to import from this path.');
+        }
+
         $access = $this->haveAccess();
         if (false === $access) {
             throw new ImporterErrorException('Could not connect to your local Firefly III instance.');
         }
 
-        $argument        = (string) ($request->get('directory') ?? './');
-        $this->directory = realpath($argument);
-        $this->line(sprintf('Going to automatically import everything found in %s (%s)', $this->directory, $argument));
+        // take code from auto importer.
+        app('log')->info(sprintf('Going to automatically import everything found in %s (%s)', $directory, $argument));
 
-        $files = $this->getFiles();
+        $files = $this->getFiles($directory);
         if (0 === count($files)) {
-            $this->line(sprintf('There are no files in directory %s', $this->directory));
-            $this->line('To learn more about this process, read the docs:');
-            $this->line('https://docs.firefly-iii.org/data-importer/');
-
-            return ' ';
+            return response('');
         }
-        $this->line(sprintf('Found %d CSV + JSON file sets in %s', count($files), $this->directory));
+        app('log')->info(sprintf('Found %d (CSV +) JSON file sets in %s', count($files), $directory));
         try {
-            $this->importFiles($files);
+            $this->importFiles($directory, $files);
         } catch (ImporterErrorException $e) {
-            Log::error($e->getMessage());
-            $this->line(sprintf('Import exception (see the logs): %s', $e->getMessage()));
+            app('log')->error($e->getMessage());
+            throw new ImporterErrorException(sprintf('Import exception (see the logs): %s', $e->getMessage()));
         }
-
-        return ' ';
-    }
-
-    public function line(string $string)
-    {
-        echo sprintf("%s: %s\n", date('Y-m-d H:i:s'), $string);
+        return response('');
     }
 
     /**
@@ -89,6 +95,11 @@ public function error($string, $verbosity = null)
         $this->line($string);
     }
 
+    public function line(string $string)
+    {
+        echo sprintf("%s: %s\n", date('Y-m-d H:i:s'), $string);
+    }
+
     /**
      * @param      $string
      * @param null $verbosity

app/Http/Controllers/AutoUploadController.php

@@ -45,7 +45,16 @@ class AutoUploadController extends Controller
      */
     public function index(AutoUploadRequest $request)
     {
-        die('todo' . __METHOD__);
+        if (false === config('importer.can_post_files')) {
+            throw new ImporterErrorException('Disabled, not allowed to import.');
+        }
+
+        $secret       = (string) ($request->get('secret') ?? '');
+        $systemSecret = (string) config('importer.auto_import_secret');
+        if ('' === $secret || '' === $systemSecret || $secret !== config('importer.auto_import_secret') || strlen($systemSecret) < 16) {
+            throw new ImporterErrorException('Bad secret, not allowed to import.');
+        }
+
         $access = $this->haveAccess();
         if (false === $access) {
             throw new ImporterErrorException('Could not connect to your local Firefly III instance.');