Skip to content

Commit 434183a

Browse files
committed
WIP more cleanups and refactor
1 parent 566707f commit 434183a

14 files changed

+190
-52
lines changed

classes/local/manager.php

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,6 @@ public static function get_objectfs_config() {
7979
$config->s3_bucket = '';
8080
$config->s3_region = 'us-east-1';
8181
$config->s3_base_url = '';
82-
$config->s3_canoverrideobjects = true;
8382
$config->key_prefix = '';
8483

8584
// Digital ocean file system.
@@ -385,7 +384,11 @@ public static function get_filename_from_header($header) {
385384
return $filename;
386385
}
387386

388-
public static function can_override_existing_objects() {
387+
/**
388+
* If is able to override existing objects in the external store.
389+
* @return bool
390+
*/
391+
public static function can_override_existing_objects(): bool {
389392
return !empty(get_config('tool_objectfs', 'canoverrideobjects'));
390393
}
391394
}

classes/local/store/object_client.php

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -153,11 +153,11 @@ public function test_set_object_tag(): stdClass;
153153
public function set_object_tags(string $contenthash, array $tags);
154154

155155
/**
156-
* Returns given objects tags queried from the external store. If object does not exist in store, return null.
156+
* Returns given objects tags queried from the external store. External object must exist.
157157
* @param string $contenthash file content has
158-
* @param array array of key=>value tag pairs, or null if object does not exist.
158+
* @param array array of key=>value tag pairs
159159
*/
160-
public function get_object_tags(string $contenthash): ?array;
160+
public function get_object_tags(string $contenthash): array;
161161

162162
/**
163163
* If the client supports object tagging feature.

classes/local/store/object_client_base.php

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -208,11 +208,11 @@ public function set_object_tags(string $contenthash, array $tags) {
208208
}
209209

210210
/**
211-
* Returns given objects tags queried from the external store. If object does not exist in store, return null.
211+
* Returns given objects tags queried from the external store. External object must exist.
212212
* @param string $contenthash file content has
213-
* @param array array of key=>value tag pairs, or null if object does not exist.
213+
* @param array array of key=>value tag pairs
214214
*/
215-
public function get_object_tags(string $contenthash): ?array {
215+
public function get_object_tags(string $contenthash): array {
216216
return [];
217217
}
218218

classes/local/store/s3/client.php

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -511,11 +511,10 @@ public function upload_to_s3($localpath, $contenthash, array $tags) {
511511

512512
try {
513513
$externalpath = $this->get_filepath_from_hash($contenthash);
514-
515514
$uploader = new \Aws\S3\ObjectUploader(
516515
$this->client, $this->bucket,
517516
$this->bucketkeyprefix . $externalpath,
518-
$filehandle,
517+
$filehandle
519518
);
520519
$uploader->upload();
521520
fclose($filehandle);
@@ -893,7 +892,6 @@ public function test_set_object_tag(): stdClass {
893892
// First ensure a test object exists to put tags on.
894893
// Note this will override the existing object if exists.
895894
$key = $this->bucketkeyprefix . 'tagging_check_file';
896-
897895
$this->client->putObject([
898896
'Bucket' => $this->bucket,
899897
'Key' => $key,
@@ -925,7 +923,12 @@ public function test_set_object_tag(): stdClass {
925923
return (object) ['success' => true, 'details' => ''];
926924
}
927925

928-
private function convert_tags_to_s3_format(array $tags) {
926+
/**
927+
* Convert key=>value to s3 tag format
928+
* @param array $tags
929+
* @return array tags in s3 format.
930+
*/
931+
private function convert_tags_to_s3_format(array $tags): array {
929932
foreach($tags as $key => $value) {
930933
$s3tags[] = [
931934
'Key' => $key,

classes/local/store/s3/file_system.php

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -88,9 +88,9 @@ public function copy_from_local_to_external($contenthash) {
8888
$localpath = $this->get_local_path_from_hash($contenthash);
8989

9090
try {
91-
// If can't override, check if it exists already, since upload object will override.
91+
// If cannot override, and already exists, skip.
92+
// We don't want to upload and potentially override.
9293
if (!manager::can_override_existing_objects() && $this->is_file_readable_externally_by_hash($contenthash)) {
93-
// TODO somehow log it was skipped here?
9494
return true;
9595
}
9696

@@ -122,5 +122,4 @@ public function copy_from_local_to_external($contenthash) {
122122
public function supports_presigned_urls() {
123123
return true;
124124
}
125-
126125
}
Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
<?php
2+
// This file is part of Moodle - http://moodle.org/
3+
//
4+
// Moodle is free software: you can redistribute it and/or modify
5+
// it under the terms of the GNU General Public License as published by
6+
// the Free Software Foundation, either version 3 of the License, or
7+
// (at your option) any later version.
8+
//
9+
// Moodle is distributed in the hope that it will be useful,
10+
// but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
// GNU General Public License for more details.
13+
//
14+
// You should have received a copy of the GNU General Public License
15+
// along with Moodle. If not, see <http://www.gnu.org/licenses/>.
16+
17+
namespace tool_objectfs\local\tag;
18+
19+
/**
20+
* Provides environment a file was uploaded in.
21+
*
22+
* @package tool_objectfs
23+
* @author Matthew Hilton <[email protected]>
24+
* @copyright Catalyst IT
25+
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
26+
*/
27+
class environment_source implements tag_source {
28+
/**
29+
* Identifier used in tagging file. Is the 'key' of the tag.
30+
* @return string
31+
*/
32+
public static function get_identifier(): string {
33+
return 'environment';
34+
}
35+
36+
public static function get_description(): string {
37+
return get_string('tagsource:environment', 'tool_objectfs', self::get_env());
38+
}
39+
40+
private static function get_env(): ?string {
41+
global $CFG;
42+
return $CFG->objectfs_environment_name ?? null;
43+
}
44+
45+
/**
46+
* Returns the tag value for the given file contenthash
47+
* @param string $contenthash
48+
* @return string|null mime type for file.
49+
*/
50+
public function get_value_for_contenthash(string $contenthash): ?string {
51+
return self::get_env();
52+
}
53+
}

classes/local/tag/file_type_source.php renamed to classes/local/tag/mime_type_source.php

Lines changed: 13 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -17,47 +17,39 @@
1717
namespace tool_objectfs\local\tag;
1818

1919
/**
20-
* File type tag source.
21-
* Provides tag values for files based on their 'type' - these are categories (not necessarily equivalent to fileareas)
22-
* that make it possible to apply different permissions to. For example, move backups to a cheaper storage class but not general files.
20+
* Provides mime type of file.
2321
*
2422
* @package tool_objectfs
2523
* @author Matthew Hilton <[email protected]>
2624
* @copyright Catalyst IT
2725
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
2826
*/
29-
class file_type_source implements tag_source {
30-
/**
31-
* @var string Unknown/uncategorised
32-
*/
33-
public const TYPE_UNCATEGORISED = 'uncategorised';
34-
35-
/**
36-
* @var string A course backup
37-
*/
38-
public const TYPE_COURSE_BACKUP = 'course_backup';
39-
27+
class mime_type_source implements tag_source {
4028
/**
4129
* Identifier used in tagging file. Is the 'key' of the tag.
4230
* @return string
4331
*/
4432
public static function get_identifier(): string {
45-
return 'type';
33+
return 'mimetype';
34+
}
35+
36+
public static function get_description(): string {
37+
return get_string('tagsource:mimetype', 'tool_objectfs');
4638
}
4739

4840
/**
4941
* Returns the tag value for the given file contenthash
5042
* @param string $contenthash
51-
* @return string|null the category of the file, one of TYPE_*
43+
* @return string|null mime type for file.
5244
*/
5345
public function get_value_for_contenthash(string $contenthash): ?string {
5446
global $DB;
55-
// TODO are there other unknown course backup files ?
56-
$iscoursebackup = $DB->record_exists('files', ['contenthash' => $contenthash, 'component' => 'backup', 'filearea' => 'course']);
57-
if ($iscoursebackup) {
58-
return self::TYPE_COURSE_BACKUP;
47+
$mime = $DB->get_field('files', 'mimetype', ['contenthash' => $contenthash]);
48+
49+
if(empty($mime)) {
50+
return null;
5951
}
6052

61-
return self::TYPE_UNCATEGORISED;
53+
return $mime;
6254
}
6355
}

classes/local/tag/tag_manager.php

Lines changed: 26 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,7 @@
1717
namespace tool_objectfs\local\tag;
1818

1919
use coding_exception;
20-
use core\lock\lock;
21-
use core\lock\lock_config;
22-
use Exception;
20+
use html_writer;
2321
use tool_objectfs\local\manager;
2422
use tool_objectfs\local\store\object_file_system;
2523

@@ -62,7 +60,8 @@ public static function get_defined_tag_sources(): array {
6260
// All possible tag sources should be defined here.
6361
// Note this should be a maximum of 10 sources, as this is an AWS limit.
6462
return [
65-
new file_type_source()
63+
new mime_type_source(),
64+
new environment_source(),
6665
];
6766
}
6867

@@ -161,21 +160,25 @@ public static function mark_object_tag_sync_status(string $contenthash, int $sta
161160
$DB->set_field('tool_objectfs_objects', 'tagsyncstatus', $status, ['contenthash' => $contenthash]);
162161
}
163162

163+
/**
164+
* Syncs an objects tags after they are uploaded.
165+
* Note - you should take out the object lock before calling this function to avoid race conditions.
166+
* @param string $contenthash
167+
* @param object_file_system $fs
168+
*/
164169
public static function sync_object_tags_post_upload(string $contenthash, object_file_system $fs) {
165-
// First get object lock
166-
// TODO
167-
168-
$client = manager::get_client(manager::get_objectfs_config());
170+
$client = $fs->get_external_client();
171+
$objectexists = $fs->is_file_readable_externally_by_hash($contenthash);
169172

170173
// If object does not exist, cannot sync tags to nothing, abort.
171-
if (!$fs->is_file_readable_externally_by_hash($contenthash)) {
174+
if (!$objectexists) {
172175
// TODO maybe this should be a failed status?
173176
self::mark_object_tag_sync_status($contenthash, self::SYNC_STATUS_SYNC_NOT_REQUIRED);
174177
return;
175178
}
176179

177180
// Cannot override and object exists, query existing and store.
178-
if (!manager::can_override_existing_objects()) {
181+
if (!manager::can_override_existing_objects() && $objectexists) {
179182
// Query existing tags and store them.
180183
$existingtags = $client->get_object_tags($contenthash);
181184
self::store_tags_locally($contenthash, $existingtags);
@@ -189,7 +192,19 @@ public static function sync_object_tags_post_upload(string $contenthash, object_
189192

190193
// Either way, it has synced.
191194
self::mark_object_tag_sync_status($contenthash, self::SYNC_STATUS_SYNC_NOT_REQUIRED);
195+
}
192196

193-
// TODO release object lock.
197+
/**
198+
* Returns a simple list of all the sources and their descriptions.
199+
* @return string html string
200+
*/
201+
public static function get_tag_summary_html(): string {
202+
$sources = self::get_defined_tag_sources();
203+
$html = '';
204+
205+
foreach ($sources as $source) {
206+
$html .= $source->get_identifier() . ': ' . $source->get_description() . '<br />';
207+
}
208+
return $html;
194209
}
195210
}

classes/local/tag/tag_source.php

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,12 @@ interface tag_source {
3434
*/
3535
public static function get_identifier(): string;
3636

37+
/**
38+
* Description for source displayed in the admin settings.
39+
* @return string
40+
*/
41+
public static function get_description(): string;
42+
3743
/**
3844
* Returns the value of this tag for the file with the given content hash.
3945
* This must be deterministic, and should never exceed 256 chars.
Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
<?php
2+
// This file is part of Moodle - http://moodle.org/
3+
//
4+
// Moodle is free software: you can redistribute it and/or modify
5+
// it under the terms of the GNU General Public License as published by
6+
// the Free Software Foundation, either version 3 of the License, or
7+
// (at your option) any later version.
8+
//
9+
// Moodle is distributed in the hope that it will be useful,
10+
// but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
// GNU General Public License for more details.
13+
//
14+
// You should have received a copy of the GNU General Public License
15+
// along with Moodle. If not, see <http://www.gnu.org/licenses/>.
16+
17+
namespace tool_objectfs\task;
18+
19+
use core\task\manager;
20+
use core\task\scheduled_task;
21+
22+
/**
23+
* Queues update_object_tags adhoc task periodically, or manually from the frontend.
24+
*
25+
* @package tool_objectfs
26+
* @author Matthew Hilton <[email protected]>
27+
* @copyright Catalyst IT
28+
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
29+
*/
30+
class trigger_update_object_tags extends scheduled_task {
31+
/**
32+
* Task name
33+
*/
34+
public function get_name() {
35+
return get_string('task:triggerupdateobjecttags', 'tool_objectfs');
36+
}
37+
/**
38+
* Execute task
39+
*/
40+
public function execute() {
41+
// Queue adhoc task, nothing else.
42+
$task = new update_object_tags();
43+
manager::queue_adhoc_task($task, true);
44+
}
45+
}

0 commit comments

Comments
 (0)