Skip to content

Commit 566707f

Browse files
matthewhiltonmatthewhilton
committed
WIP more cleanups and refactors
1 parent 8e1b1cd commit 566707f

10 files changed

+84
-95
lines changed

‎classes/local/manager.php

+4
Original file line numberDiff line numberDiff line change
@@ -384,4 +384,8 @@ public static function get_filename_from_header($header) {
384384
}
385385
return $filename;
386386
}
387+
388+
public static function can_override_existing_objects() {
389+
return !empty(get_config('tool_objectfs', 'canoverrideobjects'));
390+
}
387391
}

‎classes/local/store/object_file_system.php

-4
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,6 @@
3737
use file_storage;
3838
use BlobRestProxy;
3939
use tool_objectfs\local\manager;
40-
use tool_objectfs\local\tag\tag_manager;
4140

4241
defined('MOODLE_INTERNAL') || die();
4342

@@ -1155,7 +1154,4 @@ private function update_object(array $result): array {
11551154

11561155
return $result;
11571156
}
1158-
1159-
public function sync_tags(string $contenthash) {
1160-
}
11611157
}

‎classes/local/store/s3/client.php

+29-58
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525

2626
namespace tool_objectfs\local\store\s3;
2727

28+
use coding_exception;
2829
use tool_objectfs\local\manager;
2930
use tool_objectfs\local\store\object_client_base;
3031
use tool_objectfs\local\store\signed_url;
@@ -62,10 +63,11 @@ class client extends object_client_base {
6263
*/
6364
private $signingmethod;
6465

66+
/**
67+
* @var mixed
68+
*/
6569
protected $bucketkeyprefix;
6670

67-
protected $allowoverride;
68-
6971
/**
7072
* construct
7173
* @param mixed $config
@@ -86,7 +88,6 @@ public function __construct($config) {
8688
$this->enablepresignedurls = $config->enablepresignedurls;
8789
$this->signingmethod = $config->signingmethod;
8890
$this->bucketkeyprefix = $config->key_prefix;
89-
$this->allowoverride = $config->s3_allowoverride;
9091
$this->set_client($config);
9192
} else {
9293
parent::__construct($config);
@@ -488,10 +489,6 @@ public function define_client_section($settings, $config) {
488489
$settings->add(new \admin_setting_configtext('tool_objectfs/key_prefix',
489490
new \lang_string('settings:aws:key_prefix', 'tool_objectfs'),
490491
new \lang_string('settings:aws:key_prefix_help', 'tool_objectfs'), ''));
491-
492-
$settings->add(new \admin_setting_configcheckbox('tool_objectfs/s3_overrideobjects',
493-
new \lang_string('settings:aws:overrideobjects', 'tool_objectfs'),
494-
new \lang_string('settings:aws:overrideobjects_help', 'tool_objectfs'), 0));
495492

496493
return $settings;
497494
}
@@ -515,15 +512,20 @@ public function upload_to_s3($localpath, $contenthash, array $tags) {
515512
try {
516513
$externalpath = $this->get_filepath_from_hash($contenthash);
517514

518-
// TODO upload with tags.
519515
$uploader = new \Aws\S3\ObjectUploader(
520516
$this->client, $this->bucket,
521517
$this->bucketkeyprefix . $externalpath,
522518
$filehandle,
523-
'private',
524519
);
525520
$uploader->upload();
526521
fclose($filehandle);
522+
523+
// The S3 php api does not support uploading object tags as part of
524+
// the object uploader, so upload them directly after uploading the object.
525+
if (!empty($tags)) {
526+
$this->set_object_tags($contenthash, $tags);
527+
}
528+
527529
} catch (\Aws\Exception\MultipartUploadException $e) {
528530
$params = $e->getState()->getId();
529531
$this->client->abortMultipartUpload($params);
@@ -923,11 +925,14 @@ public function test_set_object_tag(): stdClass {
923925
return (object) ['success' => true, 'details' => ''];
924926
}
925927

926-
private function convert_array_to_s3_tag_format(array $tags): array {
927-
928-
return [
929-
'TagSet' => $s3tags
930-
];
928+
private function convert_tags_to_s3_format(array $tags) {
929+
foreach($tags as $key => $value) {
930+
$s3tags[] = [
931+
'Key' => $key,
932+
'Value' => $value,
933+
];
934+
}
935+
return $s3tags;
931936
}
932937

933938
/**
@@ -936,23 +941,14 @@ private function convert_array_to_s3_tag_format(array $tags): array {
936941
* @param array $tags array of key=>value pairs to set as tags.
937942
*/
938943
public function set_object_tags(string $contenthash, array $tags) {
939-
$key = $this->bucketkeyprefix . $this->get_filepath_from_hash($contenthash);
940-
941-
// Convert to S3 format.
942-
$s3tags = [];
943-
foreach($tags as $key => $value) {
944-
$s3tags[] = [
945-
'Key' => $key,
946-
'Value' => $value,
947-
];
948-
}
944+
$objectkey = $this->bucketkeyprefix . $this->get_filepath_from_hash($contenthash);
949945

950946
// Then put onto object.
951947
$this->client->putObjectTagging([
952948
'Bucket' => $this->bucket,
953-
'Key' => $key,
949+
'Key' => $objectkey,
954950
'Tagging' => [
955-
'TagSet' => $s3tags,
951+
'TagSet' => $this->convert_tags_to_s3_format($tags),
956952
],
957953
]);
958954
}
@@ -966,14 +962,19 @@ public function get_object_tags(string $contenthash): array {
966962
$key = $this->bucketkeyprefix . $this->get_filepath_from_hash($contenthash);
967963

968964
// Query from S3.
969-
$tags = $this->client->getObjectTagging([
965+
$result = $this->client->getObjectTagging([
970966
'Bucket' => $this->bucket,
971967
'Key' => $key,
972968
]);
973969

970+
// Ensure tags are what we expect, and AWS have not changed the format.
971+
if (empty($result->toArray()['TagSet'])) {
972+
throw new coding_exception("Unexpected tag format received"); // TODO different ex type.
973+
}
974+
974975
// Convert from S3 format to key=>value format.
975976
$tagkv = [];
976-
foreach($tags['Tagging']['TagSet'] as $tag) {
977+
foreach($result->toArray()['TagSet'] as $tag) {
977978
$tagkv[$tag['Key']] = $tag['Value'];
978979
}
979980

@@ -987,34 +988,4 @@ public function get_object_tags(string $contenthash): array {
987988
public function supports_object_tagging(): bool {
988989
return true;
989990
}
990-
991-
// TODO maybe this moves to 'global' objectfs config ?
992-
public function can_override_objects(): bool {
993-
return $this->allowoverride;
994-
}
995-
996-
public function object_exists(string $contenthash): bool {
997-
$key = $this->bucketkeyprefix . $this->get_filepath_from_hash($contenthash);
998-
999-
// Query from S3.
1000-
try {
1001-
$this->client->headObject([
1002-
'Bucket' => $this->bucket,
1003-
'Key' => $key,
1004-
]);
1005-
1006-
// Exists, headObject succeeded.
1007-
return true;
1008-
} catch (\AWS\Exception\AwsException $e) {
1009-
$is404 = false; // TOOD determine if was 404, i.e. missing object
1010-
1011-
// Object does not exist.
1012-
if($is404) {
1013-
return false;
1014-
}
1015-
1016-
// Other error - rethrow.
1017-
throw $e;
1018-
}
1019-
}
1020991
}

‎classes/local/store/s3/file_system.php

+1-1
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ public function copy_from_local_to_external($contenthash) {
8989

9090
try {
9191
// If can't override, check if it exists already, since upload object will override.
92-
if (!manager::can_override_existing_objects() && $this->get_external_client()->object_exists($contenthash)) {
92+
if (!manager::can_override_existing_objects() && $this->is_file_readable_externally_by_hash($contenthash)) {
9393
// TODO somehow log it was skipped here?
9494
return true;
9595
}

‎classes/local/tag/tag_manager.php

+9-6
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
use core\lock\lock_config;
2222
use Exception;
2323
use tool_objectfs\local\manager;
24+
use tool_objectfs\local\store\object_file_system;
2425

2526
defined('MOODLE_INTERNAL') || die();
2627
require_once($CFG->dirroot . '/admin/tool/objectfs/lib.php');
@@ -160,24 +161,26 @@ public static function mark_object_tag_sync_status(string $contenthash, int $sta
160161
$DB->set_field('tool_objectfs_objects', 'tagsyncstatus', $status, ['contenthash' => $contenthash]);
161162
}
162163

163-
public static function sync_object_tags_post_upload(string $contenthash) {
164+
public static function sync_object_tags_post_upload(string $contenthash, object_file_system $fs) {
164165
// First get object lock
165166
// TODO
166167

167168
$client = manager::get_client(manager::get_objectfs_config());
168169

169-
// If object does not exist, cannot sync tags, abort.
170-
if (!$client->object_exists($contenthash)) {
170+
// If object does not exist, cannot sync tags to nothing, abort.
171+
if (!$fs->is_file_readable_externally_by_hash($contenthash)) {
172+
// TODO maybe this should be a failed status?
173+
self::mark_object_tag_sync_status($contenthash, self::SYNC_STATUS_SYNC_NOT_REQUIRED);
171174
return;
172175
}
173176

174-
// Cannot override and object exists.
175-
if (!$client->can_override_objects()) {
177+
// Cannot override and object exists, query existing and store.
178+
if (!manager::can_override_existing_objects()) {
176179
// Query existing tags and store them.
177180
$existingtags = $client->get_object_tags($contenthash);
178181
self::store_tags_locally($contenthash, $existingtags);
179182

180-
// Else can override, upload new.
183+
// Else can override, upload new and store.
181184
} else {
182185
$tags = self::gather_object_tags_for_upload($contenthash);
183186
$client->set_object_tags($contenthash, $tags);

‎classes/task/update_object_tags.php

+27-3
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@
1818

1919
use coding_exception;
2020
use core\task\adhoc_task;
21+
use Exception;
22+
use Throwable;
2123
use tool_objectfs\local\manager;
2224
use tool_objectfs\local\tag\tag_manager;
2325

@@ -34,8 +36,6 @@ class update_object_tags extends adhoc_task {
3436
* Execute task
3537
*/
3638
public function execute() {
37-
// TODO some way to trigger this from the frontend.
38-
3939
if (!tag_manager::is_tagging_enabled_and_supported()) {
4040
mtrace("Tagging feature not enabled or supported by filesystem, exiting.");
4141
return;
@@ -50,12 +50,36 @@ public function execute() {
5050
return;
5151
}
5252

53+
// Sanity check that fs is object file system and not anything else.
54+
$fs = get_file_storage()->get_file_system();
55+
56+
if (!method_exists($fs, "acquire_object_lock")) {
57+
mtrace("File system is not object file system, exiting");
58+
return;
59+
}
60+
5361
// For each, try to sync their tags.
5462
foreach ($contenthashes as $contenthash) {
55-
tag_manager::sync_object_tags_post_upload($contenthash);
63+
// Get a lock before syncing, to ensure other parts of objectfs are not moving/interacting with this object.
64+
$lock = $fs->acquire_object_lock($contenthash, 5);
65+
66+
// No lock - just skip it.
67+
if (!$lock) {
68+
continue;
69+
}
70+
71+
try {
72+
mtrace("Syncing tags for " . $contenthash);
73+
tag_manager::sync_object_tags_post_upload($contenthash);
74+
} catch (Throwable $e) {
75+
$lock->release();
76+
throw $e;
77+
}
78+
$lock->release();
5679
}
5780

5881
// Re-queue self to process more in another iteration.
82+
mtrace("Requeing self for another iteration");
5983
$task = new update_object_tags();
6084
\core\task\manager::queue_adhoc_task($task);
6185
}

‎db/tasks.php

-9
Original file line numberDiff line numberDiff line change
@@ -107,14 +107,5 @@
107107
'dayofweek' => '*',
108108
'month' => '*'
109109
),
110-
array (
111-
'classname' => 'tool_objectfs\task\queue_objects_needing_tags',
112-
'blocking' => 0,
113-
'minute' => 'R',
114-
'hour' => '*',
115-
'day' => '*',
116-
'dayofweek' => '*',
117-
'month' => '*'
118-
),
119110
);
120111

‎lang/en/tool_objectfs.php

+4-7
Original file line numberDiff line numberDiff line change
@@ -273,13 +273,10 @@
273273
$string['check:proxyrangerequestsdisabled'] = 'The proxy range request setting is disabled.';
274274
$string['checkproxy_range_request'] = 'Pre-signed URL range request proxy';
275275

276-
$string['task:queueobjectsneedingtags'] = 'Queue objects needing tagging';
277276
$string['checktagging_status'] = 'Object tagging';
278277
$string['check:tagging:ok'] = 'Object tagging ok';
279278
$string['check:tagging:error'] = 'Error trying to tag object';
280-
$string['settings:maxtaggingperrun'] = 'Max objects tagged per run';
281-
$string['settings:maxtaggingperrun:desc'] = 'The maximum number of objects to be queued for tag updates per run of the queue_objects_needing_tags scheduled task.';
282-
$string['settings:overridetags'] = 'Override existing object tags';
283-
$string['settings:overridetags:desc'] = 'If enabled, ObjectFS will override any existing object tags as needed. If disabled and object already has tags, they will be left untouched. Primarily for use in situations where multiple environments target the same storage bucket. Generally, you will enable this in production, but disable everywhere else. See the README for more information.';
284-
$string['settings:s3:overrideobjects'] = 'Override existing objects';
285-
$string['settings:s3:overrideobjects_help'] = 'If objects are allowed to be overwritten. Turning this off will require S3 to make an additional API call beforeupload to check if the object exists. If you are using tagging, and are in a multi env setup where multiple environments target the same bucket, you should turn this on for prod, and off for everywhere else.';
279+
$string['settings:maxtaggingperrun'] = 'Object tagging migration limit';
280+
$string['settings:maxtaggingperrun:desc'] = 'The maximum number of objects to sync tags for per migration adhoc task iteration.';
281+
$string['settings:overrideobjects'] = 'Allow object override';
282+
$string['settings:overrideobjects:desc'] = 'If disabled, ObjectFS will check that an object does not exist before uploading.';

‎settings.php

+8-5
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@
2323
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
2424
*/
2525

26+
use tool_objectfs\task\update_object_tags;
27+
2628
defined('MOODLE_INTERNAL') || die();
2729

2830
require_once(__DIR__ . '/classes/local/manager.php');
@@ -125,6 +127,10 @@
125127
$settings->add(new admin_setting_configduration('tool_objectfs/consistencydelay',
126128
new lang_string('settings:consistencydelay', 'tool_objectfs'), '', 10 * MINSECS, MINSECS));
127129

130+
$settings->add(new admin_setting_configcheckbox('tool_objectfs/overrideobjects',
131+
new lang_string('settings:overrideobjects', 'tool_objectfs'),
132+
get_string('settings:overrideobjects:desc', 'tool_objectfs'),
133+
1));
128134

129135
$settings->add(new admin_setting_heading('tool_objectfs/storagefilesystemselection',
130136
new lang_string('settings:clientselection:header', 'tool_objectfs'), ''));
@@ -263,9 +269,6 @@
263269
get_string('settings:maxtaggingperrun:desc', 'tool_objectfs'),
264270
10000,
265271
PARAM_INT));
266-
267-
$settings->add(new admin_setting_configcheckbox('tool_objectfs/overridetags',
268-
new lang_string('settings:overridetags', 'tool_objectfs'),
269-
get_string('settings:overridetags:desc', 'tool_objectfs'),
270-
1));
272+
273+
// TODO some way to scheduled the migration adhoc task from the frontend
271274
}

‎version.php

+2-2
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,8 @@
2525

2626
defined('MOODLE_INTERNAL') || die();
2727

28-
$plugin->version = 2023051704; // The current plugin version (Date: YYYYMMDDXX).
29-
$plugin->release = 2023051704; // Same as version.
28+
$plugin->version = 2023051705; // The current plugin version (Date: YYYYMMDDXX).
29+
$plugin->release = 2023051705; // Same as version.
3030
$plugin->requires = 2020110900; // Requires Filesystem API.
3131
$plugin->component = "tool_objectfs";
3232
$plugin->maturity = MATURITY_STABLE;

0 commit comments

Comments
 (0)