Skip to content

Commit 93eb02b

Browse files
authored
Merge pull request #1399 from guardian/an/aws-sdk-js-v3
Upgrade AWS JS SDK to v3
2 parents d4d2aaf + 43f8bfd commit 93eb02b

File tree

9 files changed

+1297
-233
lines changed

9 files changed

+1297
-233
lines changed

package.json

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,12 +65,14 @@
6565
"xhr2": "^0.1.4"
6666
},
6767
"dependencies": {
68+
"@aws-sdk/client-s3": "3.921.0",
69+
"@aws-sdk/credential-providers": "3.921.0",
70+
"@aws-sdk/xhr-http-handler": "3.921.0",
6871
"@dnd-kit/core": "^6.0.7",
6972
"@dnd-kit/sortable": "^7.0.2",
7073
"@dnd-kit/utilities": "^3.2.1",
7174
"@reduxjs/toolkit": "^2.8.2",
7275
"@vitejs/plugin-react-swc": "^3.7.1",
73-
"aws-sdk": "2.1679.0",
7476
"fast-check": "^3.6.3",
7577
"moment": "^2.29.4",
7678
"panda-session": "^0.1.6",

pluto-message-ingestion/package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
"tscconfig": "tsc --showConfig",
1515
"test": "vitest",
1616
"test:ci": "vitest --run",
17-
"build": "esbuild src/index.ts --bundle --minify --outfile=target/index.js --external:@aws-sdk --external:aws-sdk --platform=node"
17+
"build": "esbuild src/index.ts --bundle --minify --outfile=target/index.js --external:@aws-sdk --platform=node"
1818
},
1919
"dependencies": {
2020
"@aws-sdk/client-secrets-manager": "3.901.0",

public/video-ui/src/components/VideoUpload/VideoAsset.tsx

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import moment from 'moment';
22
import React, { ChangeEventHandler, ReactNode } from 'react';
33
import { useDispatch, useSelector } from 'react-redux';
44
import {
5+
ClientAsset,
56
deleteSubtitle,
67
startSubtitleFileUpload,
78
Upload
@@ -160,7 +161,7 @@ function AssetDisplay({
160161
);
161162
}
162163

163-
function AssetProgress({ failed, current, total }: Upload['processing']) {
164+
function AssetProgress({ failed, current, total }: ClientAsset['processing']) {
164165
if (failed) {
165166
return (
166167
<div>
@@ -266,7 +267,7 @@ export function Asset({
266267
activatingAssetNumber
267268
}: {
268269
videoId: string;
269-
upload: Upload;
270+
upload: ClientAsset;
270271
isActive: boolean;
271272
selectAsset: { (): void };
272273
deleteAsset: { (): void };

public/video-ui/src/components/VideoUpload/VideoTrail.tsx

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,14 +3,14 @@ import { useDispatch } from 'react-redux';
33
import { deleteAssets } from '../../actions/VideoActions/deleteAsset';
44
import { getVideo } from '../../actions/VideoActions/getVideo';
55
import { Video } from '../../services/VideosApi';
6-
import { Upload } from '../../slices/s3Upload';
6+
import {ClientAsset} from '../../slices/s3Upload';
77
import { getUploads } from '../../slices/uploads';
88
import { AppDispatch } from '../../util/setupStore';
99
import { Asset } from './VideoAsset';
1010

1111
type Props = {
1212
video: Video;
13-
uploads: Upload[];
13+
uploads: ClientAsset[];
1414
setAsset: (version: number) => void;
1515
activatingAssetNumber: number;
1616
};
@@ -43,7 +43,7 @@ export const VideoTrail = ({
4343
};
4444
}, [dispatch, uploads, video.id]);
4545

46-
const deleteAssetsInUpload = async (asset: Upload['asset']) => {
46+
const deleteAssetsInUpload = async (asset: ClientAsset['asset']) => {
4747
if (asset.id) {
4848
// if "asset.id" property exists, it should be a Youtube video asset.
4949
// There should be one asset for Youtube video and we can delete

public/video-ui/src/services/UploadsApi.js

Lines changed: 62 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -1,35 +1,29 @@
1-
import { apiRequest } from './apiRequest';
2-
import { errorDetails } from '../util/errorDetails';
3-
4-
// TO DO - convert to typescript, use defintion of `Upload` at public/video-ui/src/components/VideoUpload/VideoAsset.tsx
5-
6-
// See http://andrewhfarmer.com/aws-sdk-with-webpack/ for why this is strange
7-
import 'aws-sdk/dist/aws-sdk';
8-
const AWS = window.AWS;
9-
10-
// The timeout for individual upload requests. Defaults to 120s. This
11-
// default causes problems on slow connections – for example, w/ a 0.5mbps
12-
// upload speed (3.75MB/minute), uploads for 8mb chunks will never complete.
13-
AWS.config.httpOptions.timeout = 240_000; // in ms
14-
const httpOptions = {
15-
// The number of multipart uploads to run concurrently. Defaults to 4,
16-
// which has caused problems with slow connections timing out requests
17-
// prematurely. We judge allowing uploads on slow connections to be
18-
// more valuable than a minor boost in upload speed due to concurrent uploads.
19-
queueSize: 1
20-
};
1+
import {apiRequest} from './apiRequest';
2+
import {errorDetails} from '../util/errorDetails';
3+
import {S3} from '@aws-sdk/client-s3';
4+
import { XhrHttpHandler } from "@aws-sdk/xhr-http-handler";
5+
6+
7+
// TO DO - convert to typescript, use definition of `Upload` at public/video-ui/src/components/VideoUpload/VideoAsset.tsx
218

229
/**
23-
*
24-
* @param {string} atomId
25-
* @returns
10+
*
11+
* @param atomId {string}
12+
* @returns {Promise<unknown>}
2613
*/
2714
export function getUploads(atomId) {
2815
return apiRequest({
2916
url: `/api/uploads?atomId=${atomId}`
3017
});
3118
}
3219

20+
/**
21+
*
22+
* @param atomId {string}
23+
* @param file {File}
24+
* @param selfHost {boolean}
25+
* @returns {Promise<unknown>}
26+
*/
3327
export function createUpload(atomId, file, selfHost) {
3428
return apiRequest({
3529
url: `/api/uploads`,
@@ -56,49 +50,73 @@ function getCredentials(id, key) {
5650
});
5751
}
5852

59-
function getS3(bucket, region, credentials) {
53+
/**
54+
*
55+
* @param region {string}
56+
* @param credentials {any}
57+
* @returns {S3}
58+
*/
59+
function getS3(region, credentials) {
6060
const { temporaryAccessId, temporarySecretKey, sessionToken } = credentials;
61-
const awsCredentials = new AWS.Credentials(
62-
temporaryAccessId,
63-
temporarySecretKey,
64-
sessionToken
65-
);
66-
67-
return new AWS.S3({
68-
apiVersion: '2006-03-01',
61+
62+
const awsCredentials = {
63+
accessKeyId: temporaryAccessId,
64+
secretAccessKey: temporarySecretKey,
65+
sessionToken: sessionToken
66+
};
67+
68+
return new S3({
6969
credentials: awsCredentials,
70-
params: { Bucket: bucket },
70+
requestHandler: XhrHttpHandler.create({
71+
requestTimeout: 240_000
72+
}),
7173
region: region,
7274
useAccelerateEndpoint: true
7375
});
7476
}
7577

78+
/**
79+
* Upload single part of file
80+
*
81+
* @param upload {Upload}
82+
* @param part {typeof Upload['parts'][number]}
83+
* @param file {File}
84+
* @param progressFn {(completed: number) => any}
85+
* @returns {Promise<unknown>}
86+
*/
7687
function uploadPart(upload, part, file, progressFn) {
7788
const slice = file.slice(part.start, part.end);
7889

7990
return getCredentials(upload.id, part.key).then(credentials => {
8091
const s3 = getS3(
81-
upload.metadata.bucket,
8292
upload.metadata.region,
8393
credentials
8494
);
8595

86-
const params = {
96+
const request = slice.arrayBuffer().then(body => s3.putObject({
97+
Bucket: upload.metadata.bucket,
8798
Key: part.key,
88-
Body: slice,
89-
ACL: 'private',
90-
Metadata: { original: file.name }
91-
};
92-
const request = s3.upload(params, httpOptions);
93-
94-
request.on('httpUploadProgress', event => {
95-
progressFn(part.start + event.loaded);
99+
Metadata: { original: file.name },
100+
Body: body
101+
}));
102+
103+
request.then(() => {
104+
progressFn(part.end);
96105
});
97106

98-
return request.promise();
107+
return request;
99108
});
100109
}
101110

111+
/**
112+
* Recursively upload all parts of file
113+
*
114+
* @param upload {Upload}
115+
* @param parts {typeof Upload['parts']}
116+
* @param file {File}
117+
* @param progressFn {(completed: number) => any}
118+
* @returns {Promise<boolean>}
119+
*/
102120
export function uploadParts(upload, parts, file, progressFn) {
103121
return new Promise((resolve, reject) => {
104122
function uploadPartRecursive(parts) {

public/video-ui/src/slices/s3Upload.ts

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,23 +15,40 @@ export type SelfHostedAsset = {
1515
id?: undefined;
1616
sources: { src?: string; mimeType?: string }[];
1717
};
18+
19+
export type VideoAsset = YouTubeAsset | SelfHostedAsset;
20+
1821
export type Upload = {
1922
id: string;
20-
asset?: YouTubeAsset | SelfHostedAsset;
23+
parts?: { end: number; key: string; start: number }[];
24+
metadata: {
25+
user: string;
26+
bucket: string;
27+
region: string;
28+
title: string;
29+
// others...
30+
};
31+
progress: {
32+
// ...
33+
};
34+
};
35+
36+
export type ClientAsset = {
37+
id: string;
38+
asset?: VideoAsset;
2139
processing?: {
2240
status: string;
2341
failed: boolean;
2442
current?: number;
2543
total?: number;
2644
};
27-
parts?: { end: number; key: string; start: number }[];
2845
metadata?: {
2946
originalFilename?: string;
3047
subtitleFilename?: string;
3148
startTimestamp?: number;
3249
user: string;
3350
};
34-
};
51+
}
3552

3653
export interface S3UploadState {
3754
id: string | null;

public/video-ui/src/slices/uploads.ts

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,19 @@
11
import { createAsyncThunk, createSlice, PayloadAction } from '@reduxjs/toolkit';
2-
import { Upload } from './s3Upload';
2+
import {ClientAsset, Upload} from './s3Upload';
33
import * as UploadsApi from '../services/UploadsApi';
44
import { showError } from './error';
55
import { errorDetails } from '../util/errorDetails';
66

7-
export const getUploads = createAsyncThunk<Upload[], string>(
7+
export const getUploads = createAsyncThunk<ClientAsset[], string>(
88
'uploads/getUploads',
99
(atomId, { dispatch }) =>
10-
(UploadsApi.getUploads(atomId) as Promise<Upload[]>).catch(err => {
10+
(UploadsApi.getUploads(atomId) as Promise<ClientAsset[]>).catch(err => {
1111
dispatch(showError(errorDetails(err), err));
1212
throw err;
1313
})
1414
);
1515

16-
const initialState: Upload[] = [];
16+
const initialState: ClientAsset[] = [];
1717

1818
const uploads = createSlice({
1919
name: 'uploads',

scripts/purge-uploads-table.js

Lines changed: 0 additions & 72 deletions
This file was deleted.

0 commit comments

Comments
 (0)