Importing zip file fails because of a hidden implementation detail

[muratcorlu]

I tried to import images of a website by making them zip file and import with Ghost admin's import section. But this request failed with the error below:

Cannot read properties of undefined (reading 'split'): An unexpected error occurred, please try again.
  File "/var/lib/ghost/versions/5.82.1/node_modules/@tryghost/mw-error-handler/lib/mw-error-handler.js", line 102, col 19, in module.exports.prepareError
    err = new errors.InternalServerError({
  File "/var/lib/ghost/versions/5.82.1/core/server/data/importer/handlers/image.js", line 18, col 68, in Object.loadFile
    const imageFolderRegexes = _.map(store.staticFileURLPrefix.split('/'), function (dir) {
  File "/var/lib/ghost/versions/5.82.1/core/server/data/importer/import-manager.js", line 318, col 44, in ImportManager.processZip
    const data = await handler.loadFile(files, baseDir);
  File "/var/lib/ghost/versions/5.82.1/core/server/data/importer/import-manager.js", line 491, col 26, in ImportManager.importFromFile
    importData = await this.loadFile(file);
  File "/var/lib/ghost/versions/5.82.1/node_modules/@tryghost/api-framework/lib/pipeline.js", line 259, col 34, in getResponse
    const response = await STAGES.query(apiUtils, apiConfig, apiImpl, frame);
  File "/var/lib/ghost/versions/5.82.1/node_modules/@tryghost/api-framework/lib/pipeline.js", line 264, col 30, in wrapper
    const response = await getResponse();
  File "/var/lib/ghost/versions/5.82.1/node_modules/@tryghost/api-framework/lib/http.js", line 64, col 28, in Http
    const result = await apiImpl(frame);

Apparently it expects a staticFileURLPrefix property of storage adapter, which is not documented in Ghost docs at all.

This is reported with an issue here: TryGhost/Ghost#16530 But apparently it's ignored. I asked this in Ghost Forum as well: https://forum.ghost.org/t/custom-storage-adapter-documentation-is-incomplete/46499

I quickly tried to add this property by setting it to pathPrefix but it didn't work. If I'll have more time, I'll check more deeply. But I wanted to open this issue to make this visible anyway.

[syhily]

I met the same issue. It's quite easy to fix. Try this index.ts

import {
  type PutObjectCommandInput,
  type S3ClientConfig,
  type ObjectCannedACL,
  S3,
} from '@aws-sdk/client-s3'
import StorageBase, { type ReadOptions, type Image } from 'ghost-storage-base'
import errors from '@tryghost/errors'
import tpl from '@tryghost/tpl'
import { join } from 'path'
import { createReadStream } from 'fs'
import type { Readable } from 'stream'
import type { Handler } from 'express'

const stripLeadingSlash = (s: string) =>
  s.indexOf('/') === 0 ? s.substring(1) : s
const stripEndingSlash = (s: string) =>
  s.indexOf('/') === s.length - 1 ? s.substring(0, s.length - 1) : s

type Config = {
  accessKeyId?: string
  assetHost?: string
  bucket?: string
  pathPrefix?: string
  region?: string
  secretAccessKey?: string
  endpoint?: string
  forcePathStyle?: boolean
  useDualstackEndpoint?: boolean
  acl?: string
  staticFileURLPrefix?: string
}

class S3Storage extends StorageBase {
  accessKeyId?: string
  secretAccessKey?: string
  region?: string
  bucket?: string
  host: string
  pathPrefix: string
  endpoint: string
  forcePathStyle: boolean
  useDualstackEndpoint: boolean
  acl?: ObjectCannedACL
  staticFileURLPrefix: string

  constructor(config: Config = {}) {
    super()

    const {
      accessKeyId,
      assetHost,
      bucket,
      pathPrefix,
      region,
      secretAccessKey,
      endpoint,
      forcePathStyle,
      useDualstackEndpoint,
      acl,
      staticFileURLPrefix,
    } = config

    // Compatible with the aws-sdk's default environment variables
    this.accessKeyId = accessKeyId
    this.secretAccessKey = secretAccessKey
    this.region = process.env.AWS_DEFAULT_REGION || region

    this.bucket = process.env.GHOST_STORAGE_ADAPTER_S3_PATH_BUCKET || bucket

    if (!this.bucket) throw new Error('S3 bucket not specified')

    // Optional configurations
    this.forcePathStyle =
      Boolean(process.env.GHOST_STORAGE_ADAPTER_S3_FORCE_PATH_STYLE) ||
      Boolean(forcePathStyle) ||
      false

    this.useDualstackEndpoint =
      Boolean(process.env.GHOST_STORAGE_ADAPTER_S3_USE_DUALSTACK_ENDPOINT) ||
      Boolean(useDualstackEndpoint) ||
      false

    let defaultHost: string

    if (this.forcePathStyle) {
      defaultHost = `https://s3${
        this.region === 'us-east-1' ? '' : `.${this.region}`
      }.amazonaws.com/${this.bucket}`
    } else {
      defaultHost = `https://${this.bucket}.s3${
        this.region === 'us-east-1' ? '' : `.${this.region}`
      }.amazonaws.com`
    }

    this.host =
      process.env.GHOST_STORAGE_ADAPTER_S3_ASSET_HOST ||
      assetHost ||
      defaultHost

    this.pathPrefix = stripLeadingSlash(
      process.env.GHOST_STORAGE_ADAPTER_S3_PATH_PREFIX || pathPrefix || ''
    )
    this.endpoint =
      process.env.GHOST_STORAGE_ADAPTER_S3_ENDPOINT || endpoint || ''
    this.acl = (process.env.GHOST_STORAGE_ADAPTER_S3_ACL ||
      acl ||
      'public-read') as ObjectCannedACL
    this.staticFileURLPrefix = staticFileURLPrefix || 'content/images'
  }

  async delete(fileName: string, targetDir?: string) {
    const directory = targetDir || this.getTargetDir(this.pathPrefix)

    try {
      await this.s3().deleteObject({
        Bucket: this.bucket,
        Key: stripLeadingSlash(join(directory, fileName)),
      })
    } catch {
      return false
    }
    return true
  }

  async exists(fileName: string, targetDir?: string) {
    try {
      await this.s3().getObject({
        Bucket: this.bucket,
        Key: stripLeadingSlash(
          targetDir ? join(targetDir, fileName) : fileName
        ),
      })
    } catch {
      return false
    }
    return true
  }

  s3() {
    const options: S3ClientConfig = {
      region: this.region,
      forcePathStyle: this.forcePathStyle,
      useDualstackEndpoint: this.useDualstackEndpoint,
    }

    // Set credentials only if provided, falls back to AWS SDK's default provider chain
    if (this.accessKeyId && this.secretAccessKey) {
      options.credentials = {
        accessKeyId: this.accessKeyId,
        secretAccessKey: this.secretAccessKey,
      }
    }

    if (this.endpoint !== '') {
      options.endpoint = this.endpoint
    }
    return new S3(options)
  }

  // Doesn't seem to be documented, but required for using this adapter for other media file types.
  // Seealso: https://github.com/laosb/ghos3/pull/6
  urlToPath(url: string) {
    const parsedUrl = new URL(url)
    return parsedUrl.pathname
  }

  async save(image: Image, targetDir?: string) {
    const directory = targetDir || this.getTargetDir(this.pathPrefix)

    const fileName = await this.getUniqueFileName(image, directory)
    const file = createReadStream(image.path)

    let config: PutObjectCommandInput = {
      ACL: this.acl,
      Body: file,
      Bucket: this.bucket,
      CacheControl: `max-age=${30 * 24 * 60 * 60}`,
      ContentType: image.type,
      Key: stripLeadingSlash(fileName),
    }
    await this.s3().putObject(config)

    return `${this.host}/${stripLeadingSlash(fileName)}`
  }

  serve(): Handler {
    return async (req, res, next) => {
      try {
        const output = await this.s3().getObject({
          Bucket: this.bucket,
          Key: stripLeadingSlash(stripEndingSlash(this.pathPrefix) + req.path),
        })

        const headers: { [key: string]: string } = {}
        if (output.AcceptRanges) headers['accept-ranges'] = output.AcceptRanges
        if (output.CacheControl) headers['cache-control'] = output.CacheControl
        if (output.ContentDisposition)
          headers['content-disposition'] = output.ContentDisposition
        if (output.ContentEncoding)
          headers['content-encoding'] = output.ContentEncoding
        if (output.ContentLanguage)
          headers['content-language'] = output.ContentLanguage
        if (output.ContentLength)
          headers['content-length'] = `${output.ContentLength}`
        if (output.ContentRange) headers['content-range'] = output.ContentRange
        if (output.ContentType) headers['content-type'] = output.ContentType
        if (output.ETag) headers['etag'] = output.ETag
        res.set(headers)

        const stream = output.Body as Readable
        stream.pipe(res)
      } catch (err) {
        if (err.name === 'NoSuchKey') {
          return next(
            new errors.NotFoundError({
              message: tpl('File not found'),
              code: 'STATIC_FILE_NOT_FOUND',
              property: err.path,
            })
          )
        } else {
          next(new errors.InternalServerError({ err: err }))
        }
      }
    }
  }

  async read(options: ReadOptions = { path: '' }) {
    let path = (options.path || '').replace(/\/$|\\$/, '')

    // check if path is stored in s3 handled by us
    if (!path.startsWith(this.host)) {
      throw new Error(`${path} is not stored in s3`)
    }
    path = path.substring(this.host.length)

    const response = await this.s3().getObject({
      Bucket: this.bucket,
      Key: stripLeadingSlash(path),
    })
    const stream = response.Body as Readable

    return await new Promise<Buffer>((resolve, reject) => {
      const chunks: Buffer[] = []
      stream.on('data', (chunk) => chunks.push(chunk))
      stream.once('end', () => resolve(Buffer.concat(chunks)))
      stream.once('error', reject)
    })
  }
}

export default S3Storage