From f21327fcb202045149d6969cb474e1165e7abe8e Mon Sep 17 00:00:00 2001 From: Daniel Riess Date: Tue, 27 Sep 2022 17:19:23 +0700 Subject: [PATCH 1/8] add support for multipart range requests according to rfc 7233 --- lib/DAV/CorePlugin.php | 312 ++++++++++++++++++++++++----------------- lib/DAV/Server.php | 222 ++++++++++++++++++++++++----- 2 files changed, 370 insertions(+), 164 deletions(-) diff --git a/lib/DAV/CorePlugin.php b/lib/DAV/CorePlugin.php index dbd8976b17..c5bcde242b 100644 --- a/lib/DAV/CorePlugin.php +++ b/lib/DAV/CorePlugin.php @@ -66,141 +66,195 @@ public function getPluginName() } /** - * This is the default implementation for the GET method. - * - * @return bool - */ - public function httpGet(RequestInterface $request, ResponseInterface $response) - { - $path = $request->getPath(); - $node = $this->server->tree->getNodeForPath($path); - - if (!$node instanceof IFile) { - return; - } - - if ('HEAD' === $request->getHeader('X-Sabre-Original-Method')) { - $body = ''; - } else { - $body = $node->get(); - - // Converting string into stream, if needed. - if (is_string($body)) { - $stream = fopen('php://temp', 'r+'); - fwrite($stream, $body); - rewind($stream); - $body = $stream; - } - } - - /* + * This is the default implementation for the GET method. + * + * @return bool + */ + public function httpGet(RequestInterface $request, ResponseInterface $response) + { + $path = $request->getPath(); + $node = $this->server->tree->getNodeForPath($path); + + if (!$node instanceof IFile) { + return; + } + + if ('HEAD' === $request->getHeader('X-Sabre-Original-Method')) { + $body = ''; + } else { + $body = $node->get(); + + // Converting string into stream, if needed. + if (is_string($body)) { + $stream = fopen('php://temp', 'r+'); + fwrite($stream, $body); + rewind($stream); + $body = $stream; + } + } + + /* * TODO: getetag, getlastmodified, getsize should also be used using * this method */ - $httpHeaders = $this->server->getHTTPHeaders($path); + $httpHeaders = $this->server->getHTTPHeaders($path); - /* ContentType needs to get a default, because many webservers will otherwise + /* ContentType needs to get a default, because many webservers will otherwise * default to text/html, and we don't want this for security reasons. */ - if (!isset($httpHeaders['Content-Type'])) { - $httpHeaders['Content-Type'] = 'application/octet-stream'; - } - - if (isset($httpHeaders['Content-Length'])) { - $nodeSize = $httpHeaders['Content-Length']; - - // Need to unset Content-Length, because we'll handle that during figuring out the range - unset($httpHeaders['Content-Length']); - } else { - $nodeSize = null; - } - - $response->addHeaders($httpHeaders); - - $range = $this->server->getHTTPRange(); - $ifRange = $request->getHeader('If-Range'); - $ignoreRangeHeader = false; - - // If ifRange is set, and range is specified, we first need to check - // the precondition. - if ($nodeSize && $range && $ifRange) { - // if IfRange is parsable as a date we'll treat it as a DateTime - // otherwise, we must treat it as an etag. - try { - $ifRangeDate = new \DateTime($ifRange); - - // It's a date. We must check if the entity is modified since - // the specified date. - if (!isset($httpHeaders['Last-Modified'])) { - $ignoreRangeHeader = true; - } else { - $modified = new \DateTime($httpHeaders['Last-Modified']); - if ($modified > $ifRangeDate) { - $ignoreRangeHeader = true; - } - } - } catch (\Exception $e) { - // It's an entity. We can do a simple comparison. - if (!isset($httpHeaders['ETag'])) { - $ignoreRangeHeader = true; - } elseif ($httpHeaders['ETag'] !== $ifRange) { - $ignoreRangeHeader = true; - } - } - } - - // We're only going to support HTTP ranges if the backend provided a filesize - if (!$ignoreRangeHeader && $nodeSize && $range) { - // Determining the exact byte offsets - if (!is_null($range[0])) { - $start = $range[0]; - $end = $range[1] ? $range[1] : $nodeSize - 1; - if ($start >= $nodeSize) { - throw new Exception\RequestedRangeNotSatisfiable('The start offset ('.$range[0].') exceeded the size of the entity ('.$nodeSize.')'); - } - if ($end < $start) { - throw new Exception\RequestedRangeNotSatisfiable('The end offset ('.$range[1].') is lower than the start offset ('.$range[0].')'); - } - if ($end >= $nodeSize) { - $end = $nodeSize - 1; - } - } else { - $start = $nodeSize - $range[1]; - $end = $nodeSize - 1; - - if ($start < 0) { - $start = 0; - } - } - - // Streams may advertise themselves as seekable, but still not - // actually allow fseek. We'll manually go forward in the stream - // if fseek failed. - if (!stream_get_meta_data($body)['seekable'] || -1 === fseek($body, $start, SEEK_SET)) { - $consumeBlock = 8192; - for ($consumed = 0; $start - $consumed > 0;) { - if (feof($body)) { - throw new Exception\RequestedRangeNotSatisfiable('The start offset ('.$start.') exceeded the size of the entity ('.$consumed.')'); - } - $consumed += strlen(fread($body, min($start - $consumed, $consumeBlock))); - } - } - - $response->setHeader('Content-Length', $end - $start + 1); - $response->setHeader('Content-Range', 'bytes '.$start.'-'.$end.'/'.$nodeSize); - $response->setStatus(206); - $response->setBody($body); - } else { - if ($nodeSize) { - $response->setHeader('Content-Length', $nodeSize); - } - $response->setStatus(200); - $response->setBody($body); - } - // Sending back false will interrupt the event chain and tell the server - // we've handled this method. - return false; - } + if (!isset($httpHeaders['Content-Type'])) { + $httpHeaders['Content-Type'] = 'application/octet-stream'; + } + + if (isset($httpHeaders['Content-Length'])) { + $nodeSize = $httpHeaders['Content-Length']; + + // Need to unset Content-Length, because we'll handle that during figuring out the range + unset($httpHeaders['Content-Length']); + } else { + $nodeSize = null; + } + + $response->addHeaders($httpHeaders); + + // this function now only checks, if the range string correctly formatted + // and returns it (or null if it isn't or it doesn't exist) + // processing of the ranges will then be done further below + $rangeString = $this->server->getHTTPRange(); + $ifRange = $request->getHeader('If-Range'); + $ignoreRangeHeader = false; + + // If ifRange is set, and range is specified, we first need to check + // the precondition. + if ($nodeSize && $rangeString && $ifRange) { + // if IfRange is parsable as a date we'll treat it as a DateTime + // otherwise, we must treat it as an etag. + try { + $ifRangeDate = new \DateTime($ifRange); + + // It's a date. We must check if the entity is modified since + // the specified date. + if (!isset($httpHeaders['Last-Modified'])) { + $ignoreRangeHeader = true; + } else { + $modified = new \DateTime($httpHeaders['Last-Modified']); + if ($modified > $ifRangeDate) { + $ignoreRangeHeader = true; + } + } + } catch (\Exception $e) { + // It's an entity. We can do a simple comparison. + if (!isset($httpHeaders['ETag'])) { + $ignoreRangeHeader = true; + } elseif ($httpHeaders['ETag'] !== $ifRange) { + $ignoreRangeHeader = true; + } + } + } + + if (!$ignoreRangeHeader && $nodeSize && $rangeString) { + // preprocess the ranges now + // this involves removing invalid ranges and security checks + $ranges = $this->server->preprocessRanges($rangeString, $nodeSize); + } + + if (!$ignoreRangeHeader && $nodeSize && $ranges) { + // create a new stream for the partial responses + $rangeBody = fopen('php://temp', 'r+'); + + // create a boundary string that is being used as identifier for multipart ranges + $boundary = hash('md5', uniqid('', true)); + + // create a content length counter + $contentLength = 0; + + foreach ($ranges as $range) { + $start = $range[0]; + $end = $range[1]; + + // Streams may advertise themselves as seekable, but still not + // actually allow fseek. We'll manually go forward in the stream + // if fseek failed. + if (!stream_get_meta_data($body)['seekable'] || -1 === fseek($body, $start, SEEK_SET)) { + // because of potential multiple ranges with descending order, rewind stream after finishing each range + if (!rewind($body)) { + throw new Exception\PreconditionFailed('Could not rewind the response stream'); + } + $consumeBlock = 8192; + for ($consumed = 0; $start - $consumed > 0;) { + if (feof($body)) { + throw new Exception\RequestedRangeNotSatisfiable('The start offset (' . $start . ') exceeded the size of the entity (' . $consumed . ')'); + } + $consumed += strlen(fread($body, min($start - $consumed, $consumeBlock))); + } + } + + // if there is only one range, write it into the reponse body + // otherwise, write multipart header info first before writing the data + if (count($ranges) === 1) { + $rangeLength = $end - $start + 1; + $consumeBlock = 8192; + + // write body data into response body (max 8192 bytes per read cycle) + while ($rangeLength > 0) { + fwrite($rangeBody, fread($body, min($rangeLength, $consumeBlock))); + $rangeLength -= min($rangeLength, $consumeBlock); + } + + // update content length + $contentLength = $end - $start + 1; + } else { + // define new boundary section and write to response body + $boundarySection = "--" . $boundary . "\nContent-Type: " . $node->getContentType() . "\nContent-Range: bytes " . $start . "-" . $end . '/' . $nodeSize . "\n\n"; + fwrite($rangeBody, $boundarySection); + + // write body data into response body (max 8192 bytes per read cycle) + $rangeLength = $end - $start + 1; + $consumeBlock = 8192; + while ($rangeLength > 0) { + fwrite($rangeBody, fread($body, min($rangeLength, $consumeBlock))); + $rangeLength -= min($rangeLength, $consumeBlock); + } + + // write line breaks at the end of section + fwrite($rangeBody, "\n\n"); + + // update content length + // +2 for double linebreak at the end of each section + $contentLength += strlen($boundarySection) + ($end - $start + 1) + 2; + } + } + + // rewind the range body after the loop + if (!rewind($rangeBody)) { + throw new Exception\PreconditionFailed('Could not rewind the response stream'); + } + + if (count($ranges) === 1) { + // if only one range, content range header MUST be set (according to spec) + $response->setHeader('Content-Range', 'bytes ' . $start . '-' . $end . '/' . $nodeSize); + } else { + // if multiple ranges, content range header MUST NOT be set (according to spec) + // content type header MUST be of type multipart/byteranges + // more specific types can be set in each body section header + $response->setHeader('Content-Type', 'multipart/byteranges; boundary=' . $boundary); + } + + $response->setHeader('Content-Length', $contentLength); + $response->setHeader('Vary', 'Range'); + + $response->setStatus(206); + $response->setBody($rangeBody); + } else { + if ($nodeSize) { + $response->setHeader('Content-Length', $nodeSize); + } + $response->setStatus(200); + $response->setBody($body); + } + + return false; + } /** * HTTP OPTIONS. diff --git a/lib/DAV/Server.php b/lib/DAV/Server.php index 1f8300d4a5..651649c027 100644 --- a/lib/DAV/Server.php +++ b/lib/DAV/Server.php @@ -609,41 +609,193 @@ public function getHTTPDepth($default = self::DEPTH_INFINITY) } /** - * Returns the HTTP range header. - * - * This method returns null if there is no well-formed HTTP range request - * header or array($start, $end). - * - * The first number is the offset of the first byte in the range. - * The second number is the offset of the last byte in the range. - * - * If the second offset is null, it should be treated as the offset of the last byte of the entity - * If the first offset is null, the second offset should be used to retrieve the last x bytes of the entity - * - * @return int[]|null - */ - public function getHTTPRange() - { - $range = $this->httpRequest->getHeader('range'); - if (is_null($range)) { - return null; - } - - // Matching "Range: bytes=1234-5678: both numbers are optional - - if (!preg_match('/^bytes=([0-9]*)-([0-9]*)$/i', $range, $matches)) { - return null; - } - - if ('' === $matches[1] && '' === $matches[2]) { - return null; - } - - return [ - '' !== $matches[1] ? (int) $matches[1] : null, - '' !== $matches[2] ? (int) $matches[2] : null, - ]; - } + * Returns the HTTP range header. + * + * This method returns null if there is no well-formed HTTP range request + * header. + * + * If the string is valid, the ranges will be evaluated and processed in + * the next function + * + * @return string|null + */ + public function getHTTPRange() + { + $rangeHeader = $this->httpRequest->getHeader('range'); + + if (is_null($rangeHeader)) { + return null; + } + + // remove all spaces before evaluation + $rangeHeader = str_replace(' ', '', $rangeHeader); + + // regex to allow multiple ranges + // e.g.'Range: bytes=0-50,100-140, 200-500' + if (!preg_match('/^bytes=(([0-9]*-{1}[0-9]*,? ?)*)$/i', $rangeHeader, $matches)) { + return null; + } + + // remove trailing comma + $matches[1] = trim($matches[1], ','); + + // remove the range string + return $matches[1]; + } + + /** + * Returns the processed ranges of the previously parsed range header + * + * Each range consists of 2 numbers + * The first number specifies the offset of the first byte in the range + * The second number specifies the offset of the last byte in the range + * + * If the second offset is null, it should be treated as the offset of the last byte of the entity + * If the first offset is null, the second offset should be used to retrieve the last x bytes of the entity + * + * If multiple ranges are specified, they will be preprocessed to avoid DOS attacks + * Amount of ranges will be capped dependent on boundary size + * Overlapping ranges will be capped to 2 per request + * Ranges in descending order will be capped dependent on boundary size + * + * @throws Exception\PreconditionFailed if ranges overlap or too many ranges are being requested + * @throws Exception\RequestedRangeNotSatisfiable if an invalid range is requested + * + * @return array|null + */ + public function preprocessRanges($rangeString, $nodeSize) + { + // create an array of all ranges + // result example: [[0, 100], [200, 300]] + $ranges = explode(',', $rangeString); + + // if an invalid range is encountered, its index will be saved in here + // invalid ranges will be removed after the for loop + // examples: start > end, '---200-300---', '-' + $rangesToRemove = []; + + // loop over the array of ranges and do some basic checks and transforms + for ($i = 0; $i < count($ranges); $i++) { + $ranges[$i] = explode('-', $ranges[$i]); + + // mark invalid ranges for removal + // "---200-300---" would be considered a valid range by the regexp + if (count($ranges[$i]) !== 2) { + $rangesToRemove[] = $i; + continue; + } + + // copy the ranges into temporary variables for comparisons + $start = $ranges[$i][0]; + $end = $ranges[$i][1]; + + // if neither start nor end value is defined, mark range for removal + if ($start === "" && $end === "") { + $rangesToRemove[] = $i; + continue; + } + + // transform range parameters for special ranges immediately + // this will make security checks later in this function much simpler + if ($start === "") { + $ranges[$i][0] = $nodeSize - intval($end); + $ranges[$i][1] = $nodeSize - 1; + } else if ($end === "" || $end > ($nodeSize - 1)) { + $ranges[$i][0] = intval($start); + $ranges[$i][1] = $nodeSize - 1; + } else { + $ranges[$i][0] = intval($start); + $ranges[$i][1] = intval($end); + } + + // if start > end, mark it for removal + if ($ranges[$i][0] > $ranges[$i][1]) { + $rangesToRemove[] = $i; + } + } + + // remove invalid ranges + foreach ($rangesToRemove as $idx) { + unset($ranges[$idx]); + } + + // if no ranges are left at this point, return + if (count($ranges) === 0) { + return null; + } + + // rearrange the array keys + $ranges = array_values($ranges); + + /** + * implement checks to prevent DOS attacks, according to rfc7233, 6.1 + */ + + // set up a sorted copy of the ranges array and define the boundary + $sortedRanges = $ranges; + sort($sortedRanges); + + $minRangeBoundary = $sortedRanges[0][0]; + $maxRangeBoundary = max(array_column($sortedRanges, 1)); + $rangeBoundary = $maxRangeBoundary - $minRangeBoundary; + + // check 1: limit the amount of ranges per request depending on range boundary + // accept 1024 requests for a boundary < 1MB, and 512 for boundaries larger than that + // rfc standard doesn't provide specification for the amount of ranges to be allowed per request + if ( + ($rangeBoundary < (1 << 20) && count($ranges) > 1024) + || ($rangeBoundary > (1 << 20) && count($ranges) > 512) + ) { + throw new Exception\PreconditionFailed('Too many ranges in this request'); + } + + // check2: check for overlapping ranges and allow a maximum of 2 + // compare each start value with either the previous end value + // or current max end range value, whichever is bigger + $overlapCounter = 0; + $maxEndValue = -1; + + foreach ($sortedRanges as $range) { + // compare start value with the current maxEndValue + if ($range[0] <= $maxEndValue) { + $overlapCounter++; + + if ($overlapCounter > 2) { + throw new Exception\PreconditionFailed('Too many overlaps'); + } + } + + // check if this ranges end value is larger than maxEndValue and update + if ($range[1] > $maxEndValue) { + $maxEndValue = $range[1]; + } + } + + // check3: ranges should be requested in ascending order + // it can be useful to do occasional descending requests, i.e. if a necessary + // piece of information is located at the end of the file. however, if a range + // consists of many unordered ranges, it can be indicative of a deliberate attack + // limit descending ranges to 32 for boundaries < 1MB and 16 for boundaries larger than that + $descendingStartCounter = 0; + + foreach ($ranges as $idx => $range) { + // skip the first range + if ($idx === 0) continue; + + if ($range[0] < $ranges[$idx - 1][0]) { + $descendingStartCounter++; + } + } + + if ( + ($rangeBoundary < (1 << 20) && $descendingStartCounter > 32) + || ($rangeBoundary > (1 << 20) && $descendingStartCounter > 16) + ) { + throw new Exception\PreconditionFailed('Too many ranges in this request'); + } + + return $ranges; + } /** * Returns the HTTP Prefer header information. From 254258d9317247e94e1d879404c4ffd8b56b21bb Mon Sep 17 00:00:00 2001 From: Daniel Riess Date: Fri, 30 Sep 2022 15:17:22 +0700 Subject: [PATCH 2/8] cs-fix, added some unit tests --- lib/DAV/CorePlugin.php | 375 ++++++++++++++------------- lib/DAV/Server.php | 382 ++++++++++++++-------------- tests/Sabre/DAV/ServerRangeTest.php | 102 ++++++++ tests/phpunit.xml | 2 +- 4 files changed, 490 insertions(+), 371 deletions(-) diff --git a/lib/DAV/CorePlugin.php b/lib/DAV/CorePlugin.php index c5bcde242b..4352f55202 100644 --- a/lib/DAV/CorePlugin.php +++ b/lib/DAV/CorePlugin.php @@ -66,195 +66,204 @@ public function getPluginName() } /** - * This is the default implementation for the GET method. - * - * @return bool - */ - public function httpGet(RequestInterface $request, ResponseInterface $response) - { - $path = $request->getPath(); - $node = $this->server->tree->getNodeForPath($path); - - if (!$node instanceof IFile) { - return; - } - - if ('HEAD' === $request->getHeader('X-Sabre-Original-Method')) { - $body = ''; - } else { - $body = $node->get(); - - // Converting string into stream, if needed. - if (is_string($body)) { - $stream = fopen('php://temp', 'r+'); - fwrite($stream, $body); - rewind($stream); - $body = $stream; - } - } - - /* + * This is the default implementation for the GET method. + * + * @return bool + */ + public function httpGet(RequestInterface $request, ResponseInterface $response) + { + $path = $request->getPath(); + $node = $this->server->tree->getNodeForPath($path); + + if (!$node instanceof IFile) { + return; + } + + if ('HEAD' === $request->getHeader('X-Sabre-Original-Method')) { + $body = ''; + } else { + $body = $node->get(); + + // Converting string into stream, if needed. + if (is_string($body)) { + $stream = fopen('php://temp', 'r+'); + fwrite($stream, $body); + rewind($stream); + $body = $stream; + } + } + + /* * TODO: getetag, getlastmodified, getsize should also be used using * this method */ - $httpHeaders = $this->server->getHTTPHeaders($path); + $httpHeaders = $this->server->getHTTPHeaders($path); - /* ContentType needs to get a default, because many webservers will otherwise + /* ContentType needs to get a default, because many webservers will otherwise * default to text/html, and we don't want this for security reasons. */ - if (!isset($httpHeaders['Content-Type'])) { - $httpHeaders['Content-Type'] = 'application/octet-stream'; - } - - if (isset($httpHeaders['Content-Length'])) { - $nodeSize = $httpHeaders['Content-Length']; - - // Need to unset Content-Length, because we'll handle that during figuring out the range - unset($httpHeaders['Content-Length']); - } else { - $nodeSize = null; - } - - $response->addHeaders($httpHeaders); - - // this function now only checks, if the range string correctly formatted - // and returns it (or null if it isn't or it doesn't exist) - // processing of the ranges will then be done further below - $rangeString = $this->server->getHTTPRange(); - $ifRange = $request->getHeader('If-Range'); - $ignoreRangeHeader = false; - - // If ifRange is set, and range is specified, we first need to check - // the precondition. - if ($nodeSize && $rangeString && $ifRange) { - // if IfRange is parsable as a date we'll treat it as a DateTime - // otherwise, we must treat it as an etag. - try { - $ifRangeDate = new \DateTime($ifRange); - - // It's a date. We must check if the entity is modified since - // the specified date. - if (!isset($httpHeaders['Last-Modified'])) { - $ignoreRangeHeader = true; - } else { - $modified = new \DateTime($httpHeaders['Last-Modified']); - if ($modified > $ifRangeDate) { - $ignoreRangeHeader = true; - } - } - } catch (\Exception $e) { - // It's an entity. We can do a simple comparison. - if (!isset($httpHeaders['ETag'])) { - $ignoreRangeHeader = true; - } elseif ($httpHeaders['ETag'] !== $ifRange) { - $ignoreRangeHeader = true; - } - } - } - - if (!$ignoreRangeHeader && $nodeSize && $rangeString) { - // preprocess the ranges now - // this involves removing invalid ranges and security checks - $ranges = $this->server->preprocessRanges($rangeString, $nodeSize); - } - - if (!$ignoreRangeHeader && $nodeSize && $ranges) { - // create a new stream for the partial responses - $rangeBody = fopen('php://temp', 'r+'); - - // create a boundary string that is being used as identifier for multipart ranges - $boundary = hash('md5', uniqid('', true)); - - // create a content length counter - $contentLength = 0; - - foreach ($ranges as $range) { - $start = $range[0]; - $end = $range[1]; - - // Streams may advertise themselves as seekable, but still not - // actually allow fseek. We'll manually go forward in the stream - // if fseek failed. - if (!stream_get_meta_data($body)['seekable'] || -1 === fseek($body, $start, SEEK_SET)) { - // because of potential multiple ranges with descending order, rewind stream after finishing each range - if (!rewind($body)) { - throw new Exception\PreconditionFailed('Could not rewind the response stream'); - } - $consumeBlock = 8192; - for ($consumed = 0; $start - $consumed > 0;) { - if (feof($body)) { - throw new Exception\RequestedRangeNotSatisfiable('The start offset (' . $start . ') exceeded the size of the entity (' . $consumed . ')'); - } - $consumed += strlen(fread($body, min($start - $consumed, $consumeBlock))); - } - } - - // if there is only one range, write it into the reponse body - // otherwise, write multipart header info first before writing the data - if (count($ranges) === 1) { - $rangeLength = $end - $start + 1; - $consumeBlock = 8192; - - // write body data into response body (max 8192 bytes per read cycle) - while ($rangeLength > 0) { - fwrite($rangeBody, fread($body, min($rangeLength, $consumeBlock))); - $rangeLength -= min($rangeLength, $consumeBlock); - } - - // update content length - $contentLength = $end - $start + 1; - } else { - // define new boundary section and write to response body - $boundarySection = "--" . $boundary . "\nContent-Type: " . $node->getContentType() . "\nContent-Range: bytes " . $start . "-" . $end . '/' . $nodeSize . "\n\n"; - fwrite($rangeBody, $boundarySection); - - // write body data into response body (max 8192 bytes per read cycle) - $rangeLength = $end - $start + 1; - $consumeBlock = 8192; - while ($rangeLength > 0) { - fwrite($rangeBody, fread($body, min($rangeLength, $consumeBlock))); - $rangeLength -= min($rangeLength, $consumeBlock); - } - - // write line breaks at the end of section - fwrite($rangeBody, "\n\n"); - - // update content length - // +2 for double linebreak at the end of each section - $contentLength += strlen($boundarySection) + ($end - $start + 1) + 2; - } - } - - // rewind the range body after the loop - if (!rewind($rangeBody)) { - throw new Exception\PreconditionFailed('Could not rewind the response stream'); - } - - if (count($ranges) === 1) { - // if only one range, content range header MUST be set (according to spec) - $response->setHeader('Content-Range', 'bytes ' . $start . '-' . $end . '/' . $nodeSize); - } else { - // if multiple ranges, content range header MUST NOT be set (according to spec) - // content type header MUST be of type multipart/byteranges - // more specific types can be set in each body section header - $response->setHeader('Content-Type', 'multipart/byteranges; boundary=' . $boundary); - } - - $response->setHeader('Content-Length', $contentLength); - $response->setHeader('Vary', 'Range'); - - $response->setStatus(206); - $response->setBody($rangeBody); - } else { - if ($nodeSize) { - $response->setHeader('Content-Length', $nodeSize); - } - $response->setStatus(200); - $response->setBody($body); - } - - return false; - } + if (!isset($httpHeaders['Content-Type'])) { + $httpHeaders['Content-Type'] = 'application/octet-stream'; + } + + if (isset($httpHeaders['Content-Length'])) { + $nodeSize = $httpHeaders['Content-Length']; + + // Need to unset Content-Length, because we'll handle that during figuring out the range + unset($httpHeaders['Content-Length']); + } else { + $nodeSize = null; + } + + $response->addHeaders($httpHeaders); + + // this function now only checks, if the range string correctly formatted + // and returns it (or null if it isn't or it doesn't exist) + // processing of the ranges will then be done further below + $rangeString = $this->server->getHTTPRange(); + $ifRange = $request->getHeader('If-Range'); + $ignoreRangeHeader = false; + + // If ifRange is set, and range is specified, we first need to check + // the precondition. + if ($nodeSize && $rangeString && $ifRange) { + // if IfRange is parsable as a date we'll treat it as a DateTime + // otherwise, we must treat it as an etag. + try { + $ifRangeDate = new \DateTime($ifRange); + + // It's a date. We must check if the entity is modified since + // the specified date. + if (!isset($httpHeaders['Last-Modified'])) { + $ignoreRangeHeader = true; + } else { + $modified = new \DateTime($httpHeaders['Last-Modified']); + if ($modified > $ifRangeDate) { + $ignoreRangeHeader = true; + } + } + } catch (\Exception $e) { + // It's an entity. We can do a simple comparison. + if (!isset($httpHeaders['ETag'])) { + $ignoreRangeHeader = true; + } elseif ($httpHeaders['ETag'] !== $ifRange) { + $ignoreRangeHeader = true; + } + } + } + + $ranges = null; + + if (!$ignoreRangeHeader && $nodeSize && $rangeString) { + // preprocess the ranges now + // this involves removing invalid ranges and security checks + $ranges = $this->server->preprocessRanges($rangeString, $nodeSize); + } + + if (!$ignoreRangeHeader && $nodeSize && $ranges) { + // create a new stream for the partial responses + $rangeBody = fopen('php://temp', 'r+'); + + // create a boundary string that is being used as identifier for multipart ranges + $boundary = hash('md5', uniqid('', true)); + + // create a content length counter + $contentLength = 0; + + // define the node's content type + $nodeContentType = $node->getContentType(); + + if (!$nodeContentType) { + $nodeContentType = 'application/octet-stream'; + } + + foreach ($ranges as $range) { + $start = $range[0]; + $end = $range[1]; + + // Streams may advertise themselves as seekable, but still not + // actually allow fseek. We'll manually go forward in the stream + // if fseek failed. + if (!stream_get_meta_data($body)['seekable'] || -1 === fseek($body, $start, SEEK_SET)) { + // don't allow multipart ranges for non-seekable streams + // these streams are not rewindable, so they would have to be closed and opened for each range + if (count($ranges) > 1) { + throw new Exception\RequestedRangeNotSatisfiable('Multipart range requests are not allowed on a non-seekable resource.'); + } + $consumeBlock = 8192; + for ($consumed = 0; $start - $consumed > 0;) { + if (feof($body)) { + throw new Exception\RequestedRangeNotSatisfiable('The start offset ('.$start.') exceeded the size of the entity ('.$consumed.')'); + } + $consumed += strlen(fread($body, min($start - $consumed, $consumeBlock))); + } + } + + // if there is only one range, write it into the reponse body + // otherwise, write multipart header info first before writing the data + if (1 === count($ranges)) { + $rangeLength = $end - $start + 1; + $consumeBlock = 8192; + + // write body data into response body (max 8192 bytes per read cycle) + while ($rangeLength > 0) { + fwrite($rangeBody, fread($body, min($rangeLength, $consumeBlock))); + $rangeLength -= min($rangeLength, $consumeBlock); + } + + // update content length + $contentLength = $end - $start + 1; + } else { + // define new boundary section and write to response body + $boundarySection = '--'.$boundary."\nContent-Type: ".$nodeContentType."\nContent-Range: bytes ".$start.'-'.$end.'/'.$nodeSize."\n\n"; + fwrite($rangeBody, $boundarySection); + + // write body data into response body (max 8192 bytes per read cycle) + $rangeLength = $end - $start + 1; + $consumeBlock = 8192; + while ($rangeLength > 0) { + fwrite($rangeBody, fread($body, min($rangeLength, $consumeBlock))); + $rangeLength -= min($rangeLength, $consumeBlock); + } + + // write line breaks at the end of section + fwrite($rangeBody, "\n\n"); + + // update content length + // +2 for double linebreak at the end of each section + $contentLength += strlen($boundarySection) + ($end - $start + 1) + 2; + } + } + + // rewind the range body after the loop + if (!rewind($rangeBody)) { + throw new Exception\PreconditionFailed('Could not rewind the response stream'); + } + + if (1 === count($ranges)) { + // if only one range, content range header MUST be set (according to spec) + $response->setHeader('Content-Range', 'bytes '.$start.'-'.$end.'/'.$nodeSize); + } else { + // if multiple ranges, content range header MUST NOT be set (according to spec) + // content type header MUST be of type multipart/byteranges + // more specific types can be set in each body section header + $response->setHeader('Content-Type', 'multipart/byteranges; boundary='.$boundary); + } + + $response->setHeader('Content-Length', $contentLength); + + $response->setStatus(206); + $response->setBody($rangeBody); + } else { + if ($nodeSize) { + $response->setHeader('Content-Length', $nodeSize); + } + $response->setStatus(200); + $response->setBody($body); + } + + return false; + } /** * HTTP OPTIONS. diff --git a/lib/DAV/Server.php b/lib/DAV/Server.php index 651649c027..25a654ceb2 100644 --- a/lib/DAV/Server.php +++ b/lib/DAV/Server.php @@ -609,193 +609,201 @@ public function getHTTPDepth($default = self::DEPTH_INFINITY) } /** - * Returns the HTTP range header. - * - * This method returns null if there is no well-formed HTTP range request - * header. - * - * If the string is valid, the ranges will be evaluated and processed in - * the next function - * - * @return string|null - */ - public function getHTTPRange() - { - $rangeHeader = $this->httpRequest->getHeader('range'); - - if (is_null($rangeHeader)) { - return null; - } - - // remove all spaces before evaluation - $rangeHeader = str_replace(' ', '', $rangeHeader); - - // regex to allow multiple ranges - // e.g.'Range: bytes=0-50,100-140, 200-500' - if (!preg_match('/^bytes=(([0-9]*-{1}[0-9]*,? ?)*)$/i', $rangeHeader, $matches)) { - return null; - } - - // remove trailing comma - $matches[1] = trim($matches[1], ','); - - // remove the range string - return $matches[1]; - } - - /** - * Returns the processed ranges of the previously parsed range header - * - * Each range consists of 2 numbers - * The first number specifies the offset of the first byte in the range - * The second number specifies the offset of the last byte in the range - * - * If the second offset is null, it should be treated as the offset of the last byte of the entity - * If the first offset is null, the second offset should be used to retrieve the last x bytes of the entity - * - * If multiple ranges are specified, they will be preprocessed to avoid DOS attacks - * Amount of ranges will be capped dependent on boundary size - * Overlapping ranges will be capped to 2 per request - * Ranges in descending order will be capped dependent on boundary size - * - * @throws Exception\PreconditionFailed if ranges overlap or too many ranges are being requested - * @throws Exception\RequestedRangeNotSatisfiable if an invalid range is requested - * - * @return array|null - */ - public function preprocessRanges($rangeString, $nodeSize) - { - // create an array of all ranges - // result example: [[0, 100], [200, 300]] - $ranges = explode(',', $rangeString); - - // if an invalid range is encountered, its index will be saved in here - // invalid ranges will be removed after the for loop - // examples: start > end, '---200-300---', '-' - $rangesToRemove = []; - - // loop over the array of ranges and do some basic checks and transforms - for ($i = 0; $i < count($ranges); $i++) { - $ranges[$i] = explode('-', $ranges[$i]); - - // mark invalid ranges for removal - // "---200-300---" would be considered a valid range by the regexp - if (count($ranges[$i]) !== 2) { - $rangesToRemove[] = $i; - continue; - } - - // copy the ranges into temporary variables for comparisons - $start = $ranges[$i][0]; - $end = $ranges[$i][1]; - - // if neither start nor end value is defined, mark range for removal - if ($start === "" && $end === "") { - $rangesToRemove[] = $i; - continue; - } - - // transform range parameters for special ranges immediately - // this will make security checks later in this function much simpler - if ($start === "") { - $ranges[$i][0] = $nodeSize - intval($end); - $ranges[$i][1] = $nodeSize - 1; - } else if ($end === "" || $end > ($nodeSize - 1)) { - $ranges[$i][0] = intval($start); - $ranges[$i][1] = $nodeSize - 1; - } else { - $ranges[$i][0] = intval($start); - $ranges[$i][1] = intval($end); - } - - // if start > end, mark it for removal - if ($ranges[$i][0] > $ranges[$i][1]) { - $rangesToRemove[] = $i; - } - } - - // remove invalid ranges - foreach ($rangesToRemove as $idx) { - unset($ranges[$idx]); - } - - // if no ranges are left at this point, return - if (count($ranges) === 0) { - return null; - } - - // rearrange the array keys - $ranges = array_values($ranges); - - /** - * implement checks to prevent DOS attacks, according to rfc7233, 6.1 - */ - - // set up a sorted copy of the ranges array and define the boundary - $sortedRanges = $ranges; - sort($sortedRanges); - - $minRangeBoundary = $sortedRanges[0][0]; - $maxRangeBoundary = max(array_column($sortedRanges, 1)); - $rangeBoundary = $maxRangeBoundary - $minRangeBoundary; - - // check 1: limit the amount of ranges per request depending on range boundary - // accept 1024 requests for a boundary < 1MB, and 512 for boundaries larger than that - // rfc standard doesn't provide specification for the amount of ranges to be allowed per request - if ( - ($rangeBoundary < (1 << 20) && count($ranges) > 1024) - || ($rangeBoundary > (1 << 20) && count($ranges) > 512) - ) { - throw new Exception\PreconditionFailed('Too many ranges in this request'); - } - - // check2: check for overlapping ranges and allow a maximum of 2 - // compare each start value with either the previous end value - // or current max end range value, whichever is bigger - $overlapCounter = 0; - $maxEndValue = -1; - - foreach ($sortedRanges as $range) { - // compare start value with the current maxEndValue - if ($range[0] <= $maxEndValue) { - $overlapCounter++; - - if ($overlapCounter > 2) { - throw new Exception\PreconditionFailed('Too many overlaps'); - } - } - - // check if this ranges end value is larger than maxEndValue and update - if ($range[1] > $maxEndValue) { - $maxEndValue = $range[1]; - } - } - - // check3: ranges should be requested in ascending order - // it can be useful to do occasional descending requests, i.e. if a necessary - // piece of information is located at the end of the file. however, if a range - // consists of many unordered ranges, it can be indicative of a deliberate attack - // limit descending ranges to 32 for boundaries < 1MB and 16 for boundaries larger than that - $descendingStartCounter = 0; - - foreach ($ranges as $idx => $range) { - // skip the first range - if ($idx === 0) continue; - - if ($range[0] < $ranges[$idx - 1][0]) { - $descendingStartCounter++; - } - } - - if ( - ($rangeBoundary < (1 << 20) && $descendingStartCounter > 32) - || ($rangeBoundary > (1 << 20) && $descendingStartCounter > 16) - ) { - throw new Exception\PreconditionFailed('Too many ranges in this request'); - } - - return $ranges; - } + * Returns the HTTP range header. + * + * This method returns null if there is no well-formed HTTP range request + * header. + * + * If the string is valid, the ranges will be evaluated and processed in + * the next function + * + * @return string|null + */ + public function getHTTPRange() + { + $rangeHeader = $this->httpRequest->getHeader('range'); + + if (is_null($rangeHeader)) { + return null; + } + + // remove all spaces before evaluation + $rangeHeader = str_replace(' ', '', $rangeHeader); + + // regex to allow multiple ranges + // e.g.'Range: bytes=0-50,100-140, 200-500' + if (!preg_match('/^bytes=(([0-9]*-{1}[0-9]*,? ?)*)$/i', $rangeHeader, $matches)) { + return null; + } + + // remove trailing comma + $matches[1] = trim($matches[1], ','); + + // remove the range string + return $matches[1]; + } + + /** + * Returns the processed ranges of the previously parsed range header. + * + * Each range consists of 2 numbers + * The first number specifies the offset of the first byte in the range + * The second number specifies the offset of the last byte in the range + * + * If the second offset is null, it should be treated as the offset of the last byte of the entity + * If the first offset is null, the second offset should be used to retrieve the last x bytes of the entity + * + * If multiple ranges are specified, they will be preprocessed to avoid DOS attacks + * Amount of ranges will be capped dependent on boundary size + * Overlapping ranges will be capped to 2 per request + * Ranges in descending order will be capped dependent on boundary size + * + * @throws Exception\PreconditionFailed if ranges overlap or too many ranges are being requested + * @throws Exception\RequestedRangeNotSatisfiable if an invalid range is requested + * + * @return array|null + */ + public function preprocessRanges($rangeString, $nodeSize) + { + // create an array of all ranges + // result example: [[0, 100], [200, 300]] + $ranges = explode(',', $rangeString); + + // if an invalid range is encountered, its index will be saved in here + // invalid ranges will be removed after the for loop + // examples: start > end, '---200-300---', '-' + $rangesToRemove = []; + + // loop over the array of ranges and do some basic checks and transforms + for ($i = 0; $i < count($ranges); ++$i) { + $ranges[$i] = explode('-', $ranges[$i]); + + // mark invalid ranges for removal + // "---200-300---" would be considered a valid range by the regexp + if (2 !== count($ranges[$i])) { + $rangesToRemove[] = $i; + continue; + } + + // copy the ranges into temporary variables for comparisons + $start = $ranges[$i][0]; + $end = $ranges[$i][1]; + + // if neither start nor end value is defined, mark range for removal + if ('' === $start && '' === $end) { + $rangesToRemove[] = $i; + continue; + } + + // if start > filesize, mark for removal + if ($start > $nodeSize) { + $rangesToRemove[] = $i; + continue; + } + + // transform range parameters for special ranges immediately + // this will make security checks later in this function much simpler + if ('' === $start) { + $ranges[$i][0] = max($nodeSize - intval($end), 0); + $ranges[$i][1] = $nodeSize - 1; + } elseif ('' === $end || $end > ($nodeSize - 1)) { + $ranges[$i][0] = intval($start); + $ranges[$i][1] = $nodeSize - 1; + } else { + $ranges[$i][0] = intval($start); + $ranges[$i][1] = intval($end); + } + + // if start > end, mark it for removal + if ($ranges[$i][0] > $ranges[$i][1]) { + $rangesToRemove[] = $i; + } + } + + // remove invalid ranges + foreach ($rangesToRemove as $idx) { + unset($ranges[$idx]); + } + + // if no ranges are left at this point, return + if (0 === count($ranges)) { + throw new Exception\RequestedRangeNotSatisfiable('None of the provided ranges satisfy the requirements. Check out RFC 7233 to learn how to form range requests.'); + } + + // rearrange the array keys + $ranges = array_values($ranges); + + /** + * implement checks to prevent DOS attacks, according to rfc7233, 6.1. + */ + + // set up a sorted copy of the ranges array and define the boundary + $sortedRanges = $ranges; + sort($sortedRanges); + + $minRangeBoundary = $sortedRanges[0][0]; + $maxRangeBoundary = max(array_column($sortedRanges, 1)); + $rangeBoundary = $maxRangeBoundary - $minRangeBoundary; + + // check 1: limit the amount of ranges per request depending on range boundary + // accept 1024 requests for a boundary < 1MB, and 512 for boundaries larger than that + // rfc standard doesn't provide specification for the amount of ranges to be allowed per request + if ( + ($rangeBoundary < (1 << 20) && count($ranges) > 1024) + || ($rangeBoundary > (1 << 20) && count($ranges) > 512) + ) { + throw new Exception\RequestedRangeNotSatisfiable('Too many ranges in this request'); + } + + // check2: check for overlapping ranges and allow a maximum of 2 + // compare each start value with either the previous end value + // or current max end range value, whichever is bigger + $overlapCounter = 0; + $maxEndValue = -1; + + foreach ($sortedRanges as $range) { + // compare start value with the current maxEndValue + if ($range[0] <= $maxEndValue) { + ++$overlapCounter; + + if ($overlapCounter > 2) { + throw new Exception\RequestedRangeNotSatisfiable('Too many overlaps. Consider coalescing your overlapping ranges'); + } + } + + // check if this ranges end value is larger than maxEndValue and update + if ($range[1] > $maxEndValue) { + $maxEndValue = $range[1]; + } + } + + // check3: ranges should be requested in ascending order + // it can be useful to do occasional descending requests, i.e. if a necessary + // piece of information is located at the end of the file. however, if a range + // consists of many unordered ranges, it can be indicative of a deliberate attack + // limit descending ranges to 32 for boundaries < 1MB and 16 for boundaries larger than that + $descendingStartCounter = 0; + + foreach ($ranges as $idx => $range) { + // skip the first range + if (0 === $idx) { + continue; + } + + if ($range[0] < $ranges[$idx - 1][0]) { + ++$descendingStartCounter; + } + } + + if ( + ($rangeBoundary < (1 << 20) && $descendingStartCounter > 32) + || ($rangeBoundary > (1 << 20) && $descendingStartCounter > 16) + ) { + throw new Exception\RequestedRangeNotSatisfiable('Too many unordered ranges in this request. Avoid listing ranges in descending order.'); + } + + return $ranges; + } /** * Returns the HTTP Prefer header information. diff --git a/tests/Sabre/DAV/ServerRangeTest.php b/tests/Sabre/DAV/ServerRangeTest.php index 6d5be46082..fedfcdc29f 100644 --- a/tests/Sabre/DAV/ServerRangeTest.php +++ b/tests/Sabre/DAV/ServerRangeTest.php @@ -249,4 +249,106 @@ public function testIfRangeModificationDateModified() $this->assertEquals(200, $response->getStatus()); $this->assertEquals('Test contents', $response->getBodyAsString()); } + + // the following section contains new multipart range request tests + // this string will be on top of every range. we will need this one a lot + public function getBoundaryResponseString($boundary, $range) + { + return '--'.$boundary."\nContent-Type: application/octet-stream\nContent-Range: bytes ".$range[0].'-'.$range[1].'/'.$range[2]."\n\n"; + } + + public function testMultipartRange() + { + $request = new HTTP\Request('GET', '/files/test.txt', ['Range' => 'bytes=2-5, 8-11']); + $response = $this->request($request); + + $boundary = explode('boundary=', $response->getHeader('Content-Type'))[1]; + + $this->assertEquals([ + 'X-Sabre-Version' => [Version::VERSION], + 'Content-Type' => ['multipart/byteranges; boundary='.$boundary], + 'Content-Length' => [219], + 'ETag' => ['"'.md5('Test contents').'"'], + 'Last-Modified' => [$this->lastModified], + ], + $response->getHeaders() + ); + $this->assertEquals(206, $response->getStatus()); + $this->assertEquals($this->getBoundaryResponseString($boundary, [2, 5, 13]).'st c'."\n\n".$this->getBoundaryResponseString($boundary, [8, 11, 13]).'tent'."\n\n", $response->getBodyAsString()); + } + + /** + * @depends testMultipartRange + */ + public function testPartiallyValidRangeWithSingleValidRange() + { + $request = new HTTP\Request('GET', '/files/test.txt', ['Range' => 'bytes=2-5, 11-8, 100-200, -5-5']); + $response = $this->request($request); + + $this->assertEquals([ + 'X-Sabre-Version' => [Version::VERSION], + 'Content-Type' => ['application/octet-stream'], + 'Content-Length' => [4], + 'Content-Range' => ['bytes 2-5/13'], + 'ETag' => ['"'.md5('Test contents').'"'], + 'Last-Modified' => [$this->lastModified], + ], + $response->getHeaders() + ); + $this->assertEquals(206, $response->getStatus()); + $this->assertEquals('st c', $response->getBodyAsString()); + } + + /** + * @depends testMultipartRange + */ + public function testPartiallyValidRangeWithMultipleValidRanges() + { + $request = new HTTP\Request('GET', '/files/test.txt', ['Range' => 'bytes=2-5, 8-11, 11-8, 100-200, -5-5']); + $response = $this->request($request); + + $boundary = explode('boundary=', $response->getHeader('Content-Type'))[1]; + + $this->assertEquals([ + 'X-Sabre-Version' => [Version::VERSION], + 'Content-Type' => ['multipart/byteranges; boundary='.$boundary], + 'Content-Length' => [219], + 'ETag' => ['"'.md5('Test contents').'"'], + 'Last-Modified' => [$this->lastModified], + ], + $response->getHeaders() + ); + $this->assertEquals(206, $response->getStatus()); + $this->assertEquals($this->getBoundaryResponseString($boundary, [2, 5, 13]).'st c'."\n\n".$this->getBoundaryResponseString($boundary, [8, 11, 13]).'tent'."\n\n", $response->getBodyAsString()); + } + + /** + * @depends testMultipartRange + */ + public function testCompletelyInvalidRange() + { + $request = new HTTP\Request('GET', '/files/test.txt', ['Range' => 'bytes=11-8, 100-200, -5-5']); + $response = $this->request($request); + + $this->assertEquals(416, $response->getStatus()); + } + + /** + * @depends testMultipartRange + */ + public function testOverlappingRange() + { + $request = new HTTP\Request('GET', '/files/test.txt', ['Range' => 'bytes=2-12, 3-5, 6-9, 9-11']); + $response = $this->request($request); + + $this->assertEquals(416, $response->getStatus()); + } + + public function testNonSeekableMultipartRange() + { + $request = new HTTP\Request('GET', '/files/no-seeking.txt', ['Range' => 'bytes=2-5, 7-11']); + $response = $this->request($request); + + $this->assertEquals(416, $response->getStatus()); + } } diff --git a/tests/phpunit.xml b/tests/phpunit.xml index b836b59964..eec9f3278b 100644 --- a/tests/phpunit.xml +++ b/tests/phpunit.xml @@ -38,4 +38,4 @@ - + \ No newline at end of file From 35097d2119294a144ac5bb902283d508113d5f97 Mon Sep 17 00:00:00 2001 From: Daniel Riess Date: Mon, 10 Oct 2022 20:57:23 +0700 Subject: [PATCH 3/8] tried fixing codecov complaints --- lib/DAV/CorePlugin.php | 4 +-- lib/DAV/Server.php | 12 +++------ tests/Sabre/DAV/ServerRangeTest.php | 42 ++++++++++++++++++++++++++++- 3 files changed, 45 insertions(+), 13 deletions(-) diff --git a/lib/DAV/CorePlugin.php b/lib/DAV/CorePlugin.php index 4352f55202..c5ad56db92 100644 --- a/lib/DAV/CorePlugin.php +++ b/lib/DAV/CorePlugin.php @@ -236,9 +236,7 @@ public function httpGet(RequestInterface $request, ResponseInterface $response) } // rewind the range body after the loop - if (!rewind($rangeBody)) { - throw new Exception\PreconditionFailed('Could not rewind the response stream'); - } + rewind($rangeBody); if (1 === count($ranges)) { // if only one range, content range header MUST be set (according to spec) diff --git a/lib/DAV/Server.php b/lib/DAV/Server.php index 25a654ceb2..4f16df8cfe 100644 --- a/lib/DAV/Server.php +++ b/lib/DAV/Server.php @@ -746,12 +746,9 @@ public function preprocessRanges($rangeString, $nodeSize) $rangeBoundary = $maxRangeBoundary - $minRangeBoundary; // check 1: limit the amount of ranges per request depending on range boundary - // accept 1024 requests for a boundary < 1MB, and 512 for boundaries larger than that + // accept 512 ranges per request // rfc standard doesn't provide specification for the amount of ranges to be allowed per request - if ( - ($rangeBoundary < (1 << 20) && count($ranges) > 1024) - || ($rangeBoundary > (1 << 20) && count($ranges) > 512) - ) { + if (count($ranges) > 512) { throw new Exception\RequestedRangeNotSatisfiable('Too many ranges in this request'); } @@ -795,10 +792,7 @@ public function preprocessRanges($rangeString, $nodeSize) } } - if ( - ($rangeBoundary < (1 << 20) && $descendingStartCounter > 32) - || ($rangeBoundary > (1 << 20) && $descendingStartCounter > 16) - ) { + if ($descendingStartCounter > 16) { throw new Exception\RequestedRangeNotSatisfiable('Too many unordered ranges in this request. Avoid listing ranges in descending order.'); } diff --git a/tests/Sabre/DAV/ServerRangeTest.php b/tests/Sabre/DAV/ServerRangeTest.php index fedfcdc29f..1ab4b3075a 100644 --- a/tests/Sabre/DAV/ServerRangeTest.php +++ b/tests/Sabre/DAV/ServerRangeTest.php @@ -27,6 +27,7 @@ public function setup(): void { parent::setUp(); $this->server->createFile('files/test.txt', 'Test contents'); + $this->server->createFile('files/rangetest.txt', 'Lorem ipsum dolor sit amet, consectetur adipiscing elit. Cras massa quam, tempus a bibendum eget, accumsan et risus. Duis volutpat diam consectetur lorem scelerisque, vitae tristique massa tristique. Donec porta elementum condimentum. Duis fringilla, est sed tempus placerat, tortor tortor pulvinar lacus, in semper magna felis id nisi. Pellentesque eleifend augue elit, non hendrerit ex euismod at. Morbi a auctor mi. Suspendisse vel imperdiet lacus. Aenean auctor nulla urna, in sagittis felis venenatis fringilla. Nulla facilisi. Suspendisse nunc.'); $this->lastModified = HTTP\toDate( new DateTime('@'.$this->server->tree->getNodeForPath('files/test.txt')->getLastModified()) @@ -148,6 +149,17 @@ public function testNonSeekableStream() $this->assertEquals('st c', $response->getBodyAsString()); } + /** + * @depends testNonSeekableStream + */ + public function testNonSeekableExceedingRange() + { + $request = new HTTP\Request('GET', '/files/no-seeking.txt', ['Range' => 'bytes=100-200']); + $response = $this->request($request); + + $this->assertEquals(416, $response->getStatus()); + } + /** * @depends testRange */ @@ -327,7 +339,7 @@ public function testPartiallyValidRangeWithMultipleValidRanges() */ public function testCompletelyInvalidRange() { - $request = new HTTP\Request('GET', '/files/test.txt', ['Range' => 'bytes=11-8, 100-200, -5-5']); + $request = new HTTP\Request('GET', '/files/test.txt', ['Range' => 'bytes=11-8, 100-200, -5-5, -']); $response = $this->request($request); $this->assertEquals(416, $response->getStatus()); @@ -344,6 +356,34 @@ public function testOverlappingRange() $this->assertEquals(416, $response->getStatus()); } + public function testTooManyRanges() + { + $ranges = []; + for ($i = 0; $i < 513; ++$i) { + $ranges[] = $i.'-'.$i; + } + $byterange = 'bytes='.implode(',', $ranges); + + $request = new HTTP\Request('GET', '/files/rangetest.txt', ['Range' => $byterange]); + $response = $this->request($request); + + $this->assertEquals(416, $response->getStatus()); + } + + public function testUnorderedRanges() + { + $ranges = []; + for ($i = 0; $i < 18; ++$i) { + $ranges[] = (20 - $i).'-'.(20 - $i); + } + $byterange = 'bytes='.implode(',', $ranges); + + $request = new HTTP\Request('GET', '/files/rangetest.txt', ['Range' => $byterange]); + $response = $this->request($request); + + $this->assertEquals(416, $response->getStatus()); + } + public function testNonSeekableMultipartRange() { $request = new HTTP\Request('GET', '/files/no-seeking.txt', ['Range' => 'bytes=2-5, 7-11']); From 5e26a53f2581cac108ccdad1bda48944e8776634 Mon Sep 17 00:00:00 2001 From: Daniel Riess Date: Tue, 27 Sep 2022 17:19:23 +0700 Subject: [PATCH 4/8] add support for multipart range requests according to rfc 7233 --- lib/DAV/CorePlugin.php | 312 ++++++++++++++++++++++++----------------- lib/DAV/Server.php | 222 ++++++++++++++++++++++++----- 2 files changed, 370 insertions(+), 164 deletions(-) diff --git a/lib/DAV/CorePlugin.php b/lib/DAV/CorePlugin.php index dbd8976b17..c5bcde242b 100644 --- a/lib/DAV/CorePlugin.php +++ b/lib/DAV/CorePlugin.php @@ -66,141 +66,195 @@ public function getPluginName() } /** - * This is the default implementation for the GET method. - * - * @return bool - */ - public function httpGet(RequestInterface $request, ResponseInterface $response) - { - $path = $request->getPath(); - $node = $this->server->tree->getNodeForPath($path); - - if (!$node instanceof IFile) { - return; - } - - if ('HEAD' === $request->getHeader('X-Sabre-Original-Method')) { - $body = ''; - } else { - $body = $node->get(); - - // Converting string into stream, if needed. - if (is_string($body)) { - $stream = fopen('php://temp', 'r+'); - fwrite($stream, $body); - rewind($stream); - $body = $stream; - } - } - - /* + * This is the default implementation for the GET method. + * + * @return bool + */ + public function httpGet(RequestInterface $request, ResponseInterface $response) + { + $path = $request->getPath(); + $node = $this->server->tree->getNodeForPath($path); + + if (!$node instanceof IFile) { + return; + } + + if ('HEAD' === $request->getHeader('X-Sabre-Original-Method')) { + $body = ''; + } else { + $body = $node->get(); + + // Converting string into stream, if needed. + if (is_string($body)) { + $stream = fopen('php://temp', 'r+'); + fwrite($stream, $body); + rewind($stream); + $body = $stream; + } + } + + /* * TODO: getetag, getlastmodified, getsize should also be used using * this method */ - $httpHeaders = $this->server->getHTTPHeaders($path); + $httpHeaders = $this->server->getHTTPHeaders($path); - /* ContentType needs to get a default, because many webservers will otherwise + /* ContentType needs to get a default, because many webservers will otherwise * default to text/html, and we don't want this for security reasons. */ - if (!isset($httpHeaders['Content-Type'])) { - $httpHeaders['Content-Type'] = 'application/octet-stream'; - } - - if (isset($httpHeaders['Content-Length'])) { - $nodeSize = $httpHeaders['Content-Length']; - - // Need to unset Content-Length, because we'll handle that during figuring out the range - unset($httpHeaders['Content-Length']); - } else { - $nodeSize = null; - } - - $response->addHeaders($httpHeaders); - - $range = $this->server->getHTTPRange(); - $ifRange = $request->getHeader('If-Range'); - $ignoreRangeHeader = false; - - // If ifRange is set, and range is specified, we first need to check - // the precondition. - if ($nodeSize && $range && $ifRange) { - // if IfRange is parsable as a date we'll treat it as a DateTime - // otherwise, we must treat it as an etag. - try { - $ifRangeDate = new \DateTime($ifRange); - - // It's a date. We must check if the entity is modified since - // the specified date. - if (!isset($httpHeaders['Last-Modified'])) { - $ignoreRangeHeader = true; - } else { - $modified = new \DateTime($httpHeaders['Last-Modified']); - if ($modified > $ifRangeDate) { - $ignoreRangeHeader = true; - } - } - } catch (\Exception $e) { - // It's an entity. We can do a simple comparison. - if (!isset($httpHeaders['ETag'])) { - $ignoreRangeHeader = true; - } elseif ($httpHeaders['ETag'] !== $ifRange) { - $ignoreRangeHeader = true; - } - } - } - - // We're only going to support HTTP ranges if the backend provided a filesize - if (!$ignoreRangeHeader && $nodeSize && $range) { - // Determining the exact byte offsets - if (!is_null($range[0])) { - $start = $range[0]; - $end = $range[1] ? $range[1] : $nodeSize - 1; - if ($start >= $nodeSize) { - throw new Exception\RequestedRangeNotSatisfiable('The start offset ('.$range[0].') exceeded the size of the entity ('.$nodeSize.')'); - } - if ($end < $start) { - throw new Exception\RequestedRangeNotSatisfiable('The end offset ('.$range[1].') is lower than the start offset ('.$range[0].')'); - } - if ($end >= $nodeSize) { - $end = $nodeSize - 1; - } - } else { - $start = $nodeSize - $range[1]; - $end = $nodeSize - 1; - - if ($start < 0) { - $start = 0; - } - } - - // Streams may advertise themselves as seekable, but still not - // actually allow fseek. We'll manually go forward in the stream - // if fseek failed. - if (!stream_get_meta_data($body)['seekable'] || -1 === fseek($body, $start, SEEK_SET)) { - $consumeBlock = 8192; - for ($consumed = 0; $start - $consumed > 0;) { - if (feof($body)) { - throw new Exception\RequestedRangeNotSatisfiable('The start offset ('.$start.') exceeded the size of the entity ('.$consumed.')'); - } - $consumed += strlen(fread($body, min($start - $consumed, $consumeBlock))); - } - } - - $response->setHeader('Content-Length', $end - $start + 1); - $response->setHeader('Content-Range', 'bytes '.$start.'-'.$end.'/'.$nodeSize); - $response->setStatus(206); - $response->setBody($body); - } else { - if ($nodeSize) { - $response->setHeader('Content-Length', $nodeSize); - } - $response->setStatus(200); - $response->setBody($body); - } - // Sending back false will interrupt the event chain and tell the server - // we've handled this method. - return false; - } + if (!isset($httpHeaders['Content-Type'])) { + $httpHeaders['Content-Type'] = 'application/octet-stream'; + } + + if (isset($httpHeaders['Content-Length'])) { + $nodeSize = $httpHeaders['Content-Length']; + + // Need to unset Content-Length, because we'll handle that during figuring out the range + unset($httpHeaders['Content-Length']); + } else { + $nodeSize = null; + } + + $response->addHeaders($httpHeaders); + + // this function now only checks, if the range string correctly formatted + // and returns it (or null if it isn't or it doesn't exist) + // processing of the ranges will then be done further below + $rangeString = $this->server->getHTTPRange(); + $ifRange = $request->getHeader('If-Range'); + $ignoreRangeHeader = false; + + // If ifRange is set, and range is specified, we first need to check + // the precondition. + if ($nodeSize && $rangeString && $ifRange) { + // if IfRange is parsable as a date we'll treat it as a DateTime + // otherwise, we must treat it as an etag. + try { + $ifRangeDate = new \DateTime($ifRange); + + // It's a date. We must check if the entity is modified since + // the specified date. + if (!isset($httpHeaders['Last-Modified'])) { + $ignoreRangeHeader = true; + } else { + $modified = new \DateTime($httpHeaders['Last-Modified']); + if ($modified > $ifRangeDate) { + $ignoreRangeHeader = true; + } + } + } catch (\Exception $e) { + // It's an entity. We can do a simple comparison. + if (!isset($httpHeaders['ETag'])) { + $ignoreRangeHeader = true; + } elseif ($httpHeaders['ETag'] !== $ifRange) { + $ignoreRangeHeader = true; + } + } + } + + if (!$ignoreRangeHeader && $nodeSize && $rangeString) { + // preprocess the ranges now + // this involves removing invalid ranges and security checks + $ranges = $this->server->preprocessRanges($rangeString, $nodeSize); + } + + if (!$ignoreRangeHeader && $nodeSize && $ranges) { + // create a new stream for the partial responses + $rangeBody = fopen('php://temp', 'r+'); + + // create a boundary string that is being used as identifier for multipart ranges + $boundary = hash('md5', uniqid('', true)); + + // create a content length counter + $contentLength = 0; + + foreach ($ranges as $range) { + $start = $range[0]; + $end = $range[1]; + + // Streams may advertise themselves as seekable, but still not + // actually allow fseek. We'll manually go forward in the stream + // if fseek failed. + if (!stream_get_meta_data($body)['seekable'] || -1 === fseek($body, $start, SEEK_SET)) { + // because of potential multiple ranges with descending order, rewind stream after finishing each range + if (!rewind($body)) { + throw new Exception\PreconditionFailed('Could not rewind the response stream'); + } + $consumeBlock = 8192; + for ($consumed = 0; $start - $consumed > 0;) { + if (feof($body)) { + throw new Exception\RequestedRangeNotSatisfiable('The start offset (' . $start . ') exceeded the size of the entity (' . $consumed . ')'); + } + $consumed += strlen(fread($body, min($start - $consumed, $consumeBlock))); + } + } + + // if there is only one range, write it into the reponse body + // otherwise, write multipart header info first before writing the data + if (count($ranges) === 1) { + $rangeLength = $end - $start + 1; + $consumeBlock = 8192; + + // write body data into response body (max 8192 bytes per read cycle) + while ($rangeLength > 0) { + fwrite($rangeBody, fread($body, min($rangeLength, $consumeBlock))); + $rangeLength -= min($rangeLength, $consumeBlock); + } + + // update content length + $contentLength = $end - $start + 1; + } else { + // define new boundary section and write to response body + $boundarySection = "--" . $boundary . "\nContent-Type: " . $node->getContentType() . "\nContent-Range: bytes " . $start . "-" . $end . '/' . $nodeSize . "\n\n"; + fwrite($rangeBody, $boundarySection); + + // write body data into response body (max 8192 bytes per read cycle) + $rangeLength = $end - $start + 1; + $consumeBlock = 8192; + while ($rangeLength > 0) { + fwrite($rangeBody, fread($body, min($rangeLength, $consumeBlock))); + $rangeLength -= min($rangeLength, $consumeBlock); + } + + // write line breaks at the end of section + fwrite($rangeBody, "\n\n"); + + // update content length + // +2 for double linebreak at the end of each section + $contentLength += strlen($boundarySection) + ($end - $start + 1) + 2; + } + } + + // rewind the range body after the loop + if (!rewind($rangeBody)) { + throw new Exception\PreconditionFailed('Could not rewind the response stream'); + } + + if (count($ranges) === 1) { + // if only one range, content range header MUST be set (according to spec) + $response->setHeader('Content-Range', 'bytes ' . $start . '-' . $end . '/' . $nodeSize); + } else { + // if multiple ranges, content range header MUST NOT be set (according to spec) + // content type header MUST be of type multipart/byteranges + // more specific types can be set in each body section header + $response->setHeader('Content-Type', 'multipart/byteranges; boundary=' . $boundary); + } + + $response->setHeader('Content-Length', $contentLength); + $response->setHeader('Vary', 'Range'); + + $response->setStatus(206); + $response->setBody($rangeBody); + } else { + if ($nodeSize) { + $response->setHeader('Content-Length', $nodeSize); + } + $response->setStatus(200); + $response->setBody($body); + } + + return false; + } /** * HTTP OPTIONS. diff --git a/lib/DAV/Server.php b/lib/DAV/Server.php index 1f8300d4a5..651649c027 100644 --- a/lib/DAV/Server.php +++ b/lib/DAV/Server.php @@ -609,41 +609,193 @@ public function getHTTPDepth($default = self::DEPTH_INFINITY) } /** - * Returns the HTTP range header. - * - * This method returns null if there is no well-formed HTTP range request - * header or array($start, $end). - * - * The first number is the offset of the first byte in the range. - * The second number is the offset of the last byte in the range. - * - * If the second offset is null, it should be treated as the offset of the last byte of the entity - * If the first offset is null, the second offset should be used to retrieve the last x bytes of the entity - * - * @return int[]|null - */ - public function getHTTPRange() - { - $range = $this->httpRequest->getHeader('range'); - if (is_null($range)) { - return null; - } - - // Matching "Range: bytes=1234-5678: both numbers are optional - - if (!preg_match('/^bytes=([0-9]*)-([0-9]*)$/i', $range, $matches)) { - return null; - } - - if ('' === $matches[1] && '' === $matches[2]) { - return null; - } - - return [ - '' !== $matches[1] ? (int) $matches[1] : null, - '' !== $matches[2] ? (int) $matches[2] : null, - ]; - } + * Returns the HTTP range header. + * + * This method returns null if there is no well-formed HTTP range request + * header. + * + * If the string is valid, the ranges will be evaluated and processed in + * the next function + * + * @return string|null + */ + public function getHTTPRange() + { + $rangeHeader = $this->httpRequest->getHeader('range'); + + if (is_null($rangeHeader)) { + return null; + } + + // remove all spaces before evaluation + $rangeHeader = str_replace(' ', '', $rangeHeader); + + // regex to allow multiple ranges + // e.g.'Range: bytes=0-50,100-140, 200-500' + if (!preg_match('/^bytes=(([0-9]*-{1}[0-9]*,? ?)*)$/i', $rangeHeader, $matches)) { + return null; + } + + // remove trailing comma + $matches[1] = trim($matches[1], ','); + + // remove the range string + return $matches[1]; + } + + /** + * Returns the processed ranges of the previously parsed range header + * + * Each range consists of 2 numbers + * The first number specifies the offset of the first byte in the range + * The second number specifies the offset of the last byte in the range + * + * If the second offset is null, it should be treated as the offset of the last byte of the entity + * If the first offset is null, the second offset should be used to retrieve the last x bytes of the entity + * + * If multiple ranges are specified, they will be preprocessed to avoid DOS attacks + * Amount of ranges will be capped dependent on boundary size + * Overlapping ranges will be capped to 2 per request + * Ranges in descending order will be capped dependent on boundary size + * + * @throws Exception\PreconditionFailed if ranges overlap or too many ranges are being requested + * @throws Exception\RequestedRangeNotSatisfiable if an invalid range is requested + * + * @return array|null + */ + public function preprocessRanges($rangeString, $nodeSize) + { + // create an array of all ranges + // result example: [[0, 100], [200, 300]] + $ranges = explode(',', $rangeString); + + // if an invalid range is encountered, its index will be saved in here + // invalid ranges will be removed after the for loop + // examples: start > end, '---200-300---', '-' + $rangesToRemove = []; + + // loop over the array of ranges and do some basic checks and transforms + for ($i = 0; $i < count($ranges); $i++) { + $ranges[$i] = explode('-', $ranges[$i]); + + // mark invalid ranges for removal + // "---200-300---" would be considered a valid range by the regexp + if (count($ranges[$i]) !== 2) { + $rangesToRemove[] = $i; + continue; + } + + // copy the ranges into temporary variables for comparisons + $start = $ranges[$i][0]; + $end = $ranges[$i][1]; + + // if neither start nor end value is defined, mark range for removal + if ($start === "" && $end === "") { + $rangesToRemove[] = $i; + continue; + } + + // transform range parameters for special ranges immediately + // this will make security checks later in this function much simpler + if ($start === "") { + $ranges[$i][0] = $nodeSize - intval($end); + $ranges[$i][1] = $nodeSize - 1; + } else if ($end === "" || $end > ($nodeSize - 1)) { + $ranges[$i][0] = intval($start); + $ranges[$i][1] = $nodeSize - 1; + } else { + $ranges[$i][0] = intval($start); + $ranges[$i][1] = intval($end); + } + + // if start > end, mark it for removal + if ($ranges[$i][0] > $ranges[$i][1]) { + $rangesToRemove[] = $i; + } + } + + // remove invalid ranges + foreach ($rangesToRemove as $idx) { + unset($ranges[$idx]); + } + + // if no ranges are left at this point, return + if (count($ranges) === 0) { + return null; + } + + // rearrange the array keys + $ranges = array_values($ranges); + + /** + * implement checks to prevent DOS attacks, according to rfc7233, 6.1 + */ + + // set up a sorted copy of the ranges array and define the boundary + $sortedRanges = $ranges; + sort($sortedRanges); + + $minRangeBoundary = $sortedRanges[0][0]; + $maxRangeBoundary = max(array_column($sortedRanges, 1)); + $rangeBoundary = $maxRangeBoundary - $minRangeBoundary; + + // check 1: limit the amount of ranges per request depending on range boundary + // accept 1024 requests for a boundary < 1MB, and 512 for boundaries larger than that + // rfc standard doesn't provide specification for the amount of ranges to be allowed per request + if ( + ($rangeBoundary < (1 << 20) && count($ranges) > 1024) + || ($rangeBoundary > (1 << 20) && count($ranges) > 512) + ) { + throw new Exception\PreconditionFailed('Too many ranges in this request'); + } + + // check2: check for overlapping ranges and allow a maximum of 2 + // compare each start value with either the previous end value + // or current max end range value, whichever is bigger + $overlapCounter = 0; + $maxEndValue = -1; + + foreach ($sortedRanges as $range) { + // compare start value with the current maxEndValue + if ($range[0] <= $maxEndValue) { + $overlapCounter++; + + if ($overlapCounter > 2) { + throw new Exception\PreconditionFailed('Too many overlaps'); + } + } + + // check if this ranges end value is larger than maxEndValue and update + if ($range[1] > $maxEndValue) { + $maxEndValue = $range[1]; + } + } + + // check3: ranges should be requested in ascending order + // it can be useful to do occasional descending requests, i.e. if a necessary + // piece of information is located at the end of the file. however, if a range + // consists of many unordered ranges, it can be indicative of a deliberate attack + // limit descending ranges to 32 for boundaries < 1MB and 16 for boundaries larger than that + $descendingStartCounter = 0; + + foreach ($ranges as $idx => $range) { + // skip the first range + if ($idx === 0) continue; + + if ($range[0] < $ranges[$idx - 1][0]) { + $descendingStartCounter++; + } + } + + if ( + ($rangeBoundary < (1 << 20) && $descendingStartCounter > 32) + || ($rangeBoundary > (1 << 20) && $descendingStartCounter > 16) + ) { + throw new Exception\PreconditionFailed('Too many ranges in this request'); + } + + return $ranges; + } /** * Returns the HTTP Prefer header information. From 2ebf6f21bacfe5b7e8af81070c234908c1bdce93 Mon Sep 17 00:00:00 2001 From: Daniel Riess Date: Fri, 30 Sep 2022 15:17:22 +0700 Subject: [PATCH 5/8] cs-fix, added some unit tests --- lib/DAV/CorePlugin.php | 375 ++++++++++++++------------- lib/DAV/Server.php | 382 ++++++++++++++-------------- tests/Sabre/DAV/ServerRangeTest.php | 102 ++++++++ tests/phpunit.xml | 2 +- 4 files changed, 490 insertions(+), 371 deletions(-) diff --git a/lib/DAV/CorePlugin.php b/lib/DAV/CorePlugin.php index c5bcde242b..4352f55202 100644 --- a/lib/DAV/CorePlugin.php +++ b/lib/DAV/CorePlugin.php @@ -66,195 +66,204 @@ public function getPluginName() } /** - * This is the default implementation for the GET method. - * - * @return bool - */ - public function httpGet(RequestInterface $request, ResponseInterface $response) - { - $path = $request->getPath(); - $node = $this->server->tree->getNodeForPath($path); - - if (!$node instanceof IFile) { - return; - } - - if ('HEAD' === $request->getHeader('X-Sabre-Original-Method')) { - $body = ''; - } else { - $body = $node->get(); - - // Converting string into stream, if needed. - if (is_string($body)) { - $stream = fopen('php://temp', 'r+'); - fwrite($stream, $body); - rewind($stream); - $body = $stream; - } - } - - /* + * This is the default implementation for the GET method. + * + * @return bool + */ + public function httpGet(RequestInterface $request, ResponseInterface $response) + { + $path = $request->getPath(); + $node = $this->server->tree->getNodeForPath($path); + + if (!$node instanceof IFile) { + return; + } + + if ('HEAD' === $request->getHeader('X-Sabre-Original-Method')) { + $body = ''; + } else { + $body = $node->get(); + + // Converting string into stream, if needed. + if (is_string($body)) { + $stream = fopen('php://temp', 'r+'); + fwrite($stream, $body); + rewind($stream); + $body = $stream; + } + } + + /* * TODO: getetag, getlastmodified, getsize should also be used using * this method */ - $httpHeaders = $this->server->getHTTPHeaders($path); + $httpHeaders = $this->server->getHTTPHeaders($path); - /* ContentType needs to get a default, because many webservers will otherwise + /* ContentType needs to get a default, because many webservers will otherwise * default to text/html, and we don't want this for security reasons. */ - if (!isset($httpHeaders['Content-Type'])) { - $httpHeaders['Content-Type'] = 'application/octet-stream'; - } - - if (isset($httpHeaders['Content-Length'])) { - $nodeSize = $httpHeaders['Content-Length']; - - // Need to unset Content-Length, because we'll handle that during figuring out the range - unset($httpHeaders['Content-Length']); - } else { - $nodeSize = null; - } - - $response->addHeaders($httpHeaders); - - // this function now only checks, if the range string correctly formatted - // and returns it (or null if it isn't or it doesn't exist) - // processing of the ranges will then be done further below - $rangeString = $this->server->getHTTPRange(); - $ifRange = $request->getHeader('If-Range'); - $ignoreRangeHeader = false; - - // If ifRange is set, and range is specified, we first need to check - // the precondition. - if ($nodeSize && $rangeString && $ifRange) { - // if IfRange is parsable as a date we'll treat it as a DateTime - // otherwise, we must treat it as an etag. - try { - $ifRangeDate = new \DateTime($ifRange); - - // It's a date. We must check if the entity is modified since - // the specified date. - if (!isset($httpHeaders['Last-Modified'])) { - $ignoreRangeHeader = true; - } else { - $modified = new \DateTime($httpHeaders['Last-Modified']); - if ($modified > $ifRangeDate) { - $ignoreRangeHeader = true; - } - } - } catch (\Exception $e) { - // It's an entity. We can do a simple comparison. - if (!isset($httpHeaders['ETag'])) { - $ignoreRangeHeader = true; - } elseif ($httpHeaders['ETag'] !== $ifRange) { - $ignoreRangeHeader = true; - } - } - } - - if (!$ignoreRangeHeader && $nodeSize && $rangeString) { - // preprocess the ranges now - // this involves removing invalid ranges and security checks - $ranges = $this->server->preprocessRanges($rangeString, $nodeSize); - } - - if (!$ignoreRangeHeader && $nodeSize && $ranges) { - // create a new stream for the partial responses - $rangeBody = fopen('php://temp', 'r+'); - - // create a boundary string that is being used as identifier for multipart ranges - $boundary = hash('md5', uniqid('', true)); - - // create a content length counter - $contentLength = 0; - - foreach ($ranges as $range) { - $start = $range[0]; - $end = $range[1]; - - // Streams may advertise themselves as seekable, but still not - // actually allow fseek. We'll manually go forward in the stream - // if fseek failed. - if (!stream_get_meta_data($body)['seekable'] || -1 === fseek($body, $start, SEEK_SET)) { - // because of potential multiple ranges with descending order, rewind stream after finishing each range - if (!rewind($body)) { - throw new Exception\PreconditionFailed('Could not rewind the response stream'); - } - $consumeBlock = 8192; - for ($consumed = 0; $start - $consumed > 0;) { - if (feof($body)) { - throw new Exception\RequestedRangeNotSatisfiable('The start offset (' . $start . ') exceeded the size of the entity (' . $consumed . ')'); - } - $consumed += strlen(fread($body, min($start - $consumed, $consumeBlock))); - } - } - - // if there is only one range, write it into the reponse body - // otherwise, write multipart header info first before writing the data - if (count($ranges) === 1) { - $rangeLength = $end - $start + 1; - $consumeBlock = 8192; - - // write body data into response body (max 8192 bytes per read cycle) - while ($rangeLength > 0) { - fwrite($rangeBody, fread($body, min($rangeLength, $consumeBlock))); - $rangeLength -= min($rangeLength, $consumeBlock); - } - - // update content length - $contentLength = $end - $start + 1; - } else { - // define new boundary section and write to response body - $boundarySection = "--" . $boundary . "\nContent-Type: " . $node->getContentType() . "\nContent-Range: bytes " . $start . "-" . $end . '/' . $nodeSize . "\n\n"; - fwrite($rangeBody, $boundarySection); - - // write body data into response body (max 8192 bytes per read cycle) - $rangeLength = $end - $start + 1; - $consumeBlock = 8192; - while ($rangeLength > 0) { - fwrite($rangeBody, fread($body, min($rangeLength, $consumeBlock))); - $rangeLength -= min($rangeLength, $consumeBlock); - } - - // write line breaks at the end of section - fwrite($rangeBody, "\n\n"); - - // update content length - // +2 for double linebreak at the end of each section - $contentLength += strlen($boundarySection) + ($end - $start + 1) + 2; - } - } - - // rewind the range body after the loop - if (!rewind($rangeBody)) { - throw new Exception\PreconditionFailed('Could not rewind the response stream'); - } - - if (count($ranges) === 1) { - // if only one range, content range header MUST be set (according to spec) - $response->setHeader('Content-Range', 'bytes ' . $start . '-' . $end . '/' . $nodeSize); - } else { - // if multiple ranges, content range header MUST NOT be set (according to spec) - // content type header MUST be of type multipart/byteranges - // more specific types can be set in each body section header - $response->setHeader('Content-Type', 'multipart/byteranges; boundary=' . $boundary); - } - - $response->setHeader('Content-Length', $contentLength); - $response->setHeader('Vary', 'Range'); - - $response->setStatus(206); - $response->setBody($rangeBody); - } else { - if ($nodeSize) { - $response->setHeader('Content-Length', $nodeSize); - } - $response->setStatus(200); - $response->setBody($body); - } - - return false; - } + if (!isset($httpHeaders['Content-Type'])) { + $httpHeaders['Content-Type'] = 'application/octet-stream'; + } + + if (isset($httpHeaders['Content-Length'])) { + $nodeSize = $httpHeaders['Content-Length']; + + // Need to unset Content-Length, because we'll handle that during figuring out the range + unset($httpHeaders['Content-Length']); + } else { + $nodeSize = null; + } + + $response->addHeaders($httpHeaders); + + // this function now only checks, if the range string correctly formatted + // and returns it (or null if it isn't or it doesn't exist) + // processing of the ranges will then be done further below + $rangeString = $this->server->getHTTPRange(); + $ifRange = $request->getHeader('If-Range'); + $ignoreRangeHeader = false; + + // If ifRange is set, and range is specified, we first need to check + // the precondition. + if ($nodeSize && $rangeString && $ifRange) { + // if IfRange is parsable as a date we'll treat it as a DateTime + // otherwise, we must treat it as an etag. + try { + $ifRangeDate = new \DateTime($ifRange); + + // It's a date. We must check if the entity is modified since + // the specified date. + if (!isset($httpHeaders['Last-Modified'])) { + $ignoreRangeHeader = true; + } else { + $modified = new \DateTime($httpHeaders['Last-Modified']); + if ($modified > $ifRangeDate) { + $ignoreRangeHeader = true; + } + } + } catch (\Exception $e) { + // It's an entity. We can do a simple comparison. + if (!isset($httpHeaders['ETag'])) { + $ignoreRangeHeader = true; + } elseif ($httpHeaders['ETag'] !== $ifRange) { + $ignoreRangeHeader = true; + } + } + } + + $ranges = null; + + if (!$ignoreRangeHeader && $nodeSize && $rangeString) { + // preprocess the ranges now + // this involves removing invalid ranges and security checks + $ranges = $this->server->preprocessRanges($rangeString, $nodeSize); + } + + if (!$ignoreRangeHeader && $nodeSize && $ranges) { + // create a new stream for the partial responses + $rangeBody = fopen('php://temp', 'r+'); + + // create a boundary string that is being used as identifier for multipart ranges + $boundary = hash('md5', uniqid('', true)); + + // create a content length counter + $contentLength = 0; + + // define the node's content type + $nodeContentType = $node->getContentType(); + + if (!$nodeContentType) { + $nodeContentType = 'application/octet-stream'; + } + + foreach ($ranges as $range) { + $start = $range[0]; + $end = $range[1]; + + // Streams may advertise themselves as seekable, but still not + // actually allow fseek. We'll manually go forward in the stream + // if fseek failed. + if (!stream_get_meta_data($body)['seekable'] || -1 === fseek($body, $start, SEEK_SET)) { + // don't allow multipart ranges for non-seekable streams + // these streams are not rewindable, so they would have to be closed and opened for each range + if (count($ranges) > 1) { + throw new Exception\RequestedRangeNotSatisfiable('Multipart range requests are not allowed on a non-seekable resource.'); + } + $consumeBlock = 8192; + for ($consumed = 0; $start - $consumed > 0;) { + if (feof($body)) { + throw new Exception\RequestedRangeNotSatisfiable('The start offset ('.$start.') exceeded the size of the entity ('.$consumed.')'); + } + $consumed += strlen(fread($body, min($start - $consumed, $consumeBlock))); + } + } + + // if there is only one range, write it into the reponse body + // otherwise, write multipart header info first before writing the data + if (1 === count($ranges)) { + $rangeLength = $end - $start + 1; + $consumeBlock = 8192; + + // write body data into response body (max 8192 bytes per read cycle) + while ($rangeLength > 0) { + fwrite($rangeBody, fread($body, min($rangeLength, $consumeBlock))); + $rangeLength -= min($rangeLength, $consumeBlock); + } + + // update content length + $contentLength = $end - $start + 1; + } else { + // define new boundary section and write to response body + $boundarySection = '--'.$boundary."\nContent-Type: ".$nodeContentType."\nContent-Range: bytes ".$start.'-'.$end.'/'.$nodeSize."\n\n"; + fwrite($rangeBody, $boundarySection); + + // write body data into response body (max 8192 bytes per read cycle) + $rangeLength = $end - $start + 1; + $consumeBlock = 8192; + while ($rangeLength > 0) { + fwrite($rangeBody, fread($body, min($rangeLength, $consumeBlock))); + $rangeLength -= min($rangeLength, $consumeBlock); + } + + // write line breaks at the end of section + fwrite($rangeBody, "\n\n"); + + // update content length + // +2 for double linebreak at the end of each section + $contentLength += strlen($boundarySection) + ($end - $start + 1) + 2; + } + } + + // rewind the range body after the loop + if (!rewind($rangeBody)) { + throw new Exception\PreconditionFailed('Could not rewind the response stream'); + } + + if (1 === count($ranges)) { + // if only one range, content range header MUST be set (according to spec) + $response->setHeader('Content-Range', 'bytes '.$start.'-'.$end.'/'.$nodeSize); + } else { + // if multiple ranges, content range header MUST NOT be set (according to spec) + // content type header MUST be of type multipart/byteranges + // more specific types can be set in each body section header + $response->setHeader('Content-Type', 'multipart/byteranges; boundary='.$boundary); + } + + $response->setHeader('Content-Length', $contentLength); + + $response->setStatus(206); + $response->setBody($rangeBody); + } else { + if ($nodeSize) { + $response->setHeader('Content-Length', $nodeSize); + } + $response->setStatus(200); + $response->setBody($body); + } + + return false; + } /** * HTTP OPTIONS. diff --git a/lib/DAV/Server.php b/lib/DAV/Server.php index 651649c027..25a654ceb2 100644 --- a/lib/DAV/Server.php +++ b/lib/DAV/Server.php @@ -609,193 +609,201 @@ public function getHTTPDepth($default = self::DEPTH_INFINITY) } /** - * Returns the HTTP range header. - * - * This method returns null if there is no well-formed HTTP range request - * header. - * - * If the string is valid, the ranges will be evaluated and processed in - * the next function - * - * @return string|null - */ - public function getHTTPRange() - { - $rangeHeader = $this->httpRequest->getHeader('range'); - - if (is_null($rangeHeader)) { - return null; - } - - // remove all spaces before evaluation - $rangeHeader = str_replace(' ', '', $rangeHeader); - - // regex to allow multiple ranges - // e.g.'Range: bytes=0-50,100-140, 200-500' - if (!preg_match('/^bytes=(([0-9]*-{1}[0-9]*,? ?)*)$/i', $rangeHeader, $matches)) { - return null; - } - - // remove trailing comma - $matches[1] = trim($matches[1], ','); - - // remove the range string - return $matches[1]; - } - - /** - * Returns the processed ranges of the previously parsed range header - * - * Each range consists of 2 numbers - * The first number specifies the offset of the first byte in the range - * The second number specifies the offset of the last byte in the range - * - * If the second offset is null, it should be treated as the offset of the last byte of the entity - * If the first offset is null, the second offset should be used to retrieve the last x bytes of the entity - * - * If multiple ranges are specified, they will be preprocessed to avoid DOS attacks - * Amount of ranges will be capped dependent on boundary size - * Overlapping ranges will be capped to 2 per request - * Ranges in descending order will be capped dependent on boundary size - * - * @throws Exception\PreconditionFailed if ranges overlap or too many ranges are being requested - * @throws Exception\RequestedRangeNotSatisfiable if an invalid range is requested - * - * @return array|null - */ - public function preprocessRanges($rangeString, $nodeSize) - { - // create an array of all ranges - // result example: [[0, 100], [200, 300]] - $ranges = explode(',', $rangeString); - - // if an invalid range is encountered, its index will be saved in here - // invalid ranges will be removed after the for loop - // examples: start > end, '---200-300---', '-' - $rangesToRemove = []; - - // loop over the array of ranges and do some basic checks and transforms - for ($i = 0; $i < count($ranges); $i++) { - $ranges[$i] = explode('-', $ranges[$i]); - - // mark invalid ranges for removal - // "---200-300---" would be considered a valid range by the regexp - if (count($ranges[$i]) !== 2) { - $rangesToRemove[] = $i; - continue; - } - - // copy the ranges into temporary variables for comparisons - $start = $ranges[$i][0]; - $end = $ranges[$i][1]; - - // if neither start nor end value is defined, mark range for removal - if ($start === "" && $end === "") { - $rangesToRemove[] = $i; - continue; - } - - // transform range parameters for special ranges immediately - // this will make security checks later in this function much simpler - if ($start === "") { - $ranges[$i][0] = $nodeSize - intval($end); - $ranges[$i][1] = $nodeSize - 1; - } else if ($end === "" || $end > ($nodeSize - 1)) { - $ranges[$i][0] = intval($start); - $ranges[$i][1] = $nodeSize - 1; - } else { - $ranges[$i][0] = intval($start); - $ranges[$i][1] = intval($end); - } - - // if start > end, mark it for removal - if ($ranges[$i][0] > $ranges[$i][1]) { - $rangesToRemove[] = $i; - } - } - - // remove invalid ranges - foreach ($rangesToRemove as $idx) { - unset($ranges[$idx]); - } - - // if no ranges are left at this point, return - if (count($ranges) === 0) { - return null; - } - - // rearrange the array keys - $ranges = array_values($ranges); - - /** - * implement checks to prevent DOS attacks, according to rfc7233, 6.1 - */ - - // set up a sorted copy of the ranges array and define the boundary - $sortedRanges = $ranges; - sort($sortedRanges); - - $minRangeBoundary = $sortedRanges[0][0]; - $maxRangeBoundary = max(array_column($sortedRanges, 1)); - $rangeBoundary = $maxRangeBoundary - $minRangeBoundary; - - // check 1: limit the amount of ranges per request depending on range boundary - // accept 1024 requests for a boundary < 1MB, and 512 for boundaries larger than that - // rfc standard doesn't provide specification for the amount of ranges to be allowed per request - if ( - ($rangeBoundary < (1 << 20) && count($ranges) > 1024) - || ($rangeBoundary > (1 << 20) && count($ranges) > 512) - ) { - throw new Exception\PreconditionFailed('Too many ranges in this request'); - } - - // check2: check for overlapping ranges and allow a maximum of 2 - // compare each start value with either the previous end value - // or current max end range value, whichever is bigger - $overlapCounter = 0; - $maxEndValue = -1; - - foreach ($sortedRanges as $range) { - // compare start value with the current maxEndValue - if ($range[0] <= $maxEndValue) { - $overlapCounter++; - - if ($overlapCounter > 2) { - throw new Exception\PreconditionFailed('Too many overlaps'); - } - } - - // check if this ranges end value is larger than maxEndValue and update - if ($range[1] > $maxEndValue) { - $maxEndValue = $range[1]; - } - } - - // check3: ranges should be requested in ascending order - // it can be useful to do occasional descending requests, i.e. if a necessary - // piece of information is located at the end of the file. however, if a range - // consists of many unordered ranges, it can be indicative of a deliberate attack - // limit descending ranges to 32 for boundaries < 1MB and 16 for boundaries larger than that - $descendingStartCounter = 0; - - foreach ($ranges as $idx => $range) { - // skip the first range - if ($idx === 0) continue; - - if ($range[0] < $ranges[$idx - 1][0]) { - $descendingStartCounter++; - } - } - - if ( - ($rangeBoundary < (1 << 20) && $descendingStartCounter > 32) - || ($rangeBoundary > (1 << 20) && $descendingStartCounter > 16) - ) { - throw new Exception\PreconditionFailed('Too many ranges in this request'); - } - - return $ranges; - } + * Returns the HTTP range header. + * + * This method returns null if there is no well-formed HTTP range request + * header. + * + * If the string is valid, the ranges will be evaluated and processed in + * the next function + * + * @return string|null + */ + public function getHTTPRange() + { + $rangeHeader = $this->httpRequest->getHeader('range'); + + if (is_null($rangeHeader)) { + return null; + } + + // remove all spaces before evaluation + $rangeHeader = str_replace(' ', '', $rangeHeader); + + // regex to allow multiple ranges + // e.g.'Range: bytes=0-50,100-140, 200-500' + if (!preg_match('/^bytes=(([0-9]*-{1}[0-9]*,? ?)*)$/i', $rangeHeader, $matches)) { + return null; + } + + // remove trailing comma + $matches[1] = trim($matches[1], ','); + + // remove the range string + return $matches[1]; + } + + /** + * Returns the processed ranges of the previously parsed range header. + * + * Each range consists of 2 numbers + * The first number specifies the offset of the first byte in the range + * The second number specifies the offset of the last byte in the range + * + * If the second offset is null, it should be treated as the offset of the last byte of the entity + * If the first offset is null, the second offset should be used to retrieve the last x bytes of the entity + * + * If multiple ranges are specified, they will be preprocessed to avoid DOS attacks + * Amount of ranges will be capped dependent on boundary size + * Overlapping ranges will be capped to 2 per request + * Ranges in descending order will be capped dependent on boundary size + * + * @throws Exception\PreconditionFailed if ranges overlap or too many ranges are being requested + * @throws Exception\RequestedRangeNotSatisfiable if an invalid range is requested + * + * @return array|null + */ + public function preprocessRanges($rangeString, $nodeSize) + { + // create an array of all ranges + // result example: [[0, 100], [200, 300]] + $ranges = explode(',', $rangeString); + + // if an invalid range is encountered, its index will be saved in here + // invalid ranges will be removed after the for loop + // examples: start > end, '---200-300---', '-' + $rangesToRemove = []; + + // loop over the array of ranges and do some basic checks and transforms + for ($i = 0; $i < count($ranges); ++$i) { + $ranges[$i] = explode('-', $ranges[$i]); + + // mark invalid ranges for removal + // "---200-300---" would be considered a valid range by the regexp + if (2 !== count($ranges[$i])) { + $rangesToRemove[] = $i; + continue; + } + + // copy the ranges into temporary variables for comparisons + $start = $ranges[$i][0]; + $end = $ranges[$i][1]; + + // if neither start nor end value is defined, mark range for removal + if ('' === $start && '' === $end) { + $rangesToRemove[] = $i; + continue; + } + + // if start > filesize, mark for removal + if ($start > $nodeSize) { + $rangesToRemove[] = $i; + continue; + } + + // transform range parameters for special ranges immediately + // this will make security checks later in this function much simpler + if ('' === $start) { + $ranges[$i][0] = max($nodeSize - intval($end), 0); + $ranges[$i][1] = $nodeSize - 1; + } elseif ('' === $end || $end > ($nodeSize - 1)) { + $ranges[$i][0] = intval($start); + $ranges[$i][1] = $nodeSize - 1; + } else { + $ranges[$i][0] = intval($start); + $ranges[$i][1] = intval($end); + } + + // if start > end, mark it for removal + if ($ranges[$i][0] > $ranges[$i][1]) { + $rangesToRemove[] = $i; + } + } + + // remove invalid ranges + foreach ($rangesToRemove as $idx) { + unset($ranges[$idx]); + } + + // if no ranges are left at this point, return + if (0 === count($ranges)) { + throw new Exception\RequestedRangeNotSatisfiable('None of the provided ranges satisfy the requirements. Check out RFC 7233 to learn how to form range requests.'); + } + + // rearrange the array keys + $ranges = array_values($ranges); + + /** + * implement checks to prevent DOS attacks, according to rfc7233, 6.1. + */ + + // set up a sorted copy of the ranges array and define the boundary + $sortedRanges = $ranges; + sort($sortedRanges); + + $minRangeBoundary = $sortedRanges[0][0]; + $maxRangeBoundary = max(array_column($sortedRanges, 1)); + $rangeBoundary = $maxRangeBoundary - $minRangeBoundary; + + // check 1: limit the amount of ranges per request depending on range boundary + // accept 1024 requests for a boundary < 1MB, and 512 for boundaries larger than that + // rfc standard doesn't provide specification for the amount of ranges to be allowed per request + if ( + ($rangeBoundary < (1 << 20) && count($ranges) > 1024) + || ($rangeBoundary > (1 << 20) && count($ranges) > 512) + ) { + throw new Exception\RequestedRangeNotSatisfiable('Too many ranges in this request'); + } + + // check2: check for overlapping ranges and allow a maximum of 2 + // compare each start value with either the previous end value + // or current max end range value, whichever is bigger + $overlapCounter = 0; + $maxEndValue = -1; + + foreach ($sortedRanges as $range) { + // compare start value with the current maxEndValue + if ($range[0] <= $maxEndValue) { + ++$overlapCounter; + + if ($overlapCounter > 2) { + throw new Exception\RequestedRangeNotSatisfiable('Too many overlaps. Consider coalescing your overlapping ranges'); + } + } + + // check if this ranges end value is larger than maxEndValue and update + if ($range[1] > $maxEndValue) { + $maxEndValue = $range[1]; + } + } + + // check3: ranges should be requested in ascending order + // it can be useful to do occasional descending requests, i.e. if a necessary + // piece of information is located at the end of the file. however, if a range + // consists of many unordered ranges, it can be indicative of a deliberate attack + // limit descending ranges to 32 for boundaries < 1MB and 16 for boundaries larger than that + $descendingStartCounter = 0; + + foreach ($ranges as $idx => $range) { + // skip the first range + if (0 === $idx) { + continue; + } + + if ($range[0] < $ranges[$idx - 1][0]) { + ++$descendingStartCounter; + } + } + + if ( + ($rangeBoundary < (1 << 20) && $descendingStartCounter > 32) + || ($rangeBoundary > (1 << 20) && $descendingStartCounter > 16) + ) { + throw new Exception\RequestedRangeNotSatisfiable('Too many unordered ranges in this request. Avoid listing ranges in descending order.'); + } + + return $ranges; + } /** * Returns the HTTP Prefer header information. diff --git a/tests/Sabre/DAV/ServerRangeTest.php b/tests/Sabre/DAV/ServerRangeTest.php index 6d5be46082..fedfcdc29f 100644 --- a/tests/Sabre/DAV/ServerRangeTest.php +++ b/tests/Sabre/DAV/ServerRangeTest.php @@ -249,4 +249,106 @@ public function testIfRangeModificationDateModified() $this->assertEquals(200, $response->getStatus()); $this->assertEquals('Test contents', $response->getBodyAsString()); } + + // the following section contains new multipart range request tests + // this string will be on top of every range. we will need this one a lot + public function getBoundaryResponseString($boundary, $range) + { + return '--'.$boundary."\nContent-Type: application/octet-stream\nContent-Range: bytes ".$range[0].'-'.$range[1].'/'.$range[2]."\n\n"; + } + + public function testMultipartRange() + { + $request = new HTTP\Request('GET', '/files/test.txt', ['Range' => 'bytes=2-5, 8-11']); + $response = $this->request($request); + + $boundary = explode('boundary=', $response->getHeader('Content-Type'))[1]; + + $this->assertEquals([ + 'X-Sabre-Version' => [Version::VERSION], + 'Content-Type' => ['multipart/byteranges; boundary='.$boundary], + 'Content-Length' => [219], + 'ETag' => ['"'.md5('Test contents').'"'], + 'Last-Modified' => [$this->lastModified], + ], + $response->getHeaders() + ); + $this->assertEquals(206, $response->getStatus()); + $this->assertEquals($this->getBoundaryResponseString($boundary, [2, 5, 13]).'st c'."\n\n".$this->getBoundaryResponseString($boundary, [8, 11, 13]).'tent'."\n\n", $response->getBodyAsString()); + } + + /** + * @depends testMultipartRange + */ + public function testPartiallyValidRangeWithSingleValidRange() + { + $request = new HTTP\Request('GET', '/files/test.txt', ['Range' => 'bytes=2-5, 11-8, 100-200, -5-5']); + $response = $this->request($request); + + $this->assertEquals([ + 'X-Sabre-Version' => [Version::VERSION], + 'Content-Type' => ['application/octet-stream'], + 'Content-Length' => [4], + 'Content-Range' => ['bytes 2-5/13'], + 'ETag' => ['"'.md5('Test contents').'"'], + 'Last-Modified' => [$this->lastModified], + ], + $response->getHeaders() + ); + $this->assertEquals(206, $response->getStatus()); + $this->assertEquals('st c', $response->getBodyAsString()); + } + + /** + * @depends testMultipartRange + */ + public function testPartiallyValidRangeWithMultipleValidRanges() + { + $request = new HTTP\Request('GET', '/files/test.txt', ['Range' => 'bytes=2-5, 8-11, 11-8, 100-200, -5-5']); + $response = $this->request($request); + + $boundary = explode('boundary=', $response->getHeader('Content-Type'))[1]; + + $this->assertEquals([ + 'X-Sabre-Version' => [Version::VERSION], + 'Content-Type' => ['multipart/byteranges; boundary='.$boundary], + 'Content-Length' => [219], + 'ETag' => ['"'.md5('Test contents').'"'], + 'Last-Modified' => [$this->lastModified], + ], + $response->getHeaders() + ); + $this->assertEquals(206, $response->getStatus()); + $this->assertEquals($this->getBoundaryResponseString($boundary, [2, 5, 13]).'st c'."\n\n".$this->getBoundaryResponseString($boundary, [8, 11, 13]).'tent'."\n\n", $response->getBodyAsString()); + } + + /** + * @depends testMultipartRange + */ + public function testCompletelyInvalidRange() + { + $request = new HTTP\Request('GET', '/files/test.txt', ['Range' => 'bytes=11-8, 100-200, -5-5']); + $response = $this->request($request); + + $this->assertEquals(416, $response->getStatus()); + } + + /** + * @depends testMultipartRange + */ + public function testOverlappingRange() + { + $request = new HTTP\Request('GET', '/files/test.txt', ['Range' => 'bytes=2-12, 3-5, 6-9, 9-11']); + $response = $this->request($request); + + $this->assertEquals(416, $response->getStatus()); + } + + public function testNonSeekableMultipartRange() + { + $request = new HTTP\Request('GET', '/files/no-seeking.txt', ['Range' => 'bytes=2-5, 7-11']); + $response = $this->request($request); + + $this->assertEquals(416, $response->getStatus()); + } } diff --git a/tests/phpunit.xml b/tests/phpunit.xml index b836b59964..eec9f3278b 100644 --- a/tests/phpunit.xml +++ b/tests/phpunit.xml @@ -38,4 +38,4 @@ - + \ No newline at end of file From 26c04b0f88f0cb29df953488bbb132b8f8e9d317 Mon Sep 17 00:00:00 2001 From: Daniel Riess Date: Mon, 10 Oct 2022 20:57:23 +0700 Subject: [PATCH 6/8] tried fixing codecov complaints --- lib/DAV/CorePlugin.php | 4 +-- lib/DAV/Server.php | 12 +++------ tests/Sabre/DAV/ServerRangeTest.php | 42 ++++++++++++++++++++++++++++- 3 files changed, 45 insertions(+), 13 deletions(-) diff --git a/lib/DAV/CorePlugin.php b/lib/DAV/CorePlugin.php index 4352f55202..c5ad56db92 100644 --- a/lib/DAV/CorePlugin.php +++ b/lib/DAV/CorePlugin.php @@ -236,9 +236,7 @@ public function httpGet(RequestInterface $request, ResponseInterface $response) } // rewind the range body after the loop - if (!rewind($rangeBody)) { - throw new Exception\PreconditionFailed('Could not rewind the response stream'); - } + rewind($rangeBody); if (1 === count($ranges)) { // if only one range, content range header MUST be set (according to spec) diff --git a/lib/DAV/Server.php b/lib/DAV/Server.php index 25a654ceb2..4f16df8cfe 100644 --- a/lib/DAV/Server.php +++ b/lib/DAV/Server.php @@ -746,12 +746,9 @@ public function preprocessRanges($rangeString, $nodeSize) $rangeBoundary = $maxRangeBoundary - $minRangeBoundary; // check 1: limit the amount of ranges per request depending on range boundary - // accept 1024 requests for a boundary < 1MB, and 512 for boundaries larger than that + // accept 512 ranges per request // rfc standard doesn't provide specification for the amount of ranges to be allowed per request - if ( - ($rangeBoundary < (1 << 20) && count($ranges) > 1024) - || ($rangeBoundary > (1 << 20) && count($ranges) > 512) - ) { + if (count($ranges) > 512) { throw new Exception\RequestedRangeNotSatisfiable('Too many ranges in this request'); } @@ -795,10 +792,7 @@ public function preprocessRanges($rangeString, $nodeSize) } } - if ( - ($rangeBoundary < (1 << 20) && $descendingStartCounter > 32) - || ($rangeBoundary > (1 << 20) && $descendingStartCounter > 16) - ) { + if ($descendingStartCounter > 16) { throw new Exception\RequestedRangeNotSatisfiable('Too many unordered ranges in this request. Avoid listing ranges in descending order.'); } diff --git a/tests/Sabre/DAV/ServerRangeTest.php b/tests/Sabre/DAV/ServerRangeTest.php index fedfcdc29f..1ab4b3075a 100644 --- a/tests/Sabre/DAV/ServerRangeTest.php +++ b/tests/Sabre/DAV/ServerRangeTest.php @@ -27,6 +27,7 @@ public function setup(): void { parent::setUp(); $this->server->createFile('files/test.txt', 'Test contents'); + $this->server->createFile('files/rangetest.txt', 'Lorem ipsum dolor sit amet, consectetur adipiscing elit. Cras massa quam, tempus a bibendum eget, accumsan et risus. Duis volutpat diam consectetur lorem scelerisque, vitae tristique massa tristique. Donec porta elementum condimentum. Duis fringilla, est sed tempus placerat, tortor tortor pulvinar lacus, in semper magna felis id nisi. Pellentesque eleifend augue elit, non hendrerit ex euismod at. Morbi a auctor mi. Suspendisse vel imperdiet lacus. Aenean auctor nulla urna, in sagittis felis venenatis fringilla. Nulla facilisi. Suspendisse nunc.'); $this->lastModified = HTTP\toDate( new DateTime('@'.$this->server->tree->getNodeForPath('files/test.txt')->getLastModified()) @@ -148,6 +149,17 @@ public function testNonSeekableStream() $this->assertEquals('st c', $response->getBodyAsString()); } + /** + * @depends testNonSeekableStream + */ + public function testNonSeekableExceedingRange() + { + $request = new HTTP\Request('GET', '/files/no-seeking.txt', ['Range' => 'bytes=100-200']); + $response = $this->request($request); + + $this->assertEquals(416, $response->getStatus()); + } + /** * @depends testRange */ @@ -327,7 +339,7 @@ public function testPartiallyValidRangeWithMultipleValidRanges() */ public function testCompletelyInvalidRange() { - $request = new HTTP\Request('GET', '/files/test.txt', ['Range' => 'bytes=11-8, 100-200, -5-5']); + $request = new HTTP\Request('GET', '/files/test.txt', ['Range' => 'bytes=11-8, 100-200, -5-5, -']); $response = $this->request($request); $this->assertEquals(416, $response->getStatus()); @@ -344,6 +356,34 @@ public function testOverlappingRange() $this->assertEquals(416, $response->getStatus()); } + public function testTooManyRanges() + { + $ranges = []; + for ($i = 0; $i < 513; ++$i) { + $ranges[] = $i.'-'.$i; + } + $byterange = 'bytes='.implode(',', $ranges); + + $request = new HTTP\Request('GET', '/files/rangetest.txt', ['Range' => $byterange]); + $response = $this->request($request); + + $this->assertEquals(416, $response->getStatus()); + } + + public function testUnorderedRanges() + { + $ranges = []; + for ($i = 0; $i < 18; ++$i) { + $ranges[] = (20 - $i).'-'.(20 - $i); + } + $byterange = 'bytes='.implode(',', $ranges); + + $request = new HTTP\Request('GET', '/files/rangetest.txt', ['Range' => $byterange]); + $response = $this->request($request); + + $this->assertEquals(416, $response->getStatus()); + } + public function testNonSeekableMultipartRange() { $request = new HTTP\Request('GET', '/files/no-seeking.txt', ['Range' => 'bytes=2-5, 7-11']); From 957b9b17f6fb60f085e8127da667fdd597127864 Mon Sep 17 00:00:00 2001 From: Phil Davis Date: Wed, 18 Jan 2023 12:06:32 +0545 Subject: [PATCH 7/8] Typos in comments --- lib/DAV/CorePlugin.php | 4 ++-- tests/phpunit.xml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/DAV/CorePlugin.php b/lib/DAV/CorePlugin.php index c5ad56db92..9aaa7a0a8d 100644 --- a/lib/DAV/CorePlugin.php +++ b/lib/DAV/CorePlugin.php @@ -117,7 +117,7 @@ public function httpGet(RequestInterface $request, ResponseInterface $response) $response->addHeaders($httpHeaders); - // this function now only checks, if the range string correctly formatted + // this function now only checks, if the range string is correctly formatted // and returns it (or null if it isn't or it doesn't exist) // processing of the ranges will then be done further below $rangeString = $this->server->getHTTPRange(); @@ -199,7 +199,7 @@ public function httpGet(RequestInterface $request, ResponseInterface $response) } } - // if there is only one range, write it into the reponse body + // if there is only one range, write it into the response body // otherwise, write multipart header info first before writing the data if (1 === count($ranges)) { $rangeLength = $end - $start + 1; diff --git a/tests/phpunit.xml b/tests/phpunit.xml index eec9f3278b..b836b59964 100644 --- a/tests/phpunit.xml +++ b/tests/phpunit.xml @@ -38,4 +38,4 @@ - \ No newline at end of file + From d2879cf4b4482882305a8aed1671dcffe2a95c2e Mon Sep 17 00:00:00 2001 From: Daniel Riess Date: Thu, 22 Jun 2023 18:03:38 +0700 Subject: [PATCH 8/8] added old getHTTPRange function for BC and marked as deprecated, renamed new function to getHTTPRangeHeaderValue, moved preprocessRanges to CorePlugin --- lib/DAV/CorePlugin.php | 164 +++++++++++++++++++++++++++++++++- lib/DAV/Server.php | 197 +++++++++-------------------------------- 2 files changed, 200 insertions(+), 161 deletions(-) diff --git a/lib/DAV/CorePlugin.php b/lib/DAV/CorePlugin.php index 9aaa7a0a8d..3dbd9b1b5b 100644 --- a/lib/DAV/CorePlugin.php +++ b/lib/DAV/CorePlugin.php @@ -120,13 +120,13 @@ public function httpGet(RequestInterface $request, ResponseInterface $response) // this function now only checks, if the range string is correctly formatted // and returns it (or null if it isn't or it doesn't exist) // processing of the ranges will then be done further below - $rangeString = $this->server->getHTTPRange(); + $rangeHeaderValue = $this->server->getHTTPRangeHeaderValue(); $ifRange = $request->getHeader('If-Range'); $ignoreRangeHeader = false; // If ifRange is set, and range is specified, we first need to check // the precondition. - if ($nodeSize && $rangeString && $ifRange) { + if ($nodeSize && $rangeHeaderValue && $ifRange) { // if IfRange is parsable as a date we'll treat it as a DateTime // otherwise, we must treat it as an etag. try { @@ -154,10 +154,10 @@ public function httpGet(RequestInterface $request, ResponseInterface $response) $ranges = null; - if (!$ignoreRangeHeader && $nodeSize && $rangeString) { + if (!$ignoreRangeHeader && $nodeSize && $rangeHeaderValue) { // preprocess the ranges now // this involves removing invalid ranges and security checks - $ranges = $this->server->preprocessRanges($rangeString, $nodeSize); + $ranges = $this->preprocessRanges($rangeHeaderValue, $nodeSize); } if (!$ignoreRangeHeader && $nodeSize && $ranges) { @@ -965,4 +965,160 @@ public function getPluginInfo() 'link' => null, ]; } + + /** + * Returns the processed ranges of the previously parsed range header. + * + * Each range consists of 2 numbers + * The first number specifies the offset of the first byte in the range + * The second number specifies the offset of the last byte in the range + * + * If the second offset is null, it should be treated as the offset of the last byte of the entity + * If the first offset is null, the second offset should be used to retrieve the last x bytes of the entity + * + * If multiple ranges are specified, they will be preprocessed to avoid DOS attacks + * Amount of ranges will be capped dependent on boundary size + * Overlapping ranges will be capped to 2 per request + * Ranges in descending order will be capped dependent on boundary size + * + * @throws Exception\PreconditionFailed if ranges overlap or too many ranges are being requested + * @throws Exception\RequestedRangeNotSatisfiable if an invalid range is requested + * + * @return array|null + */ + private function preprocessRanges($rangeString, $nodeSize) + { + // create an array of all ranges + // result example: [[0, 100], [200, 300]] + $ranges = explode(',', $rangeString); + + // if an invalid range is encountered, its index will be saved in here + // invalid ranges will be removed after the for loop + // examples: start > end, '---200-300---', '-' + $rangesToRemove = []; + + // loop over the array of ranges and do some basic checks and transforms + for ($i = 0; $i < count($ranges); ++$i) { + $ranges[$i] = explode('-', $ranges[$i]); + + // mark invalid ranges for removal + // "---200-300---" would be considered a valid range by the regexp + if (2 !== count($ranges[$i])) { + $rangesToRemove[] = $i; + continue; + } + + // copy the ranges into temporary variables for comparisons + $start = $ranges[$i][0]; + $end = $ranges[$i][1]; + + // if neither start nor end value is defined, mark range for removal + if ('' === $start && '' === $end) { + $rangesToRemove[] = $i; + continue; + } + + // if start > filesize, mark for removal + if ($start > $nodeSize) { + $rangesToRemove[] = $i; + continue; + } + + // transform range parameters for special ranges immediately + // this will make security checks later in this function much simpler + if ('' === $start) { + $ranges[$i][0] = max($nodeSize - intval($end), 0); + $ranges[$i][1] = $nodeSize - 1; + } elseif ('' === $end || $end > ($nodeSize - 1)) { + $ranges[$i][0] = intval($start); + $ranges[$i][1] = $nodeSize - 1; + } else { + $ranges[$i][0] = intval($start); + $ranges[$i][1] = intval($end); + } + + // if start > end, mark it for removal + if ($ranges[$i][0] > $ranges[$i][1]) { + $rangesToRemove[] = $i; + } + } + + // remove invalid ranges + foreach ($rangesToRemove as $idx) { + unset($ranges[$idx]); + } + + // if no ranges are left at this point, return + if (0 === count($ranges)) { + throw new Exception\RequestedRangeNotSatisfiable('None of the provided ranges satisfy the requirements. Check out RFC 7233 to learn how to form range requests.'); + } + + // rearrange the array keys + $ranges = array_values($ranges); + + /** + * implement checks to prevent DOS attacks, according to rfc7233, 6.1. + */ + + // set up a sorted copy of the ranges array and define the boundary + $sortedRanges = $ranges; + sort($sortedRanges); + + $minRangeBoundary = $sortedRanges[0][0]; + $maxRangeBoundary = max(array_column($sortedRanges, 1)); + $rangeBoundary = $maxRangeBoundary - $minRangeBoundary; + + // check 1: limit the amount of ranges per request depending on range boundary + // accept 512 ranges per request + // rfc standard doesn't provide specification for the amount of ranges to be allowed per request + if (count($ranges) > 512) { + throw new Exception\RequestedRangeNotSatisfiable('Too many ranges in this request'); + } + + // check2: check for overlapping ranges and allow a maximum of 2 + // compare each start value with either the previous end value + // or current max end range value, whichever is bigger + $overlapCounter = 0; + $maxEndValue = -1; + + foreach ($sortedRanges as $range) { + // compare start value with the current maxEndValue + if ($range[0] <= $maxEndValue) { + ++$overlapCounter; + + if ($overlapCounter > 2) { + throw new Exception\RequestedRangeNotSatisfiable('Too many overlaps. Consider coalescing your overlapping ranges'); + } + } + + // check if this ranges end value is larger than maxEndValue and update + if ($range[1] > $maxEndValue) { + $maxEndValue = $range[1]; + } + } + + // check3: ranges should be requested in ascending order + // it can be useful to do occasional descending requests, i.e. if a necessary + // piece of information is located at the end of the file. however, if a range + // consists of many unordered ranges, it can be indicative of a deliberate attack + // limit descending ranges to 32 for boundaries < 1MB and 16 for boundaries larger than that + $descendingStartCounter = 0; + + foreach ($ranges as $idx => $range) { + // skip the first range + if (0 === $idx) { + continue; + } + + if ($range[0] < $ranges[$idx - 1][0]) { + ++$descendingStartCounter; + } + } + + if ($descendingStartCounter > 16) { + throw new Exception\RequestedRangeNotSatisfiable('Too many unordered ranges in this request. Avoid listing ranges in descending order.'); + } + + return $ranges; + } } diff --git a/lib/DAV/Server.php b/lib/DAV/Server.php index 4f16df8cfe..a243dbd170 100644 --- a/lib/DAV/Server.php +++ b/lib/DAV/Server.php @@ -612,191 +612,74 @@ public function getHTTPDepth($default = self::DEPTH_INFINITY) * Returns the HTTP range header. * * This method returns null if there is no well-formed HTTP range request - * header. + * header or array($start, $end). * - * If the string is valid, the ranges will be evaluated and processed in - * the next function + * The first number is the offset of the first byte in the range. + * The second number is the offset of the last byte in the range. * - * @return string|null + * If the second offset is null, it should be treated as the offset of the last byte of the entity + * If the first offset is null, the second offset should be used to retrieve the last x bytes of the entity + * + * @return int[]|null + * + * @deprecated Use getHTTPRangeHeaderValue in combination with CorePlugin->preprocessRanges to account for multipart ranges */ public function getHTTPRange() { - $rangeHeader = $this->httpRequest->getHeader('range'); - - if (is_null($rangeHeader)) { + $range = $this->httpRequest->getHeader('range'); + if (is_null($range)) { return null; } - // remove all spaces before evaluation - $rangeHeader = str_replace(' ', '', $rangeHeader); + // Matching "Range: bytes=1234-5678: both numbers are optional - // regex to allow multiple ranges - // e.g.'Range: bytes=0-50,100-140, 200-500' - if (!preg_match('/^bytes=(([0-9]*-{1}[0-9]*,? ?)*)$/i', $rangeHeader, $matches)) { + if (!preg_match('/^bytes=([0-9]*)-([0-9]*)$/i', $range, $matches)) { return null; } - // remove trailing comma - $matches[1] = trim($matches[1], ','); + if ('' === $matches[1] && '' === $matches[2]) { + return null; + } - // remove the range string - return $matches[1]; + return [ + '' !== $matches[1] ? (int) $matches[1] : null, + '' !== $matches[2] ? (int) $matches[2] : null, + ]; } /** - * Returns the processed ranges of the previously parsed range header. - * - * Each range consists of 2 numbers - * The first number specifies the offset of the first byte in the range - * The second number specifies the offset of the last byte in the range - * - * If the second offset is null, it should be treated as the offset of the last byte of the entity - * If the first offset is null, the second offset should be used to retrieve the last x bytes of the entity + * Returns the HTTP range header. * - * If multiple ranges are specified, they will be preprocessed to avoid DOS attacks - * Amount of ranges will be capped dependent on boundary size - * Overlapping ranges will be capped to 2 per request - * Ranges in descending order will be capped dependent on boundary size + * This method returns null if there is no well-formed HTTP range request + * header. * - * @throws Exception\PreconditionFailed if ranges overlap or too many ranges are being requested - * @throws Exception\RequestedRangeNotSatisfiable if an invalid range is requested + * If the string is valid, the ranges will be evaluated and processed in + * the next function * - * @return array|null + * @return string|null */ - public function preprocessRanges($rangeString, $nodeSize) + public function getHTTPRangeHeaderValue() { - // create an array of all ranges - // result example: [[0, 100], [200, 300]] - $ranges = explode(',', $rangeString); - - // if an invalid range is encountered, its index will be saved in here - // invalid ranges will be removed after the for loop - // examples: start > end, '---200-300---', '-' - $rangesToRemove = []; - - // loop over the array of ranges and do some basic checks and transforms - for ($i = 0; $i < count($ranges); ++$i) { - $ranges[$i] = explode('-', $ranges[$i]); - - // mark invalid ranges for removal - // "---200-300---" would be considered a valid range by the regexp - if (2 !== count($ranges[$i])) { - $rangesToRemove[] = $i; - continue; - } - - // copy the ranges into temporary variables for comparisons - $start = $ranges[$i][0]; - $end = $ranges[$i][1]; - - // if neither start nor end value is defined, mark range for removal - if ('' === $start && '' === $end) { - $rangesToRemove[] = $i; - continue; - } - - // if start > filesize, mark for removal - if ($start > $nodeSize) { - $rangesToRemove[] = $i; - continue; - } - - // transform range parameters for special ranges immediately - // this will make security checks later in this function much simpler - if ('' === $start) { - $ranges[$i][0] = max($nodeSize - intval($end), 0); - $ranges[$i][1] = $nodeSize - 1; - } elseif ('' === $end || $end > ($nodeSize - 1)) { - $ranges[$i][0] = intval($start); - $ranges[$i][1] = $nodeSize - 1; - } else { - $ranges[$i][0] = intval($start); - $ranges[$i][1] = intval($end); - } - - // if start > end, mark it for removal - if ($ranges[$i][0] > $ranges[$i][1]) { - $rangesToRemove[] = $i; - } - } - - // remove invalid ranges - foreach ($rangesToRemove as $idx) { - unset($ranges[$idx]); - } - - // if no ranges are left at this point, return - if (0 === count($ranges)) { - throw new Exception\RequestedRangeNotSatisfiable('None of the provided ranges satisfy the requirements. Check out RFC 7233 to learn how to form range requests.'); - } - - // rearrange the array keys - $ranges = array_values($ranges); - - /** - * implement checks to prevent DOS attacks, according to rfc7233, 6.1. - */ - - // set up a sorted copy of the ranges array and define the boundary - $sortedRanges = $ranges; - sort($sortedRanges); - - $minRangeBoundary = $sortedRanges[0][0]; - $maxRangeBoundary = max(array_column($sortedRanges, 1)); - $rangeBoundary = $maxRangeBoundary - $minRangeBoundary; - - // check 1: limit the amount of ranges per request depending on range boundary - // accept 512 ranges per request - // rfc standard doesn't provide specification for the amount of ranges to be allowed per request - if (count($ranges) > 512) { - throw new Exception\RequestedRangeNotSatisfiable('Too many ranges in this request'); - } - - // check2: check for overlapping ranges and allow a maximum of 2 - // compare each start value with either the previous end value - // or current max end range value, whichever is bigger - $overlapCounter = 0; - $maxEndValue = -1; - - foreach ($sortedRanges as $range) { - // compare start value with the current maxEndValue - if ($range[0] <= $maxEndValue) { - ++$overlapCounter; - - if ($overlapCounter > 2) { - throw new Exception\RequestedRangeNotSatisfiable('Too many overlaps. Consider coalescing your overlapping ranges'); - } - } + $rangeHeader = $this->httpRequest->getHeader('range'); - // check if this ranges end value is larger than maxEndValue and update - if ($range[1] > $maxEndValue) { - $maxEndValue = $range[1]; - } + if (is_null($rangeHeader)) { + return null; } - // check3: ranges should be requested in ascending order - // it can be useful to do occasional descending requests, i.e. if a necessary - // piece of information is located at the end of the file. however, if a range - // consists of many unordered ranges, it can be indicative of a deliberate attack - // limit descending ranges to 32 for boundaries < 1MB and 16 for boundaries larger than that - $descendingStartCounter = 0; - - foreach ($ranges as $idx => $range) { - // skip the first range - if (0 === $idx) { - continue; - } + // remove all spaces before evaluation + $rangeHeader = str_replace(' ', '', $rangeHeader); - if ($range[0] < $ranges[$idx - 1][0]) { - ++$descendingStartCounter; - } + // regex to allow multiple ranges + // e.g.'Range: bytes=0-50,100-140, 200-500' + if (!preg_match('/^bytes=(([0-9]*-{1}[0-9]*,? ?)*)$/i', $rangeHeader, $matches)) { + return null; } - if ($descendingStartCounter > 16) { - throw new Exception\RequestedRangeNotSatisfiable('Too many unordered ranges in this request. Avoid listing ranges in descending order.'); - } + // remove trailing comma + $matches[1] = trim($matches[1], ','); - return $ranges; + // remove the range string + return $matches[1]; } /**