feat: v4

- Global CLI, you do not need to wrap active downloads with downloadSequence
- Auto increase parallel stream, maximize download speed
- Not reusing redirected URL by default - prevent token expires for long downloads
- Performance & stability improvements
- Remote CLI progress, show download progress in another process easily

BREAKING CHANGE:
- `partsURL` removed in favor of `partURLs`
- Not reusing redirected URL by default
- Different chunk size based on programType
- You can recall `.download()` and it will not throw an error - will return the same promise of downloading (do *not* redownload the file)
- You can change the parallel stream count after download has started

Author: ido-pluto

.github/workflows/build.yml

@@ -22,19 +22,19 @@ jobs:
       - name: Generate docs
         run: npm run generate-docs
       - name: Upload build artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: "build"
           path: "dist"
       - name: Upload build artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: "docs"
           path: "docs"
 
   release:
     name: Release
-    if: github.ref == 'refs/heads/main'
+    if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/beta'
     runs-on: ubuntu-latest
     concurrency: release-${{ github.ref }}
     environment:
@@ -52,10 +52,10 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
-          node-version: "20"
+          node-version: "22"
       - name: Install modules
         run: npm ci --ignore-scripts
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           path: artifacts
       - name: Move artifacts
@@ -73,16 +73,16 @@ jobs:
         id: set-npm-url
         run: |
           if [ -f .semanticRelease.npmPackage.deployedVersion.txt ]; then
-            echo "npm-url=https://www.npmjs.com/package/node-llama-cpp/v/$(cat .semanticRelease.npmPackage.deployedVersion.txt)" >> $GITHUB_OUTPUT
+            echo "npm-url=https://www.npmjs.com/package/ipull/v/$(cat .semanticRelease.npmPackage.deployedVersion.txt)" >> $GITHUB_OUTPUT
           fi
       - name: Upload docs to GitHub Pages
         if: steps.set-npm-url.outputs.npm-url != ''
-        uses: actions/upload-pages-artifact@v2
+        uses: actions/upload-pages-artifact@v3
         with:
           name: pages-docs
           path: docs
       - name: Deploy docs to GitHub Pages
-        if: steps.set-npm-url.outputs.npm-url != ''
-        uses: actions/deploy-pages@v2
+        if: steps.set-npm-url.outputs.npm-url != '' && github.ref == 'refs/heads/main'
+        uses: actions/deploy-pages@v4
         with:
           artifact_name: pages-docs

.releaserc.json

@@ -1,6 +1,10 @@
 {
     "branches": [
-        "main"
+        "main",
+        {
+            "name": "beta",
+            "prerelease": true
+        }
     ],
     "ci": true,
     "plugins": [

README.md

@@ -25,6 +25,7 @@ npx ipull http://example.com/file.large
 ## Features
 
 - Download using parallels connections
+- Maximize download speed (automatic parallelization, 3+)
 - Pausing and resuming downloads
 - Node.js and browser support
 - Smart retry on fail
@@ -73,8 +74,9 @@ import {downloadFileBrowser} from "ipull/dist/browser.js";
 
 const downloader = await downloadFileBrowser({
     url: 'https://example.com/file.large',
-    onWrite: (cursor: number, buffer: Uint8Array, options) => {
-        console.log(`Writing ${buffer.length} bytes at cursor ${cursor}, with options: ${JSON.stringify(options)}`);
+    onWrite: (cursor: number, buffers: Uint8Array[], options) => {
+        const totalLength = buffers.reduce((acc, buffer) => acc + buffer.length, 0);
+        console.log(`Writing ${totalLength} bytes at cursor ${cursor}, with options: ${JSON.stringify(options)}`);
     }
 });
 
@@ -234,12 +236,12 @@ If the maximum reties was reached the download will fail and an error will be th
 ```ts
 import {downloadFile} from 'ipull';
 
-const downloader = await downloadFile({
-    url: 'https://example.com/file.large',
-    directory: './this/path'
-});
-
 try {
+    const downloader = await downloadFile({
+        url: 'https://example.com/file.large',
+        directory: './this/path'
+    });
+
     await downloader.download();
 } catch (error) {
     console.error(`Download failed: ${error.message}`);
@@ -250,6 +252,8 @@ try {
 
 In some edge cases, the re-try mechanism may give the illusion that the download is stuck.
 
+(You can see this in the progress object that "retrying" is true)
+
 To debug this, disable the re-try mechanism:
 
 ```js
@@ -258,6 +262,9 @@ const downloader = await downloadFile({
     directory: './this/path',
     retry: {
         retries: 0
+    },
+    retryFetchDownloadInfo: {
+        retries: 0
     }
 });
 ```
@@ -286,6 +293,27 @@ downloader.on("progress", (progress) => {
 });
 ```
 
+### Remote Download Listing
+
+If you want to show in the CLI the progress of a file downloading in remote.
+
+```ts
+const originaldownloader = await downloadFile({
+    url: 'https://example.com/file.large',
+    directory: './this/path'
+});
+
+const remoteDownloader = downloadFileRemote({
+    cliProgress: true
+});
+
+originaldownloader.on("progress", (progress) => {
+    remoteDownloader.emitRemoteProgress(progress);
+});
+
+await originaldownloader.download();
+```
+
 ### Download multiple files
 
 If you want to download multiple files, you can use the `downloadSequence` function.

examples/browser-log.ts

@@ -1,11 +1,12 @@
-import {downloadFileBrowser} from "ipull/dist/browser.js";
+import {downloadFileBrowser} from "ipull/browser";
 
 const BIG_IMAGE = "https://upload.wikimedia.org/wikipedia/commons/9/9e/1_dubrovnik_pano_-_edit1.jpg"; // 40mb
 
 const downloader = await downloadFileBrowser({
     url: BIG_IMAGE,
-    onWrite: (cursor: number, buffer: Uint8Array, options) => {
-        console.log(`Writing ${buffer.length} bytes at cursor ${cursor}, with options: ${JSON.stringify(options)}`);
+    onWrite: (cursor: number, buffers: Uint8Array[], options: DownloadEngineWriteStreamOptionsBrowser) => {
+        const totalLength = buffers.reduce((acc, b) => acc + b.byteLength, 0);
+        console.log(`Writing ${totalLength} bytes at cursor ${cursor}, with options: ${JSON.stringify(options)}`);
     }
 });
 

package-lock.json

@@ -147,6 +147,7 @@
     "sleep-promise": "^9.1.0",
     "slice-ansi": "^7.1.0",
     "stdout-update": "^4.0.1",
-    "strip-ansi": "^7.1.0"
+    "strip-ansi": "^7.1.0",
+    "uid": "^2.0.2"
   }
 }

package.json

@@ -1,4 +1,4 @@
-import {downloadFileBrowser, DownloadFileBrowserOptions, downloadSequenceBrowser} from "./download/browser-download.js";
+import {downloadFileBrowser, DownloadFileBrowserOptions, downloadFileRemoteBrowser, downloadSequenceBrowser, DownloadSequenceBrowserOptions} from "./download/browser-download.js";
 import DownloadEngineBrowser from "./download/download-engine/engine/download-engine-browser.js";
 import EmptyResponseError from "./download/download-engine/streams/download-engine-fetch-stream/errors/empty-response-error.js";
 import StatusCodeError from "./download/download-engine/streams/download-engine-fetch-stream/errors/status-code-error.js";
@@ -9,17 +9,21 @@ import FetchStreamError from "./download/download-engine/streams/download-engine
 import IpullError from "./errors/ipull-error.js";
 import EngineError from "./download/download-engine/engine/error/engine-error.js";
 import {FormattedStatus} from "./download/transfer-visualize/format-transfer-status.js";
-import DownloadEngineMultiDownload from "./download/download-engine/engine/download-engine-multi-download.js";
+import DownloadEngineMultiDownload, {DownloadEngineMultiAllowedEngines} from "./download/download-engine/engine/download-engine-multi-download.js";
 import HttpError from "./download/download-engine/streams/download-engine-fetch-stream/errors/http-error.js";
 import BaseDownloadEngine from "./download/download-engine/engine/base-download-engine.js";
 import {InvalidOptionError} from "./download/download-engine/engine/error/InvalidOptionError.js";
 import {DownloadFlags, DownloadStatus} from "./download/download-engine/download-file/progress-status-file.js";
-import {NoDownloadEngineProvidedError} from "./download/download-engine/engine/error/no-download-engine-provided-error.js";
+import {DownloadEngineRemote} from "./download/download-engine/engine/DownloadEngineRemote.js";
+import {
+    DownloadEngineWriteStreamOptionsBrowser
+} from "./download/download-engine/streams/download-engine-write-stream/download-engine-write-stream-browser.js";
 
 export {
     DownloadFlags,
     DownloadStatus,
     downloadFileBrowser,
+    downloadFileRemoteBrowser,
     downloadSequenceBrowser,
     EmptyResponseError,
     HttpError,
@@ -29,15 +33,18 @@ export {
     FetchStreamError,
     IpullError,
     EngineError,
-    InvalidOptionError,
-    NoDownloadEngineProvidedError
+    InvalidOptionError
 };
 
 export type {
+    DownloadEngineRemote,
     BaseDownloadEngine,
     DownloadFileBrowserOptions,
     DownloadEngineBrowser,
     DownloadEngineMultiDownload,
+    DownloadEngineMultiAllowedEngines,
     FormattedStatus,
-    SaveProgressInfo
+    SaveProgressInfo,
+    DownloadSequenceBrowserOptions,
+    DownloadEngineWriteStreamOptionsBrowser
 };