diff --git a/.dockerignore b/.dockerignore index fbcebd3ae..9a906c9e4 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,8 @@ __mocks__ .husky **/build -**/docker +**/docker/electron-builder **/coverage **/node_modules +docker +!docker/guardoni/guardoni.config.json \ No newline at end of file diff --git a/.github/workflows/guardoni_pull_request.yml b/.github/workflows/guardoni_pull_request.yml index 9f792c116..4841329d5 100644 --- a/.github/workflows/guardoni_pull_request.yml +++ b/.github/workflows/guardoni_pull_request.yml @@ -10,8 +10,8 @@ on: - fix/** - refactor/** paths: - - "packages/shared/**" - - "platforms/**" + - 'packages/shared/**' + - 'platforms/**' jobs: pull_request: @@ -34,7 +34,7 @@ jobs: uses: actions/setup-node@v2 with: node-version: ${{ matrix.node-version }} - cache: "yarn" + cache: 'yarn' cache-dependency-path: yarn.lock - name: Install modules @@ -69,9 +69,6 @@ jobs: guardoni_build: name: Guardoni build runs-on: ${{ matrix.config.os }} - # defaults: - # run: - # shell: bash --noprofile --norc -exo pipefail {0} strategy: max-parallel: 1 matrix: @@ -126,21 +123,28 @@ jobs: platforms/guardoni/dist platforms/ycai/studio/build + - name: Install Xvfb + run: | + sudo apt install -y xvfb x11-utils libxkbcommon-x11-0 libxcb-icccm4 libxcb-image0 libxcb-keysyms1 libxcb-randr0 libxcb-render-util0 libxcb-xinerama0 libxcb-xfixes0 xdotool + - name: Install dependencies run: yarn - name: Start PM2 - run: | - yarn pm2 start --env test ./platforms/ecosystem.config.js - sleep 5 + run: yarn pm2 start --env test ./platforms/ecosystem.config.js - name: Build Guardoni for ${{ matrix.config.os }} working-directory: ./platforms/guardoni run: ./scripts/cli-build.mjs - - name: Test YT cli + - name: Test CLI - YT Home working-directory: ./platforms/guardoni - run: ./scripts/cli-yt-test.mjs + env: + DISPLAY: ':99.0' + run: xvfb-run --auto-servernum ./scripts/cli-yt-test-home.mjs + + - name: Test CLI - YT Videos + run: yarn guardoni cli-yt-test-videos - name: Stop PM2 run: yarn pm2 stop all diff --git a/.yarn/cache/@mrmlnc-readdir-enhanced-npm-2.2.1-5286808663-d3b82b2936.zip b/.yarn/cache/@mrmlnc-readdir-enhanced-npm-2.2.1-5286808663-d3b82b2936.zip deleted file mode 100644 index 71770e76a..000000000 Binary files a/.yarn/cache/@mrmlnc-readdir-enhanced-npm-2.2.1-5286808663-d3b82b2936.zip and /dev/null differ diff --git a/.yarn/cache/@nodelib-fs.stat-npm-1.1.3-95bc1892a0-318deab369.zip b/.yarn/cache/@nodelib-fs.stat-npm-1.1.3-95bc1892a0-318deab369.zip deleted file mode 100644 index 4c68ef6d9..000000000 Binary files a/.yarn/cache/@nodelib-fs.stat-npm-1.1.3-95bc1892a0-318deab369.zip and /dev/null differ diff --git a/.yarn/cache/@typescript-eslint-eslint-plugin-npm-5.30.0-c72e6ffad8-f2fe96082c.zip b/.yarn/cache/@typescript-eslint-eslint-plugin-npm-5.30.5-56e8842647-cf763fb091.zip similarity index 94% rename from .yarn/cache/@typescript-eslint-eslint-plugin-npm-5.30.0-c72e6ffad8-f2fe96082c.zip rename to .yarn/cache/@typescript-eslint-eslint-plugin-npm-5.30.5-56e8842647-cf763fb091.zip index 9a89bc80b..9ed7baa45 100644 Binary files a/.yarn/cache/@typescript-eslint-eslint-plugin-npm-5.30.0-c72e6ffad8-f2fe96082c.zip and b/.yarn/cache/@typescript-eslint-eslint-plugin-npm-5.30.5-56e8842647-cf763fb091.zip differ diff --git a/.yarn/cache/@typescript-eslint-parser-npm-4.33.0-799c6ce8d5-102457eae1.zip b/.yarn/cache/@typescript-eslint-parser-npm-4.33.0-799c6ce8d5-102457eae1.zip deleted file mode 100644 index 2e52119d4..000000000 Binary files a/.yarn/cache/@typescript-eslint-parser-npm-4.33.0-799c6ce8d5-102457eae1.zip and /dev/null differ diff --git a/.yarn/cache/@typescript-eslint-parser-npm-5.30.0-02dd191940-7067ba4ede.zip b/.yarn/cache/@typescript-eslint-parser-npm-5.30.5-27a499c345-6c16821e12.zip similarity index 88% rename from .yarn/cache/@typescript-eslint-parser-npm-5.30.0-02dd191940-7067ba4ede.zip rename to .yarn/cache/@typescript-eslint-parser-npm-5.30.5-27a499c345-6c16821e12.zip index 1a03371e2..48eb12769 100644 Binary files a/.yarn/cache/@typescript-eslint-parser-npm-5.30.0-02dd191940-7067ba4ede.zip and b/.yarn/cache/@typescript-eslint-parser-npm-5.30.5-27a499c345-6c16821e12.zip differ diff --git a/.yarn/cache/@typescript-eslint-scope-manager-npm-4.33.0-28014c179d-9a25fb7ba7.zip b/.yarn/cache/@typescript-eslint-scope-manager-npm-4.33.0-28014c179d-9a25fb7ba7.zip deleted file mode 100644 index ada83acdc..000000000 Binary files a/.yarn/cache/@typescript-eslint-scope-manager-npm-4.33.0-28014c179d-9a25fb7ba7.zip and /dev/null differ diff --git a/.yarn/cache/@typescript-eslint-scope-manager-npm-5.30.0-53c13117cb-51246d0f6c.zip b/.yarn/cache/@typescript-eslint-scope-manager-npm-5.30.5-3e1fbf788a-509bee6d62.zip similarity index 89% rename from .yarn/cache/@typescript-eslint-scope-manager-npm-5.30.0-53c13117cb-51246d0f6c.zip rename to .yarn/cache/@typescript-eslint-scope-manager-npm-5.30.5-3e1fbf788a-509bee6d62.zip index 868fb8389..a3c62d9be 100644 Binary files a/.yarn/cache/@typescript-eslint-scope-manager-npm-5.30.0-53c13117cb-51246d0f6c.zip and b/.yarn/cache/@typescript-eslint-scope-manager-npm-5.30.5-3e1fbf788a-509bee6d62.zip differ diff --git a/.yarn/cache/@typescript-eslint-type-utils-npm-5.30.0-de31c989eb-6185117638.zip b/.yarn/cache/@typescript-eslint-type-utils-npm-5.30.5-765594d88b-080cc12317.zip similarity index 87% rename from .yarn/cache/@typescript-eslint-type-utils-npm-5.30.0-de31c989eb-6185117638.zip rename to .yarn/cache/@typescript-eslint-type-utils-npm-5.30.5-765594d88b-080cc12317.zip index 62712097b..7a816b538 100644 Binary files a/.yarn/cache/@typescript-eslint-type-utils-npm-5.30.0-de31c989eb-6185117638.zip and b/.yarn/cache/@typescript-eslint-type-utils-npm-5.30.5-765594d88b-080cc12317.zip differ diff --git a/.yarn/cache/@typescript-eslint-types-npm-4.33.0-9e9b956afa-3baae1ca35.zip b/.yarn/cache/@typescript-eslint-types-npm-4.33.0-9e9b956afa-3baae1ca35.zip deleted file mode 100644 index a790352ff..000000000 Binary files a/.yarn/cache/@typescript-eslint-types-npm-4.33.0-9e9b956afa-3baae1ca35.zip and /dev/null differ diff --git a/.yarn/cache/@typescript-eslint-types-npm-5.30.0-1829c252da-f83a506880.zip b/.yarn/cache/@typescript-eslint-types-npm-5.30.5-77f495433c-c70420618c.zip similarity index 90% rename from .yarn/cache/@typescript-eslint-types-npm-5.30.0-1829c252da-f83a506880.zip rename to .yarn/cache/@typescript-eslint-types-npm-5.30.5-77f495433c-c70420618c.zip index c3630af0e..c71017ab5 100644 Binary files a/.yarn/cache/@typescript-eslint-types-npm-5.30.0-1829c252da-f83a506880.zip and b/.yarn/cache/@typescript-eslint-types-npm-5.30.5-77f495433c-c70420618c.zip differ diff --git a/.yarn/cache/@typescript-eslint-typescript-estree-npm-4.33.0-b6b79c10d0-2566984390.zip b/.yarn/cache/@typescript-eslint-typescript-estree-npm-4.33.0-b6b79c10d0-2566984390.zip deleted file mode 100644 index 7cc21eeab..000000000 Binary files a/.yarn/cache/@typescript-eslint-typescript-estree-npm-4.33.0-b6b79c10d0-2566984390.zip and /dev/null differ diff --git a/.yarn/cache/@typescript-eslint-typescript-estree-npm-5.30.0-7d89cc3db4-cf8caea435.zip b/.yarn/cache/@typescript-eslint-typescript-estree-npm-5.30.5-da9a64fff6-19dce426c8.zip similarity index 98% rename from .yarn/cache/@typescript-eslint-typescript-estree-npm-5.30.0-7d89cc3db4-cf8caea435.zip rename to .yarn/cache/@typescript-eslint-typescript-estree-npm-5.30.5-da9a64fff6-19dce426c8.zip index 74b774b00..5ea0d78cb 100644 Binary files a/.yarn/cache/@typescript-eslint-typescript-estree-npm-5.30.0-7d89cc3db4-cf8caea435.zip and b/.yarn/cache/@typescript-eslint-typescript-estree-npm-5.30.5-da9a64fff6-19dce426c8.zip differ diff --git a/.yarn/cache/@typescript-eslint-utils-npm-5.30.0-6128c1c85a-176eda4629.zip b/.yarn/cache/@typescript-eslint-utils-npm-5.30.5-4cd4c166a4-12f68cb34a.zip similarity index 91% rename from .yarn/cache/@typescript-eslint-utils-npm-5.30.0-6128c1c85a-176eda4629.zip rename to .yarn/cache/@typescript-eslint-utils-npm-5.30.5-4cd4c166a4-12f68cb34a.zip index 2dca4d7f2..0a0f10c4b 100644 Binary files a/.yarn/cache/@typescript-eslint-utils-npm-5.30.0-6128c1c85a-176eda4629.zip and b/.yarn/cache/@typescript-eslint-utils-npm-5.30.5-4cd4c166a4-12f68cb34a.zip differ diff --git a/.yarn/cache/@typescript-eslint-visitor-keys-npm-4.33.0-8b7e72a3c9-59953e474a.zip b/.yarn/cache/@typescript-eslint-visitor-keys-npm-4.33.0-8b7e72a3c9-59953e474a.zip deleted file mode 100644 index 80d6a78e7..000000000 Binary files a/.yarn/cache/@typescript-eslint-visitor-keys-npm-4.33.0-8b7e72a3c9-59953e474a.zip and /dev/null differ diff --git a/.yarn/cache/@typescript-eslint-visitor-keys-npm-5.30.0-1edc3a593e-bf2219cbb9.zip b/.yarn/cache/@typescript-eslint-visitor-keys-npm-5.30.5-027fead89e-c0de9ae483.zip similarity index 79% rename from .yarn/cache/@typescript-eslint-visitor-keys-npm-5.30.0-1edc3a593e-bf2219cbb9.zip rename to .yarn/cache/@typescript-eslint-visitor-keys-npm-5.30.5-027fead89e-c0de9ae483.zip index 9d484837e..42a93a37c 100644 Binary files a/.yarn/cache/@typescript-eslint-visitor-keys-npm-5.30.0-1edc3a593e-bf2219cbb9.zip and b/.yarn/cache/@typescript-eslint-visitor-keys-npm-5.30.5-027fead89e-c0de9ae483.zip differ diff --git a/.yarn/cache/app-builder-lib-npm-23.2.0-ffed6033e5-b16da4f536.zip b/.yarn/cache/app-builder-lib-npm-23.3.0-b7c238063d-c85f098d25.zip similarity index 92% rename from .yarn/cache/app-builder-lib-npm-23.2.0-ffed6033e5-b16da4f536.zip rename to .yarn/cache/app-builder-lib-npm-23.3.0-b7c238063d-c85f098d25.zip index 72d39f130..436f4913a 100644 Binary files a/.yarn/cache/app-builder-lib-npm-23.2.0-ffed6033e5-b16da4f536.zip and b/.yarn/cache/app-builder-lib-npm-23.3.0-b7c238063d-c85f098d25.zip differ diff --git a/.yarn/cache/array-union-npm-1.0.2-cc61ee268f-82cec6421b.zip b/.yarn/cache/array-union-npm-1.0.2-cc61ee268f-82cec6421b.zip deleted file mode 100644 index 9b896dc08..000000000 Binary files a/.yarn/cache/array-union-npm-1.0.2-cc61ee268f-82cec6421b.zip and /dev/null differ diff --git a/.yarn/cache/arrify-npm-2.0.1-38c408f77c-067c4c1afd.zip b/.yarn/cache/arrify-npm-2.0.1-38c408f77c-067c4c1afd.zip deleted file mode 100644 index 5dbd5402d..000000000 Binary files a/.yarn/cache/arrify-npm-2.0.1-38c408f77c-067c4c1afd.zip and /dev/null differ diff --git a/.yarn/cache/base-x-npm-1.1.0-3f30e013ed-54e24c3291.zip b/.yarn/cache/base-x-npm-1.1.0-3f30e013ed-54e24c3291.zip deleted file mode 100644 index d3f5030c0..000000000 Binary files a/.yarn/cache/base-x-npm-1.1.0-3f30e013ed-54e24c3291.zip and /dev/null differ diff --git a/.yarn/cache/bs58-npm-3.1.0-c7b525bd2a-6d757a4995.zip b/.yarn/cache/bs58-npm-3.1.0-c7b525bd2a-6d757a4995.zip deleted file mode 100644 index 9c0bb8306..000000000 Binary files a/.yarn/cache/bs58-npm-3.1.0-c7b525bd2a-6d757a4995.zip and /dev/null differ diff --git a/.yarn/cache/builder-util-npm-23.0.9-683316d6e1-14cee9beb7.zip b/.yarn/cache/builder-util-npm-23.3.0-97282db393-88cac3ff16.zip similarity index 93% rename from .yarn/cache/builder-util-npm-23.0.9-683316d6e1-14cee9beb7.zip rename to .yarn/cache/builder-util-npm-23.3.0-97282db393-88cac3ff16.zip index 46a0ec2b2..f4de67976 100644 Binary files a/.yarn/cache/builder-util-npm-23.0.9-683316d6e1-14cee9beb7.zip and b/.yarn/cache/builder-util-npm-23.3.0-97282db393-88cac3ff16.zip differ diff --git a/.yarn/cache/builder-util-runtime-npm-9.0.2-ca99d9e3e5-867768865f.zip b/.yarn/cache/builder-util-runtime-npm-9.0.3-cf3d9828d8-593a0aac49.zip similarity index 94% rename from .yarn/cache/builder-util-runtime-npm-9.0.2-ca99d9e3e5-867768865f.zip rename to .yarn/cache/builder-util-runtime-npm-9.0.3-cf3d9828d8-593a0aac49.zip index 7359d258c..b91e15e0e 100644 Binary files a/.yarn/cache/builder-util-runtime-npm-9.0.2-ca99d9e3e5-867768865f.zip and b/.yarn/cache/builder-util-runtime-npm-9.0.3-cf3d9828d8-593a0aac49.zip differ diff --git a/.yarn/cache/builtins-npm-5.0.1-6d4820dd76-66d204657f.zip b/.yarn/cache/builtins-npm-5.0.1-6d4820dd76-66d204657f.zip new file mode 100644 index 000000000..9c03e4748 Binary files /dev/null and b/.yarn/cache/builtins-npm-5.0.1-6d4820dd76-66d204657f.zip differ diff --git a/.yarn/cache/camelcase-npm-2.1.1-2ed296a336-20a3ef08f3.zip b/.yarn/cache/camelcase-npm-2.1.1-2ed296a336-20a3ef08f3.zip deleted file mode 100644 index 69472c072..000000000 Binary files a/.yarn/cache/camelcase-npm-2.1.1-2ed296a336-20a3ef08f3.zip and /dev/null differ diff --git a/.yarn/cache/cliui-npm-3.2.0-b68c4dcdcb-c68d1dbc3e.zip b/.yarn/cache/cliui-npm-3.2.0-b68c4dcdcb-c68d1dbc3e.zip deleted file mode 100644 index 8718dd72b..000000000 Binary files a/.yarn/cache/cliui-npm-3.2.0-b68c4dcdcb-c68d1dbc3e.zip and /dev/null differ diff --git a/.yarn/cache/connect-history-api-fallback-npm-2.0.0-27b00b1571-dc5368690f.zip b/.yarn/cache/connect-history-api-fallback-npm-2.0.0-27b00b1571-dc5368690f.zip new file mode 100644 index 000000000..ba60c660a Binary files /dev/null and b/.yarn/cache/connect-history-api-fallback-npm-2.0.0-27b00b1571-dc5368690f.zip differ diff --git a/.yarn/cache/cp-file-npm-7.0.0-f8cf3451db-dd60ed8d86.zip b/.yarn/cache/cp-file-npm-7.0.0-f8cf3451db-dd60ed8d86.zip deleted file mode 100644 index c1a91a892..000000000 Binary files a/.yarn/cache/cp-file-npm-7.0.0-f8cf3451db-dd60ed8d86.zip and /dev/null differ diff --git a/.yarn/cache/cpy-npm-8.1.2-1dc1b9bc19-e121f13f2b.zip b/.yarn/cache/cpy-npm-8.1.2-1dc1b9bc19-e121f13f2b.zip deleted file mode 100644 index 113cb8846..000000000 Binary files a/.yarn/cache/cpy-npm-8.1.2-1dc1b9bc19-e121f13f2b.zip and /dev/null differ diff --git a/.yarn/cache/cwd-npm-0.10.0-ccb304998d-55ab180af8.zip b/.yarn/cache/cwd-npm-0.10.0-ccb304998d-55ab180af8.zip deleted file mode 100644 index ec48ba6c8..000000000 Binary files a/.yarn/cache/cwd-npm-0.10.0-ccb304998d-55ab180af8.zip and /dev/null differ diff --git a/.yarn/cache/dir-glob-npm-2.2.2-932e08b501-3aa48714a9.zip b/.yarn/cache/dir-glob-npm-2.2.2-932e08b501-3aa48714a9.zip deleted file mode 100644 index 9b2ad619d..000000000 Binary files a/.yarn/cache/dir-glob-npm-2.2.2-932e08b501-3aa48714a9.zip and /dev/null differ diff --git a/.yarn/cache/dmg-builder-npm-23.2.0-0e7a4e1ee9-052c154733.zip b/.yarn/cache/dmg-builder-npm-23.3.0-c310f73773-942390af28.zip similarity index 96% rename from .yarn/cache/dmg-builder-npm-23.2.0-0e7a4e1ee9-052c154733.zip rename to .yarn/cache/dmg-builder-npm-23.3.0-c310f73773-942390af28.zip index c532de86c..06c5eefef 100644 Binary files a/.yarn/cache/dmg-builder-npm-23.2.0-0e7a4e1ee9-052c154733.zip and b/.yarn/cache/dmg-builder-npm-23.3.0-c310f73773-942390af28.zip differ diff --git a/.yarn/cache/electron-builder-npm-23.2.0-8903e6a4f1-73be64f3aa.zip b/.yarn/cache/electron-builder-npm-23.3.0-282385b307-77cfdf105f.zip similarity index 92% rename from .yarn/cache/electron-builder-npm-23.2.0-8903e6a4f1-73be64f3aa.zip rename to .yarn/cache/electron-builder-npm-23.3.0-282385b307-77cfdf105f.zip index 1dbe9813b..1fd93390a 100644 Binary files a/.yarn/cache/electron-builder-npm-23.2.0-8903e6a4f1-73be64f3aa.zip and b/.yarn/cache/electron-builder-npm-23.3.0-282385b307-77cfdf105f.zip differ diff --git a/.yarn/cache/electron-publish-npm-23.0.9-b94ed33c9c-866515f5ee.zip b/.yarn/cache/electron-publish-npm-23.3.0-e4a0c5ddc4-c4f8ad9344.zip similarity index 55% rename from .yarn/cache/electron-publish-npm-23.0.9-b94ed33c9c-866515f5ee.zip rename to .yarn/cache/electron-publish-npm-23.3.0-e4a0c5ddc4-c4f8ad9344.zip index f016f2bc7..b4b6ad165 100644 Binary files a/.yarn/cache/electron-publish-npm-23.0.9-b94ed33c9c-866515f5ee.zip and b/.yarn/cache/electron-publish-npm-23.3.0-e4a0c5ddc4-c4f8ad9344.zip differ diff --git a/.yarn/cache/eslint-config-standard-npm-16.0.3-9922e724a8-6ae193634f.zip b/.yarn/cache/eslint-config-standard-npm-16.0.3-9922e724a8-6ae193634f.zip deleted file mode 100644 index a1fd20016..000000000 Binary files a/.yarn/cache/eslint-config-standard-npm-16.0.3-9922e724a8-6ae193634f.zip and /dev/null differ diff --git a/.yarn/cache/eslint-config-standard-npm-17.0.0-2803f6a79a-dc0ed51e18.zip b/.yarn/cache/eslint-config-standard-npm-17.0.0-2803f6a79a-dc0ed51e18.zip new file mode 100644 index 000000000..0cb3ae760 Binary files /dev/null and b/.yarn/cache/eslint-config-standard-npm-17.0.0-2803f6a79a-dc0ed51e18.zip differ diff --git a/.yarn/cache/eslint-config-standard-with-typescript-npm-21.0.1-41f5d9a387-c3912558f2.zip b/.yarn/cache/eslint-config-standard-with-typescript-npm-21.0.1-41f5d9a387-c3912558f2.zip deleted file mode 100644 index bff9464aa..000000000 Binary files a/.yarn/cache/eslint-config-standard-with-typescript-npm-21.0.1-41f5d9a387-c3912558f2.zip and /dev/null differ diff --git a/.yarn/cache/eslint-config-standard-with-typescript-npm-22.0.0-012af209d5-4ed15e7383.zip b/.yarn/cache/eslint-config-standard-with-typescript-npm-22.0.0-012af209d5-4ed15e7383.zip new file mode 100644 index 000000000..4e18387c0 Binary files /dev/null and b/.yarn/cache/eslint-config-standard-with-typescript-npm-22.0.0-012af209d5-4ed15e7383.zip differ diff --git a/.yarn/cache/eslint-npm-8.18.0-3b3bfb190f-d9b4b7488a.zip b/.yarn/cache/eslint-npm-8.18.0-3b3bfb190f-d9b4b7488a.zip deleted file mode 100644 index 63f7bd8c2..000000000 Binary files a/.yarn/cache/eslint-npm-8.18.0-3b3bfb190f-d9b4b7488a.zip and /dev/null differ diff --git a/.yarn/cache/eslint-npm-8.19.0-147f1e0c86-0bc9df1a3a.zip b/.yarn/cache/eslint-npm-8.19.0-147f1e0c86-0bc9df1a3a.zip new file mode 100644 index 000000000..26e088b30 Binary files /dev/null and b/.yarn/cache/eslint-npm-8.19.0-147f1e0c86-0bc9df1a3a.zip differ diff --git a/.yarn/cache/eslint-plugin-es-npm-4.1.0-a4cf26d3cd-26b87a216d.zip b/.yarn/cache/eslint-plugin-es-npm-4.1.0-a4cf26d3cd-26b87a216d.zip new file mode 100644 index 000000000..e42ed00d8 Binary files /dev/null and b/.yarn/cache/eslint-plugin-es-npm-4.1.0-a4cf26d3cd-26b87a216d.zip differ diff --git a/.yarn/cache/eslint-plugin-n-npm-15.2.4-301f532bba-dd651651ab.zip b/.yarn/cache/eslint-plugin-n-npm-15.2.4-301f532bba-dd651651ab.zip new file mode 100644 index 000000000..9d7dcd97f Binary files /dev/null and b/.yarn/cache/eslint-plugin-n-npm-15.2.4-301f532bba-dd651651ab.zip differ diff --git a/.yarn/cache/eslint-plugin-promise-npm-5.2.0-bbf641e133-5d6b2d2840.zip b/.yarn/cache/eslint-plugin-promise-npm-5.2.0-bbf641e133-5d6b2d2840.zip deleted file mode 100644 index a188e9631..000000000 Binary files a/.yarn/cache/eslint-plugin-promise-npm-5.2.0-bbf641e133-5d6b2d2840.zip and /dev/null differ diff --git a/.yarn/cache/eslint-plugin-promise-npm-6.0.0-5a0de876d5-7e761507c5.zip b/.yarn/cache/eslint-plugin-promise-npm-6.0.0-5a0de876d5-7e761507c5.zip new file mode 100644 index 000000000..48b2de5c4 Binary files /dev/null and b/.yarn/cache/eslint-plugin-promise-npm-6.0.0-5a0de876d5-7e761507c5.zip differ diff --git a/.yarn/cache/expand-tilde-npm-1.2.2-9ff99585a9-18051cd104.zip b/.yarn/cache/expand-tilde-npm-1.2.2-9ff99585a9-18051cd104.zip deleted file mode 100644 index 7c13f83c3..000000000 Binary files a/.yarn/cache/expand-tilde-npm-1.2.2-9ff99585a9-18051cd104.zip and /dev/null differ diff --git a/.yarn/cache/fast-glob-npm-2.2.7-f211fb26f4-304ccff1d4.zip b/.yarn/cache/fast-glob-npm-2.2.7-f211fb26f4-304ccff1d4.zip deleted file mode 100644 index 0aa8cf2b1..000000000 Binary files a/.yarn/cache/fast-glob-npm-2.2.7-f211fb26f4-304ccff1d4.zip and /dev/null differ diff --git a/.yarn/cache/filemanager-webpack-plugin-npm-6.1.7-a0f6bbdb4a-d24ef64db7.zip b/.yarn/cache/filemanager-webpack-plugin-npm-6.1.7-a0f6bbdb4a-d24ef64db7.zip deleted file mode 100644 index 89dc8faa4..000000000 Binary files a/.yarn/cache/filemanager-webpack-plugin-npm-6.1.7-a0f6bbdb4a-d24ef64db7.zip and /dev/null differ diff --git a/.yarn/cache/filemanager-webpack-plugin-npm-7.0.0-af3636f7b6-00fc939060.zip b/.yarn/cache/filemanager-webpack-plugin-npm-7.0.0-af3636f7b6-00fc939060.zip new file mode 100644 index 000000000..53aa05c29 Binary files /dev/null and b/.yarn/cache/filemanager-webpack-plugin-npm-7.0.0-af3636f7b6-00fc939060.zip differ diff --git a/.yarn/cache/find-file-up-npm-0.1.3-69349c9744-95475fee7b.zip b/.yarn/cache/find-file-up-npm-0.1.3-69349c9744-95475fee7b.zip deleted file mode 100644 index 769b8074c..000000000 Binary files a/.yarn/cache/find-file-up-npm-0.1.3-69349c9744-95475fee7b.zip and /dev/null differ diff --git a/.yarn/cache/find-pkg-npm-0.1.2-94afd4107b-cd797bfa7d.zip b/.yarn/cache/find-pkg-npm-0.1.2-94afd4107b-cd797bfa7d.zip deleted file mode 100644 index d3a2590ed..000000000 Binary files a/.yarn/cache/find-pkg-npm-0.1.2-94afd4107b-cd797bfa7d.zip and /dev/null differ diff --git a/.yarn/cache/find-process-npm-1.4.7-e91aca1924-1953e6a16a.zip b/.yarn/cache/find-process-npm-1.4.7-e91aca1924-1953e6a16a.zip deleted file mode 100644 index 29a6fa2cb..000000000 Binary files a/.yarn/cache/find-process-npm-1.4.7-e91aca1924-1953e6a16a.zip and /dev/null differ diff --git a/.yarn/cache/fs-exists-sync-npm-0.1.0-c8642ca841-850a0d6e4c.zip b/.yarn/cache/fs-exists-sync-npm-0.1.0-c8642ca841-850a0d6e4c.zip deleted file mode 100644 index ab61ba524..000000000 Binary files a/.yarn/cache/fs-exists-sync-npm-0.1.0-c8642ca841-850a0d6e4c.zip and /dev/null differ diff --git a/.yarn/cache/glob-to-regexp-npm-0.3.0-4f55888857-d34b3219d8.zip b/.yarn/cache/glob-to-regexp-npm-0.3.0-4f55888857-d34b3219d8.zip deleted file mode 100644 index 399bb738f..000000000 Binary files a/.yarn/cache/glob-to-regexp-npm-0.3.0-4f55888857-d34b3219d8.zip and /dev/null differ diff --git a/.yarn/cache/global-modules-npm-0.2.3-36612c55cb-3801788df5.zip b/.yarn/cache/global-modules-npm-0.2.3-36612c55cb-3801788df5.zip deleted file mode 100644 index f7df1f221..000000000 Binary files a/.yarn/cache/global-modules-npm-0.2.3-36612c55cb-3801788df5.zip and /dev/null differ diff --git a/.yarn/cache/global-prefix-npm-0.1.5-1fd688e1ae-ea1b818a18.zip b/.yarn/cache/global-prefix-npm-0.1.5-1fd688e1ae-ea1b818a18.zip deleted file mode 100644 index 6ebc63419..000000000 Binary files a/.yarn/cache/global-prefix-npm-0.1.5-1fd688e1ae-ea1b818a18.zip and /dev/null differ diff --git a/.yarn/cache/globby-npm-9.2.0-686548dc5f-9b4cb70aa0.zip b/.yarn/cache/globby-npm-9.2.0-686548dc5f-9b4cb70aa0.zip deleted file mode 100644 index de3770ca0..000000000 Binary files a/.yarn/cache/globby-npm-9.2.0-686548dc5f-9b4cb70aa0.zip and /dev/null differ diff --git a/.yarn/cache/has-glob-npm-1.0.0-a2151352c8-cafad93e59.zip b/.yarn/cache/has-glob-npm-1.0.0-a2151352c8-cafad93e59.zip deleted file mode 100644 index 3b286987b..000000000 Binary files a/.yarn/cache/has-glob-npm-1.0.0-a2151352c8-cafad93e59.zip and /dev/null differ diff --git a/.yarn/cache/homedir-polyfill-npm-1.0.3-da1a29ce00-18dd4db870.zip b/.yarn/cache/homedir-polyfill-npm-1.0.3-da1a29ce00-18dd4db870.zip deleted file mode 100644 index fca8dc7f4..000000000 Binary files a/.yarn/cache/homedir-polyfill-npm-1.0.3-da1a29ce00-18dd4db870.zip and /dev/null differ diff --git a/.yarn/cache/i18next-npm-21.8.11-2a68e1411d-94b9de3811.zip b/.yarn/cache/i18next-npm-21.8.11-2a68e1411d-94b9de3811.zip deleted file mode 100644 index 00b9ae5b5..000000000 Binary files a/.yarn/cache/i18next-npm-21.8.11-2a68e1411d-94b9de3811.zip and /dev/null differ diff --git a/.yarn/cache/i18next-npm-21.8.13-8ed1812650-0cc9a52133.zip b/.yarn/cache/i18next-npm-21.8.13-8ed1812650-0cc9a52133.zip new file mode 100644 index 000000000..a32dad04e Binary files /dev/null and b/.yarn/cache/i18next-npm-21.8.13-8ed1812650-0cc9a52133.zip differ diff --git a/.yarn/cache/ignore-npm-4.0.6-66c0d6543e-248f82e50a.zip b/.yarn/cache/ignore-npm-4.0.6-66c0d6543e-248f82e50a.zip deleted file mode 100644 index f5bcbcf28..000000000 Binary files a/.yarn/cache/ignore-npm-4.0.6-66c0d6543e-248f82e50a.zip and /dev/null differ diff --git a/.yarn/cache/invert-kv-npm-1.0.0-114e48e289-aebeee31dd.zip b/.yarn/cache/invert-kv-npm-1.0.0-114e48e289-aebeee31dd.zip deleted file mode 100644 index cf4053ea9..000000000 Binary files a/.yarn/cache/invert-kv-npm-1.0.0-114e48e289-aebeee31dd.zip and /dev/null differ diff --git a/.yarn/cache/is-windows-npm-0.2.0-32c20e83a7-3df25afda2.zip b/.yarn/cache/is-windows-npm-0.2.0-32c20e83a7-3df25afda2.zip deleted file mode 100644 index 5ea52e6d4..000000000 Binary files a/.yarn/cache/is-windows-npm-0.2.0-32c20e83a7-3df25afda2.zip and /dev/null differ diff --git a/.yarn/cache/jest-dev-server-npm-6.0.3-f5a7626f26-58b41b3c39.zip b/.yarn/cache/jest-dev-server-npm-6.0.3-f5a7626f26-58b41b3c39.zip deleted file mode 100644 index fd8083918..000000000 Binary files a/.yarn/cache/jest-dev-server-npm-6.0.3-f5a7626f26-58b41b3c39.zip and /dev/null differ diff --git a/.yarn/cache/junk-npm-3.1.0-aa1fa701c6-6c4d68e8f8.zip b/.yarn/cache/junk-npm-3.1.0-aa1fa701c6-6c4d68e8f8.zip deleted file mode 100644 index 4e80d3e19..000000000 Binary files a/.yarn/cache/junk-npm-3.1.0-aa1fa701c6-6c4d68e8f8.zip and /dev/null differ diff --git a/.yarn/cache/lcid-npm-1.0.0-02d845072b-e8c7a4db07.zip b/.yarn/cache/lcid-npm-1.0.0-02d845072b-e8c7a4db07.zip deleted file mode 100644 index cf100d82b..000000000 Binary files a/.yarn/cache/lcid-npm-1.0.0-02d845072b-e8c7a4db07.zip and /dev/null differ diff --git a/.yarn/cache/nconf-npm-0.8.5-3db739b5ad-5f51381c7f.zip b/.yarn/cache/nconf-npm-0.8.5-3db739b5ad-5f51381c7f.zip deleted file mode 100644 index 5fb394cdd..000000000 Binary files a/.yarn/cache/nconf-npm-0.8.5-3db739b5ad-5f51381c7f.zip and /dev/null differ diff --git a/.yarn/cache/nested-error-stacks-npm-2.1.1-0b1da05af0-5f452fad75.zip b/.yarn/cache/nested-error-stacks-npm-2.1.1-0b1da05af0-5f452fad75.zip deleted file mode 100644 index ec5bdf17d..000000000 Binary files a/.yarn/cache/nested-error-stacks-npm-2.1.1-0b1da05af0-5f452fad75.zip and /dev/null differ diff --git a/.yarn/cache/os-homedir-npm-1.0.2-01f82faa88-af609f5a7a.zip b/.yarn/cache/os-homedir-npm-1.0.2-01f82faa88-af609f5a7a.zip deleted file mode 100644 index 0d88db41e..000000000 Binary files a/.yarn/cache/os-homedir-npm-1.0.2-01f82faa88-af609f5a7a.zip and /dev/null differ diff --git a/.yarn/cache/os-locale-npm-1.4.0-924760b837-0161a1b6b5.zip b/.yarn/cache/os-locale-npm-1.4.0-924760b837-0161a1b6b5.zip deleted file mode 100644 index 4d11aa366..000000000 Binary files a/.yarn/cache/os-locale-npm-1.4.0-924760b837-0161a1b6b5.zip and /dev/null differ diff --git a/.yarn/cache/p-all-npm-2.1.0-af954bf089-6c20134eb3.zip b/.yarn/cache/p-all-npm-2.1.0-af954bf089-6c20134eb3.zip deleted file mode 100644 index 5cc9f5d86..000000000 Binary files a/.yarn/cache/p-all-npm-2.1.0-af954bf089-6c20134eb3.zip and /dev/null differ diff --git a/.yarn/cache/p-event-npm-4.2.0-1d17e9941e-8a3588f7a8.zip b/.yarn/cache/p-event-npm-4.2.0-1d17e9941e-8a3588f7a8.zip deleted file mode 100644 index 5cb9fdd75..000000000 Binary files a/.yarn/cache/p-event-npm-4.2.0-1d17e9941e-8a3588f7a8.zip and /dev/null differ diff --git a/.yarn/cache/p-filter-npm-2.1.0-f1136c698e-76e552ca62.zip b/.yarn/cache/p-filter-npm-2.1.0-f1136c698e-76e552ca62.zip deleted file mode 100644 index f502db5c2..000000000 Binary files a/.yarn/cache/p-filter-npm-2.1.0-f1136c698e-76e552ca62.zip and /dev/null differ diff --git a/.yarn/cache/p-map-npm-2.1.0-d9e865dc7c-9e3ad3c9f6.zip b/.yarn/cache/p-map-npm-2.1.0-d9e865dc7c-9e3ad3c9f6.zip deleted file mode 100644 index 67932a068..000000000 Binary files a/.yarn/cache/p-map-npm-2.1.0-d9e865dc7c-9e3ad3c9f6.zip and /dev/null differ diff --git a/.yarn/cache/p-map-npm-3.0.0-e4f17c4167-49b0fcbc66.zip b/.yarn/cache/p-map-npm-3.0.0-e4f17c4167-49b0fcbc66.zip deleted file mode 100644 index cb604862f..000000000 Binary files a/.yarn/cache/p-map-npm-3.0.0-e4f17c4167-49b0fcbc66.zip and /dev/null differ diff --git a/.yarn/cache/p-timeout-npm-3.2.0-7fdb33f733-3dd0eaa048.zip b/.yarn/cache/p-timeout-npm-3.2.0-7fdb33f733-3dd0eaa048.zip deleted file mode 100644 index eaf8f71c7..000000000 Binary files a/.yarn/cache/p-timeout-npm-3.2.0-7fdb33f733-3dd0eaa048.zip and /dev/null differ diff --git a/.yarn/cache/parse-passwd-npm-1.0.0-ace6effa1d-4e55e0231d.zip b/.yarn/cache/parse-passwd-npm-1.0.0-ace6effa1d-4e55e0231d.zip deleted file mode 100644 index 8181edb9b..000000000 Binary files a/.yarn/cache/parse-passwd-npm-1.0.0-ace6effa1d-4e55e0231d.zip and /dev/null differ diff --git a/.yarn/cache/react-i18next-npm-11.17.3-55b193ecac-6b4959eb62.zip b/.yarn/cache/react-i18next-npm-11.17.3-55b193ecac-6b4959eb62.zip deleted file mode 100644 index 20e06c338..000000000 Binary files a/.yarn/cache/react-i18next-npm-11.17.3-55b193ecac-6b4959eb62.zip and /dev/null differ diff --git a/.yarn/cache/react-i18next-npm-11.18.0-1bd3def65c-63bf023c49.zip b/.yarn/cache/react-i18next-npm-11.18.0-1bd3def65c-63bf023c49.zip new file mode 100644 index 000000000..4e3dd65ed Binary files /dev/null and b/.yarn/cache/react-i18next-npm-11.18.0-1bd3def65c-63bf023c49.zip differ diff --git a/.yarn/cache/react-refresh-typescript-npm-2.0.5-d904a69875-04b1765280.zip b/.yarn/cache/react-refresh-typescript-npm-2.0.5-d904a69875-04b1765280.zip deleted file mode 100644 index 79a901c85..000000000 Binary files a/.yarn/cache/react-refresh-typescript-npm-2.0.5-d904a69875-04b1765280.zip and /dev/null differ diff --git a/.yarn/cache/react-refresh-typescript-npm-2.0.7-84efedb58d-a95a44f7e7.zip b/.yarn/cache/react-refresh-typescript-npm-2.0.7-84efedb58d-a95a44f7e7.zip new file mode 100644 index 000000000..3a5e18228 Binary files /dev/null and b/.yarn/cache/react-refresh-typescript-npm-2.0.7-84efedb58d-a95a44f7e7.zip differ diff --git a/.yarn/cache/resolve-dir-npm-0.1.1-0debfbb554-cc3e188593.zip b/.yarn/cache/resolve-dir-npm-0.1.1-0debfbb554-cc3e188593.zip deleted file mode 100644 index eb1c8d4ad..000000000 Binary files a/.yarn/cache/resolve-dir-npm-0.1.1-0debfbb554-cc3e188593.zip and /dev/null differ diff --git a/.yarn/cache/slash-npm-2.0.0-69009eac54-512d435073.zip b/.yarn/cache/slash-npm-2.0.0-69009eac54-512d435073.zip deleted file mode 100644 index 39c8b1359..000000000 Binary files a/.yarn/cache/slash-npm-2.0.0-69009eac54-512d435073.zip and /dev/null differ diff --git a/.yarn/cache/sleep-npm-6.1.0-aadd96e7a0-607465cdcf.zip b/.yarn/cache/sleep-npm-6.1.0-aadd96e7a0-607465cdcf.zip new file mode 100644 index 000000000..dd16e2bbb Binary files /dev/null and b/.yarn/cache/sleep-npm-6.1.0-aadd96e7a0-607465cdcf.zip differ diff --git a/.yarn/cache/spawnd-npm-6.0.2-b6fc1c6559-39060a101e.zip b/.yarn/cache/spawnd-npm-6.0.2-b6fc1c6559-39060a101e.zip deleted file mode 100644 index e5d9e0763..000000000 Binary files a/.yarn/cache/spawnd-npm-6.0.2-b6fc1c6559-39060a101e.zip and /dev/null differ diff --git a/.yarn/cache/superagent-npm-7.1.5-b82717b291-4782ca099b.zip b/.yarn/cache/superagent-npm-7.1.5-b82717b291-4782ca099b.zip deleted file mode 100644 index 7f816b688..000000000 Binary files a/.yarn/cache/superagent-npm-7.1.5-b82717b291-4782ca099b.zip and /dev/null differ diff --git a/.yarn/cache/superagent-npm-8.0.0-23cfd5d736-14343e5932.zip b/.yarn/cache/superagent-npm-8.0.0-23cfd5d736-14343e5932.zip new file mode 100644 index 000000000..b46b21881 Binary files /dev/null and b/.yarn/cache/superagent-npm-8.0.0-23cfd5d736-14343e5932.zip differ diff --git a/.yarn/cache/supertest-npm-6.2.3-280f27cd97-c1bed86c31.zip b/.yarn/cache/supertest-npm-6.2.4-6901daa695-f2ddc4f3ba.zip similarity index 77% rename from .yarn/cache/supertest-npm-6.2.3-280f27cd97-c1bed86c31.zip rename to .yarn/cache/supertest-npm-6.2.4-6901daa695-f2ddc4f3ba.zip index e8eea9a8a..db0428a64 100644 Binary files a/.yarn/cache/supertest-npm-6.2.3-280f27cd97-c1bed86c31.zip and b/.yarn/cache/supertest-npm-6.2.4-6901daa695-f2ddc4f3ba.zip differ diff --git a/.yarn/cache/ts-node-npm-10.8.2-f3c0c9eaee-1eede939be.zip b/.yarn/cache/ts-node-npm-10.8.2-f3c0c9eaee-1eede939be.zip new file mode 100644 index 000000000..61b690272 Binary files /dev/null and b/.yarn/cache/ts-node-npm-10.8.2-f3c0c9eaee-1eede939be.zip differ diff --git a/.yarn/cache/webpack-dev-server-npm-4.9.3-8ff96f1b16-845f2cc8e7.zip b/.yarn/cache/webpack-dev-server-npm-4.9.3-8ff96f1b16-845f2cc8e7.zip new file mode 100644 index 000000000..2e4515dd2 Binary files /dev/null and b/.yarn/cache/webpack-dev-server-npm-4.9.3-8ff96f1b16-845f2cc8e7.zip differ diff --git a/.yarn/cache/window-size-npm-0.1.4-6c180982b5-409accca0b.zip b/.yarn/cache/window-size-npm-0.1.4-6c180982b5-409accca0b.zip deleted file mode 100644 index af868837c..000000000 Binary files a/.yarn/cache/window-size-npm-0.1.4-6c180982b5-409accca0b.zip and /dev/null differ diff --git a/.yarn/cache/wrap-ansi-npm-2.1.0-1fd9d50973-2dacd4b363.zip b/.yarn/cache/wrap-ansi-npm-2.1.0-1fd9d50973-2dacd4b363.zip deleted file mode 100644 index da0cd5ebe..000000000 Binary files a/.yarn/cache/wrap-ansi-npm-2.1.0-1fd9d50973-2dacd4b363.zip and /dev/null differ diff --git a/.yarn/cache/xvfb-npm-0.4.0-70ab42b6f9-d8c446dd6e.zip b/.yarn/cache/xvfb-npm-0.4.0-70ab42b6f9-d8c446dd6e.zip new file mode 100644 index 000000000..4e36d9d54 Binary files /dev/null and b/.yarn/cache/xvfb-npm-0.4.0-70ab42b6f9-d8c446dd6e.zip differ diff --git a/.yarn/cache/y18n-npm-3.2.2-f9b6b42101-6154fd7544.zip b/.yarn/cache/y18n-npm-3.2.2-f9b6b42101-6154fd7544.zip deleted file mode 100644 index 95d740e86..000000000 Binary files a/.yarn/cache/y18n-npm-3.2.2-f9b6b42101-6154fd7544.zip and /dev/null differ diff --git a/.yarn/cache/yargs-npm-3.32.0-e5d4941694-3e0f7fc1bc.zip b/.yarn/cache/yargs-npm-3.32.0-e5d4941694-3e0f7fc1bc.zip deleted file mode 100644 index f4a7a0cfb..000000000 Binary files a/.yarn/cache/yargs-npm-3.32.0-e5d4941694-3e0f7fc1bc.zip and /dev/null differ diff --git a/docker/guardoni/Dockerfile b/docker/guardoni/Dockerfile new file mode 100644 index 000000000..53cec7238 --- /dev/null +++ b/docker/guardoni/Dockerfile @@ -0,0 +1,31 @@ +FROM node:16-slim + +RUN mkdir /guardoni +WORKDIR /guardoni +RUN apt-get update +RUN apt-get install -y python3 build-essential libcairo2-dev libpango1.0-dev libjpeg-dev libgif-dev librsvg2-dev + +ADD ./ ./ + +RUN yarn install +RUN yarn tk:ext build +RUN yarn yt:ext build +RUN yarn guardoni build:cli +RUN yarn guardoni pkg +WORKDIR /guardoni/platforms/guardoni/dist/ +RUN mv $(ls -1 guardoni-cli*-linux) guardoni-cli + +FROM selenium/standalone-chrome:102.0 +WORKDIR /guardoni + +COPY --from=0 /guardoni/platforms/guardoni/dist/guardoni-cli guardoni-cli +COPY --from=0 "/guardoni/node_modules/@tktrex/extension/build" tktrex +COPY --from=0 "/guardoni/node_modules/@yttrex/extension/build" yttrex +COPY --from=0 /guardoni/docker/guardoni/guardoni.config.json . + +USER root +RUN chmod -R 777 /guardoni/yttrex +RUN chmod -R 777 /guardoni/tktrex +USER seluser + +ENTRYPOINT [ "/guardoni/guardoni-cli" ] diff --git a/docker/guardoni/build.sh b/docker/guardoni/build.sh new file mode 100755 index 000000000..8d8ab33c0 --- /dev/null +++ b/docker/guardoni/build.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -e + +SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]:-$0}"; )" &> /dev/null && pwd 2> /dev/null; )" +cd "$SCRIPT_DIR/../../" +docker build -f "docker/guardoni/Dockerfile" -t guardoni . diff --git a/docker/guardoni/guardoni.config.json b/docker/guardoni/guardoni.config.json new file mode 100644 index 000000000..e724f96c6 --- /dev/null +++ b/docker/guardoni/guardoni.config.json @@ -0,0 +1,17 @@ +{ + "profileName": "dockerized-guardoni", + "headless": true, + "verbose": false, + "loadFor": 3000, + "yt": { + "name": "youtube", + "backend": "https://youtube.tracking.exposed/api", + "extensionDir": "/guardoni/yttrex" + }, + "tk": { + "name": "tiktok", + "backend": "https://tiktok.tracking.exposed/api", + "extensionDir": "/guardoni/tktrex" + }, + "chromePath": "/usr/bin/google-chrome" +} diff --git a/docs/package.json b/docs/package.json index 953073d00..10efc5446 100644 --- a/docs/package.json +++ b/docs/package.json @@ -27,7 +27,7 @@ "@tsconfig/docusaurus": "^1.0.4", "docusaurus-plugin-openapi": "^0.5.0", "docusaurus-preset-openapi": "^0.5.0", - "typescript": "^4.7.2" + "typescript": "^4.7.4" }, "browserslist": { "production": [ diff --git a/package.json b/package.json index adb96f59e..671160444 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,6 @@ "packages/*", "platforms/guardoni", "platforms/ycai/studio", - "platforms/makhno/*", "platforms/yttrex/*", "platforms/tktrex/docs", "platforms/tktrex/extension", @@ -26,7 +25,7 @@ "clean": "yarn workspaces foreach -v run clean", "lint": "yarn workspaces foreach -v run lint", "type-check": "yarn workspaces foreach run tsc --noEmit", - "build": "yarn workspaces foreach run build", + "build": "scripts/build.sh", "test": "jest", "docs": "yarn workspace @trex/docs", "taboule": "yarn workspace @trex/taboule", @@ -40,7 +39,6 @@ "yt:shared": "yarn workspace @yttrex/shared", "yt:backend": "yarn workspace @yttrex/backend", "yt:ext": "yarn workspace @yttrex/extension", - "makhno": "yarn workspace @makhno/backend", "ycai": "yarn workspace @trex/ycai", "lint-staged": "lint-staged", "postinstall": "husky install", @@ -57,16 +55,16 @@ "@release-it/conventional-changelog": "^4.3.0", "@types/node": "^16.11.36", "@types/prettier": "^2.6.3", - "eslint": "^8.16.0", + "eslint": "^8.19.0", + "eslint-plugin-n": "^15.2.4", "husky": "^7.0.4", "jest": "^27.5.1", - "jest-dev-server": "^6.0.3", "lint-staged": "^12.3.3", - "prettier": "^2.6.2", + "prettier": "^2.7.1", "release-it": "^14.14.3", "release-it-yarn-workspaces": "^2.0.1", "ts-jest": "^27.1.5", - "typescript": "^4.7.2", + "typescript": "^4.7.4", "webpack": "^5.73.0", "zx": "^6.2.5" }, diff --git a/packages/shared/package.json b/packages/shared/package.json index e22174fa2..5e0f041f5 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -24,12 +24,12 @@ "io-ts-types": "^0.5.16", "react": "^17.0.2", "react-dom": "^17.0.2", - "react-i18next": "^11.15.3", + "react-i18next": "^11.18.0", "ts-endpoint": "^2.0.0", "ts-endpoint-express": "^2.0.0", "ts-io-error": "^2.0.0", "typelevel-ts": "^0.4.0", - "typescript": "^4.7.2" + "typescript": "^4.7.4" }, "devDependencies": { "@pmmmwh/react-refresh-webpack-plugin": "^0.5.7", @@ -37,31 +37,31 @@ "@types/debug": "^4.1.7", "@types/dotenv-webpack": "^7.0.3", "@types/node": "^16.11.36", - "@types/react": "^17.0.45", + "@types/react": "^17.0.47", "@types/react-dom": "^17.0.17", "@types/webpack-bundle-analyzer": "^4.4.1", - "@typescript-eslint/eslint-plugin": "^5.27.0", - "@typescript-eslint/parser": "^5.27.0", + "@typescript-eslint/eslint-plugin": "^5.30.5", + "@typescript-eslint/parser": "^5.30.5", "copy-webpack-plugin": "^11.0.0", "dotenv-webpack": "^7.1.0", - "eslint": "^8.16.0", + "eslint": "^8.19.0", "eslint-config-prettier": "^8.5.0", - "eslint-config-standard-with-typescript": "^21.0.1", + "eslint-config-standard-with-typescript": "^22.0.0", "eslint-plugin-import": "^2.26.0", "eslint-plugin-node": "^11.1.0", - "eslint-plugin-promise": "^5.2.0", - "eslint-plugin-react": "^7.30.0", + "eslint-plugin-promise": "^6.0.0", + "eslint-plugin-react": "^7.30.1", "fast-check": "^2.25.0", "fast-check-io-ts": "^0.5.0", - "filemanager-webpack-plugin": "^6.1.7", + "filemanager-webpack-plugin": "^7.0.0", "html-webpack-plugin": "^5.5.0", "jest": "^27.5.1", "mini-css-extract-plugin": "^2.6.1", "node-loader": "^2.0.0", "node-sass": "^7.0.1", - "prettier": "^2.6.2", + "prettier": "^2.7.1", "react-refresh": "^0.12.0", - "react-refresh-typescript": "^2.0.5", + "react-refresh-typescript": "^2.0.7", "sass-loader": "^12.6.0", "style-loader": "^3.3.1", "tsconfig-paths-webpack-plugin": "^3.5.2", diff --git a/packages/shared/src/backend/utils/endpoint.ts b/packages/shared/src/backend/utils/endpoint.ts index 58531fe42..1bad585ff 100644 --- a/packages/shared/src/backend/utils/endpoint.ts +++ b/packages/shared/src/backend/utils/endpoint.ts @@ -37,7 +37,7 @@ const toError = (e: unknown): IOError => { }; }; -declare type UndefinedOrRuntime = N extends RecordCodec +export declare type UndefinedOrRuntime = N extends RecordCodec ? { [k in keyof N['props']]: runtimeType; } diff --git a/packages/shared/src/backend/utils/routeHandlerMiddleware.ts b/packages/shared/src/backend/utils/routeHandlerMiddleware.ts new file mode 100644 index 000000000..a82b4031b --- /dev/null +++ b/packages/shared/src/backend/utils/routeHandlerMiddleware.ts @@ -0,0 +1,131 @@ +import express from 'express'; +import _ from 'lodash'; +import { APIError } from '../../errors/APIError'; +import { GetLogger } from '../../logger'; + +const logger = GetLogger('route-handler'); + +const logAPICount = { requests: {}, responses: {}, errors: {} }; + +function loginc(kind: string, fname: string): void { + (logAPICount as any)[kind][fname] = (logAPICount as any)[kind][fname] + ? (logAPICount as any)[kind][fname]++ + : 1; +} + +type RouteHandlerMiddleware> = + ( + fname: keyof R + ) => (req: express.Request, res: express.Response) => Promise; + +export const routeHandleMiddleware = < + R extends Record +>( + apiList: R +): RouteHandlerMiddleware => { + setInterval(() => { + let print = false; + _.each(_.keys(logAPICount), function (k) { + if (!_.keys((logAPICount as any)[k]).length) + // eslint-disable-next-line @typescript-eslint/no-dynamic-delete + delete (logAPICount as any)[k]; + else print = true; + }); + if (print) logger.debug('%j', logAPICount); + logAPICount.responses = {}; + logAPICount.errors = {}; + logAPICount.requests = {}; + }, 60 * 1000); + + return (fn) => async (req, res) => { + const fname = fn as any; + try { + loginc('requests', fname); + const funct = apiList[fname]; + const httpresult = await (funct as any)(req, res); + + if (!httpresult) { + logger.debug("API (%s) didn't return anything!?", fname); + loginc('errors', fname); + res.send('Fatal error: Invalid output'); + res.status(501); + return; + } + + if (httpresult.json?.error) { + const statusCode = httpresult.json.status ?? 500; + logger.debug('API (%s) failure, returning %d', fname, statusCode); + loginc('errors', fname); + res.status(statusCode); + res.json(httpresult.json); + return; + } + + if (httpresult.headers) + _.each(httpresult.headers, function (value, key) { + logger.debug('Setting header %s: %s', key, value); + res.setHeader(key, value); + }); + + if (httpresult.json) { + // logger("API (%s) success, returning %d bytes JSON", fname, _.size(JSON.stringify(httpresult.json))); + loginc('responses', fname); + res.setHeader('Access-Control-Allow-Origin', '*'); + res.json(httpresult.json); + } else if (httpresult.text) { + // logger("API (%s) success, returning text (size %d)", fname, _.size(httpresult.text)); + loginc('responses', fname); + res.send(httpresult.text); + } else if (httpresult.status) { + // logger("Returning empty status %d from API (%s)", httpresult.status, fname); + loginc('responses', fname); + res.status(httpresult.status); + } else { + logger.debug( + 'Undetermined failure in API (%s) → %j', + fname, + httpresult + ); + loginc('errors', fname); + res.status(502); + res.send('Error?'); + } + } catch (error) { + logger.error('Route handler (%s) error: %O', fname, error); + + // todo: this should be 500 + res.status(502); + + if (error instanceof APIError) { + logger.error( + 'APIError - %s: (%s) %s %s', + error.name, + error.message, + error.stack + ); + res.status(error.status); + res.send({ + name: error.type, + message: error.message, + details: error.details, + }); + } else if (error instanceof Error) { + logger.error('Error - %s: %s', error.name, error.message); + + res.send('Software error: ' + error.message); + logger.error( + 'Error in HTTP handler API(%s): %s %s', + fname, + error.message, + error.stack + ); + } else { + res.status(502); + res.send('Software error: ' + (error as any).message); + loginc('errors', fname); + logger.debug('Error in HTTP handler API(%s): %o', fname, error); + } + } + res.end(); + }; +}; diff --git a/platforms/yttrex/backend/lib/endpoint.ts b/packages/shared/src/endpoints/helper.ts similarity index 54% rename from platforms/yttrex/backend/lib/endpoint.ts rename to packages/shared/src/endpoints/helper.ts index d0d13cfc0..53088ea24 100644 --- a/platforms/yttrex/backend/lib/endpoint.ts +++ b/packages/shared/src/endpoints/helper.ts @@ -5,24 +5,30 @@ import * as t from 'io-ts'; import { PathReporter } from 'io-ts/lib/PathReporter'; import { InferEndpointInstanceParams, - MinimalEndpointInstance, -} from '@shared/endpoints'; + UndefinedOrRuntime, +} from '../backend/utils/endpoint'; import { serializedType } from 'ts-io-error/lib/Codec'; +import { MinimalEndpointInstance } from './MinimalEndpoint'; +import { APIError } from '../errors/APIError'; + +interface DecodeError { + type: 'error'; + result: string[]; +} + +interface DecodeRequestSuccess { + type: 'success'; + result: { + params: UndefinedOrRuntime['params']>; + query: UndefinedOrRuntime['query']>; + headers: UndefinedOrRuntime['query']>; + body: UndefinedOrRuntime['body']>; + }; +} type DecodeRequestResult = - | { - type: 'error'; - result: string[]; - } - | { - type: 'success'; - result: { - params: InferEndpointInstanceParams['params']; - query: InferEndpointInstanceParams['query']; - headers: InferEndpointInstanceParams['query']; - body: InferEndpointInstanceParams['body']; - }; - }; + | DecodeError + | DecodeRequestSuccess; const decodeRequest = ( e: E, @@ -45,15 +51,14 @@ const decodeRequest = ( ); }; +interface DecodeResponseSuccess { + type: 'success'; + result: serializedType; +} + type DecodeResponseResult = - | { - type: 'error'; - result: string[]; - } - | { - type: 'success'; - result: serializedType; - }; + | DecodeError + | DecodeResponseSuccess; const decodeResponse = ( e: E, @@ -71,4 +76,17 @@ const decodeResponse = ( ); }; -export { decodeRequest, decodeResponse }; +const decodeOrThrowRequest = ( + e: E, + r: unknown +): DecodeRequestSuccess['result'] => { + return pipe(decodeRequest(e, r), (r) => { + if (r.type === 'error') { + throw new APIError(400, 'Bad Request', 'Request decode failed', r.result); + } + + return r.result; + }); +}; + +export { decodeRequest, decodeResponse, decodeOrThrowRequest }; diff --git a/packages/shared/src/errors/APIError.ts b/packages/shared/src/errors/APIError.ts index 3f655c4b0..f79176489 100644 --- a/packages/shared/src/errors/APIError.ts +++ b/packages/shared/src/errors/APIError.ts @@ -2,12 +2,15 @@ export const isAPIError = (e: unknown): e is APIError => { return (e as any).name === 'APIError'; }; -export class APIError { +export class APIError extends Error { public readonly name = 'APIError'; constructor( + public readonly status: number, public readonly type: string, public readonly message: string, public readonly details: string[] - ) {} + ) { + super(message); + } } diff --git a/packages/shared/src/errors/ValidationError.ts b/packages/shared/src/errors/ValidationError.ts index 14130aad7..e996b7833 100644 --- a/packages/shared/src/errors/ValidationError.ts +++ b/packages/shared/src/errors/ValidationError.ts @@ -4,7 +4,7 @@ export const toValidationError = ( message: string, details: string[] ): APIError => { - return new APIError('ValidationError', message, details); + return new APIError(400, 'ValidationError', message, details); }; export const isValidationError = ( diff --git a/packages/shared/src/extension/app.ts b/packages/shared/src/extension/app.ts index 360707d2a..c226f39cd 100644 --- a/packages/shared/src/extension/app.ts +++ b/packages/shared/src/extension/app.ts @@ -2,13 +2,13 @@ import { debounce } from '@material-ui/core'; import _ from 'lodash'; import { clearCache } from '../providers/dataDonation.provider'; import { localLookup, serverLookup } from './chrome/background/sendMessage'; -import { Config } from './config'; import * as dom from './dom'; import { registerHandlers } from './handlers/index'; import { Hub } from './hub'; import log from './logger'; import HubEvent from './models/HubEvent'; import { ServerLookup } from './models/Message'; +import UserSettings from './models/UserSettings'; // instantiate a proper logger const appLog = log.extend('app'); @@ -18,7 +18,8 @@ export interface BaseObserverHandler { handle: ( n: HTMLElement, opts: Omit, - selectorName: string + selectorName: string, + s: UserSettings ) => void; } @@ -58,13 +59,16 @@ interface SetupObserverOpts { onLocationChange: (oldLocation: string, newLocation: string) => void; } -interface BootOpts { +export interface BootOpts { payload: ServerLookup['payload']; - mapLocalConfig: (c: Config, payload: ServerLookup['payload']) => Config; + mapLocalConfig: ( + c: UserSettings, + payload: ServerLookup['payload'] + ) => UserSettings; observe: SetupObserverOpts; hub: { hub: Hub; - onRegister: (h: Hub, config: Config) => void; + onRegister: (h: Hub, config: UserSettings) => void; }; onAuthenticated: (res: any) => void; } @@ -84,20 +88,25 @@ let oldHref: string; * setup the mutation observer with * given callbacks for selectors */ -function setupObserver({ - handlers, - platformMatch, - onLocationChange, -}: SetupObserverOpts): MutationObserver { +function setupObserver( + { handlers, platformMatch, onLocationChange }: SetupObserverOpts, + config: UserSettings +): MutationObserver { const handlersList = Object.keys(handlers); // register selector and 'selector-with-parents' handlers handlersList.forEach((h) => { const { handle, ...handler } = handlers[h]; + if ( handler.match.type === 'selector' || handler.match.type === 'selector-with-parents' ) { - dom.on(handler.match.selector, (node) => handle(node, handler, h)); + dom.on(handler.match.selector, (node) => + handle(node, handler, h, { + ...config, + href: window.location.toString(), + } as any) + ); } }); @@ -126,6 +135,7 @@ function setupObserver({ oldHref = newHref; } + // always call the route handler const routeHandlerKey = handlersList.find((h) => { const handler = handlers[h]; @@ -137,15 +147,20 @@ function setupObserver({ if (routeHandlerKey) { appLog.debug('Route handler key %s', routeHandlerKey); - const { handle, ...routeHandlerOpts } = handlers[routeHandlerKey]; - handle(window.document.body, routeHandlerOpts, routeHandlerKey); + const { handle, ...routeHandlerOpts } = handlers[ + routeHandlerKey + ] as RouteObserverHandler; + handle(window.document.body, routeHandlerOpts, routeHandlerKey, { + ...config, + href: window.location.toString(), + } as any); } } } - }, 300) + }, 500) ); - const config = { + const observerConfig = { childList: true, subtree: true, }; @@ -156,7 +171,7 @@ function setupObserver({ }); if (body) { - observer.observe(body, config); + observer.observe(body, observerConfig); } else { appLog.error('setupObserver: body not found'); } @@ -169,6 +184,7 @@ function setupObserver({ */ interface App { + config: UserSettings; destroy: () => void; } @@ -195,6 +211,7 @@ export async function boot(opts: BootOpts): Promise { if (!settings.active) { appLog.info('extension disabled!'); return resolve({ + config: settings, destroy: () => { opts.hub.hub.clear(); }, @@ -226,10 +243,11 @@ export async function boot(opts: BootOpts): Promise { if (response.type === 'Error') { throw response.error; } - const observer = setupObserver(opts.observe); + const observer = setupObserver(opts.observe, config); opts.onAuthenticated(response.result); const context = { + config, destroy: () => { opts.hub.hub.clear(); observer.disconnect(); diff --git a/packages/shared/src/extension/chrome/api.ts b/packages/shared/src/extension/chrome/api.ts deleted file mode 100644 index a93285c8c..000000000 --- a/packages/shared/src/extension/chrome/api.ts +++ /dev/null @@ -1,65 +0,0 @@ -import nacl from 'tweetnacl'; -import bs58 from 'bs58'; - -import config from '../config'; -import UserSettings from '../models/UserSettings'; -import { decodeString, decodeFromBase58 } from '../../utils/decode.utils'; -import db from './db'; - -const post = - (path: string) => - (data: Record, cookieId: string): Promise => { - return new Promise((resolve, reject) => { - db.getValid(UserSettings)('local') - .then((settings) => { - const xhr = new XMLHttpRequest(); - const payload = JSON.stringify({ - ...data, - researchTag: settings.researchTag, - }); - const url = `${config.API_ROOT}/${path}`; - - xhr.open('POST', url, true); - - xhr.setRequestHeader('Content-Type', 'application/json'); - xhr.setRequestHeader('X-tktrex-Version', config.VERSION); - xhr.setRequestHeader('X-tktrex-Build', config.BUILD); - - const signature = nacl.sign.detached( - decodeString(payload), - decodeFromBase58(settings.secretKey) - ); - - xhr.setRequestHeader('X-tktrex-NonAuthCookieId', cookieId); - xhr.setRequestHeader('X-tktrex-PublicKey', settings.publicKey); - xhr.setRequestHeader('X-tktrex-Signature', bs58.encode(signature)); - - xhr.send(payload); - xhr.onload = function () { - if (this.status >= 200 && this.status < 300) { - try { - resolve(JSON.parse(this.response)); - } catch (e) { - reject(new Error('Invalid JSON received from API')); - } - } else { - reject(this.statusText); - } - }; - - xhr.onerror = function () { - reject(this.statusText); - }; - }) - .catch(reject); - }); - }; - -const api = { - postEvents: post('events'), - postAPIEvents: post('apiEvents'), - validate: post('validate'), - handshake: post('handshake'), -}; - -export default api; diff --git a/packages/shared/src/extension/chrome/background/account.ts b/packages/shared/src/extension/chrome/background/account.ts index 391c780df..c1961d32a 100644 --- a/packages/shared/src/extension/chrome/background/account.ts +++ b/packages/shared/src/extension/chrome/background/account.ts @@ -18,7 +18,12 @@ export interface KeyPair { const FIXED_USER_NAME = 'local'; // defaults of the settings stored in 'config' and controlled by popup -const DEFAULT_SETTINGS = { active: true, ux: false, researchTag: '' }; +const DEFAULT_SETTINGS = { + active: true, + ux: false, + researchTag: '', + experimentId: undefined, +}; /** * Create publicKey and secretKey diff --git a/packages/shared/src/extension/chrome/background/sendMessage.ts b/packages/shared/src/extension/chrome/background/sendMessage.ts index c350e9692..f27e1e11a 100644 --- a/packages/shared/src/extension/chrome/background/sendMessage.ts +++ b/packages/shared/src/extension/chrome/background/sendMessage.ts @@ -31,6 +31,7 @@ const ifValid = v ).join('\n')}`; log.error(msg); + // eslint-disable-next-line n/no-callback-literal cb({ type: 'Error', error: new Error( @@ -38,6 +39,7 @@ const ifValid = ), }); } else { + // eslint-disable-next-line n/no-callback-literal cb({ type: 'Success', result: v.right }); } }; @@ -61,7 +63,7 @@ export const localLookup = (cb: SendResponse): void => export const serverLookup = ( payload: ServerLookup['payload'], - cb: SendResponse< HandshakeResponse> + cb: SendResponse ): void => sendMessage( { @@ -73,7 +75,7 @@ export const serverLookup = ( export const configUpdate = ( payload: Partial, - cb: SendResponse< UserSettings> + cb: SendResponse ): void => sendMessage( { diff --git a/packages/shared/src/extension/chrome/background/sync.ts b/packages/shared/src/extension/chrome/background/sync.ts index b01d90989..a86386096 100644 --- a/packages/shared/src/extension/chrome/background/sync.ts +++ b/packages/shared/src/extension/chrome/background/sync.ts @@ -15,7 +15,7 @@ export interface LoadOpts { export const handleAPISyncMessage = ({ api, getHeadersForDataDonation }: LoadOpts) => - (request: any, sender: any, sendResponse: any): void => { + (request: any, sender: any, sendResponse: any): boolean => { void getHeadersForDataDonation(request) .then((headers) => (api as any).v2.Public.AddAPIEvents({ @@ -42,11 +42,12 @@ export const handleAPISyncMessage = error: e, }) ); + return true; }; export const handleSyncMessage = ({ api, getHeadersForDataDonation }: LoadOpts) => - (request: any, sender: any, sendResponse: any): void => { + (request: any, sender: any, sendResponse: any): boolean => { // log.debug('Sync request %O', request.payload); void getHeadersForDataDonation(request) .then((headers) => { @@ -69,6 +70,7 @@ export const handleSyncMessage = }); } }); + return true; }; export const load = ({ api, getHeadersForDataDonation }: LoadOpts): void => { diff --git a/packages/shared/src/extension/config.ts b/packages/shared/src/extension/config.ts index 6499d8f0c..8defdbbf6 100644 --- a/packages/shared/src/extension/config.ts +++ b/packages/shared/src/extension/config.ts @@ -1,6 +1,4 @@ export interface Config { - active: boolean; - ux: boolean; API_ROOT: string; BUILD: string; DEVELOPMENT: boolean; @@ -28,8 +26,6 @@ if (!process.env.BUILD) { } const config: Config = { - active: process.env.DATA_CONTRIBUTION_ENABLED === 'true', - ux: false, BUILD: process.env.BUILD, BUILD_DATE: process.env.BUILD_DATE, DEVELOPMENT: process.env.NODE_ENV === 'development', diff --git a/packages/shared/src/extension/models/UserSettings.ts b/packages/shared/src/extension/models/UserSettings.ts index 25d8b58c1..363ab08a5 100644 --- a/packages/shared/src/extension/models/UserSettings.ts +++ b/packages/shared/src/extension/models/UserSettings.ts @@ -6,7 +6,8 @@ export const UserSettings = t.type( publicKey: t.string, secretKey: t.string, ux: t.boolean, - researchTag: t.string, + researchTag: t.union([t.string, t.undefined]), + experimentId: t.union([t.string, t.undefined]), }, 'UserSettings' ); diff --git a/packages/shared/src/models/ContributionEvent.ts b/packages/shared/src/models/ContributionEvent.ts index 566560be7..7b178d56b 100644 --- a/packages/shared/src/models/ContributionEvent.ts +++ b/packages/shared/src/models/ContributionEvent.ts @@ -1,14 +1,23 @@ import * as t from 'io-ts'; +const ContributionBaseEvent = t.strict( + { + randomUUID: t.string, + incremental: t.number, + clientTime: t.string, + href: t.string, + researchTag: t.union([t.string, t.undefined]), + experimentId: t.union([t.string, t.undefined]), + }, + 'ContributionBaseEvent' +); + export const VideoContributionEvent = t.strict( { + ...ContributionBaseEvent.type.props, type: t.union([t.literal('video'), t.literal('NewVideo')]), element: t.string, size: t.number, - href: t.string, - randomUUID: t.string, - incremental: t.number, - clientTime: t.string, }, 'VideoContributionEvent' ); @@ -17,16 +26,13 @@ export type VideoContributionEvent = t.TypeOf; export const ADVContributionEvent = t.strict( { + ...ContributionBaseEvent.type.props, type: t.literal('leaf'), html: t.string, hash: t.number, offsetTop: t.number, offsetLeft: t.number, - href: t.string, selectorName: t.string, - randomUUID: t.string, - incremental: t.number, - clientTime: t.string, }, 'ADVContributionEvent' ); diff --git a/packages/shared/src/models/Experiment.ts b/packages/shared/src/models/Experiment.ts index 0e8e85a16..019be38ed 100644 --- a/packages/shared/src/models/Experiment.ts +++ b/packages/shared/src/models/Experiment.ts @@ -18,16 +18,6 @@ export const GetDirectiveOutput = nonEmptyArray( ); export type GetDirectiveOutput = t.TypeOf; -export const ConcludeGuardoniExperimentOutput = t.type( - { - acknowledged: t.boolean, - }, - 'ConcludeGuardoniExperimentOutput' -); -export type ConcludeGuardoniExperimentOutput = t.TypeOf< - typeof ConcludeGuardoniExperimentOutput ->; - export const GetExperimentListOutput = t.strict( { content: t.array(GuardoniExperiment), diff --git a/packages/shared/src/models/HandshakeBody.ts b/packages/shared/src/models/HandshakeBody.ts index dae58c32e..300c1dbad 100644 --- a/packages/shared/src/models/HandshakeBody.ts +++ b/packages/shared/src/models/HandshakeBody.ts @@ -10,7 +10,7 @@ export const HandshakeBody = t.type( experimentId: t.union([t.string, t.undefined]), execount: t.union([t.number, t.undefined]), newProfile: t.union([t.boolean, t.undefined]), - evidencetag: t.union([t.string, t.undefined]), + researchTag: t.union([t.string, t.undefined]), directiveType: t.union([t.string, t.undefined]), testTime: t.union([DateFromISOString, t.undefined]), }, diff --git a/platforms/tktrex/shared/src/models/common/Format.ts b/packages/shared/src/models/common/Format.ts similarity index 100% rename from platforms/tktrex/shared/src/models/common/Format.ts rename to packages/shared/src/models/common/Format.ts diff --git a/platforms/tktrex/shared/src/models/common/What.ts b/packages/shared/src/models/common/What.ts similarity index 95% rename from platforms/tktrex/shared/src/models/common/What.ts rename to packages/shared/src/models/common/What.ts index 78e8bad6a..0d2536455 100644 --- a/platforms/tktrex/shared/src/models/common/What.ts +++ b/packages/shared/src/models/common/What.ts @@ -3,6 +3,6 @@ import * as t from 'io-ts'; // still a copy from YT to be converted export const What = t.union( [t.literal('foryou'), t.literal('following'), t.literal('search')], - 'What', + 'What' ); export type What = t.TypeOf; diff --git a/platforms/tktrex/shared/src/models/common/index.ts b/packages/shared/src/models/common/index.ts similarity index 60% rename from platforms/tktrex/shared/src/models/common/index.ts rename to packages/shared/src/models/common/index.ts index 8c8fc6cd0..01230a654 100644 --- a/platforms/tktrex/shared/src/models/common/index.ts +++ b/packages/shared/src/models/common/index.ts @@ -1,2 +1,3 @@ export * from './What'; export * from './Format'; +export * from './StringOrNull'; diff --git a/packages/shared/src/models/index.ts b/packages/shared/src/models/index.ts index b82460433..92e42fdce 100644 --- a/packages/shared/src/models/index.ts +++ b/packages/shared/src/models/index.ts @@ -1,4 +1,4 @@ -import { StringOrNull } from './common/StringOrNull'; +import { StringOrNull, What, Format } from './common'; import { AuthResponse } from './Auth'; import { AuthorizedContentCreator, @@ -24,7 +24,6 @@ import { } from './ContributionEvent'; import { GuardoniExperiment, - ConcludeGuardoniExperimentOutput, GetDirectiveOutput, GetPublicDirectivesOutput, GetExperimentListOutput, @@ -35,6 +34,8 @@ import { Supporter } from './Supporter'; export default { // common StringOrNull, + What, + Format, // handshake HandshakeBody, HandshakeResponse, @@ -46,7 +47,6 @@ export default { // guardoni // guardoni experiments (to be moved) GuardoniExperiment, - ConcludeGuardoniExperimentOutput, GetDirectiveOutput, GetExperimentListOutput, GetPublicDirectivesOutput, @@ -72,5 +72,5 @@ export default { CreatorStatContent, ChannelRelated, GetRelatedChannelsOutput, - ...Directive + ...Directive, }; diff --git a/packages/shared/src/providers/api.provider.ts b/packages/shared/src/providers/api.provider.ts index 0cb28ebb0..e79899102 100644 --- a/packages/shared/src/providers/api.provider.ts +++ b/packages/shared/src/providers/api.provider.ts @@ -16,6 +16,7 @@ import * as t from 'io-ts'; import { PathReporter } from 'io-ts/lib/PathReporter'; import { MinimalEndpointInstance, TypeOfEndpointInstance } from '../endpoints'; import { APIError } from '../errors/APIError'; +import { toValidationError } from '../errors/ValidationError'; import { trexLogger } from '../logger'; export const apiLogger = trexLogger.extend('API'); @@ -26,15 +27,18 @@ export const toAPIError = (e: unknown): APIError => { if (e instanceof Error) { if (e.message === 'Network Error') { return new APIError( + 502, 'Network Error', 'The API endpoint is not reachable', ["Be sure you're connected to internet."] ); } - return new APIError('UnknownError', e.message, []); + return new APIError(500, 'UnknownError', e.message, []); } - return new APIError('UnknownError', 'An error occurred', [JSON.stringify(e)]); + return new APIError(500, 'UnknownError', 'An error occurred', [ + JSON.stringify(e), + ]); }; const liftFetch = ( @@ -51,7 +55,7 @@ const liftFetch = ( E.mapLeft((e): APIError => { const details = PathReporter.report(E.left(e)); apiLogger.error('toAPIError Validation failed %O', details); - return new APIError('ValidationError', 'Validation failed', details); + return toValidationError('Validation failed', details); }), TE.fromEither ); @@ -125,7 +129,7 @@ export const MakeHTTPClient = (client: AxiosInstance): HTTPClient => { TE.mapLeft((e): APIError => { const details = PathReporter.report(E.left(e)); apiLogger.error('MakeHTTPClient Validation failed %O', details); - return new APIError('ValidationError', 'Validation failed', details); + return toValidationError('Validation failed', details); }), TE.chain((input) => { const url = e.getPath(input.params); diff --git a/packages/shared/src/providers/dataDonation.provider.ts b/packages/shared/src/providers/dataDonation.provider.ts index 353f9bcf1..e92da8f89 100644 --- a/packages/shared/src/providers/dataDonation.provider.ts +++ b/packages/shared/src/providers/dataDonation.provider.ts @@ -282,6 +282,8 @@ function manageNodes( leavesCache[hash] = 1; // as it is the first observation, take infos and send it const acquired: ADVContributionEvent = { + experimentId: undefined, + researchTag: undefined, type: 'leaf', html, hash, @@ -407,6 +409,8 @@ export const GetDataDonationProvider = ( if (sendableNode === null || !sizeCheck(sendableNode.outerHTML)) return; addContribution({ + researchTag: undefined, + experimentId: undefined, type: 'video', element: sendableNode.outerHTML, size: sendableNode.outerHTML.length, diff --git a/packages/shared/src/providers/puppeteer/DirectiveHook.ts b/packages/shared/src/providers/puppeteer/DirectiveHook.ts index c0b142045..7dfad1714 100644 --- a/packages/shared/src/providers/puppeteer/DirectiveHook.ts +++ b/packages/shared/src/providers/puppeteer/DirectiveHook.ts @@ -1,14 +1,12 @@ import * as puppeteer from 'puppeteer-core'; import { OpenURLDirective } from '../../models/Directive'; +type Hook = (page: puppeteer.Page, directive: any, opts?: any) => Promise; + export interface DirectiveHooks< DO extends string, CS extends { - [key: string]: ( - page: puppeteer.Page, - directive: any, - opts?: any - ) => Promise; + [key: string]: Hook; } > { openURL: { diff --git a/packages/shared/src/providers/swagger/IOTSToOpenAPISchema.ts b/packages/shared/src/providers/swagger/IOTSToOpenAPISchema.ts index 92b106e61..fe4ce98ba 100644 --- a/packages/shared/src/providers/swagger/IOTSToOpenAPISchema.ts +++ b/packages/shared/src/providers/swagger/IOTSToOpenAPISchema.ts @@ -191,6 +191,8 @@ export const getOpenAPISchema = (codec: T): any => { } } + // swaggerLogger.info('Union type %O', nonNullableTypes); + return { type: 'object', description: type.name, diff --git a/packages/shared/src/providers/swagger/swagger.provider.ts b/packages/shared/src/providers/swagger/swagger.provider.ts index 216e0bff8..666055df4 100644 --- a/packages/shared/src/providers/swagger/swagger.provider.ts +++ b/packages/shared/src/providers/swagger/swagger.provider.ts @@ -194,7 +194,7 @@ const apiSchemaFromEndpoint = ( return { summary: (e as any).title ?? key, description: getDocumentation(e), - tags: tags, + tags, parameters, security, ...requestBody, @@ -372,6 +372,6 @@ export const generateDoc = ( ...modelSchema, }, }, - paths: paths, + paths, }; }; diff --git a/packages/shared/src/utils/arbitrary.utils.ts b/packages/shared/src/utils/arbitrary.utils.ts index 420d9c742..7772796d7 100644 --- a/packages/shared/src/utils/arbitrary.utils.ts +++ b/packages/shared/src/utils/arbitrary.utils.ts @@ -16,3 +16,17 @@ export const propsOmit =

>( ) ) as any ); + +export const propsOmitType =

>( + codec: t.TypeC

, + props: PP +): Omit => + pipe( + codec.props, + R.filterMapWithIndex((k, p) => + pipe( + p, + O.fromPredicate(() => !props.includes(k)) + ) + ) as any + ); diff --git a/packages/taboule/package.json b/packages/taboule/package.json index f3b133325..5e6e24445 100644 --- a/packages/taboule/package.json +++ b/packages/taboule/package.json @@ -35,29 +35,29 @@ "react-dom": "^17.0.2", "ts-endpoint": "^2.0.0", "ts-io-error": "^2.0.0", - "typescript": "^4.7.2" + "typescript": "^4.7.4" }, "devDependencies": { "@types/node": "^16.11.36", "@types/prettier": "^2.6.3", - "@types/react": "^17.0.45", + "@types/react": "^17.0.47", "@types/react-dom": "^17.0.17", "@types/webpack-bundle-analyzer": "^4.4.1", - "@typescript-eslint/eslint-plugin": "^5.27.0", - "@typescript-eslint/parser": "^5.27.0", - "eslint": "^8.16.0", + "@typescript-eslint/eslint-plugin": "^5.30.5", + "@typescript-eslint/parser": "^5.30.5", + "eslint": "^8.19.0", "eslint-config-prettier": "^8.5.0", - "eslint-config-standard-with-typescript": "^21.0.1", + "eslint-config-standard-with-typescript": "^22.0.0", "eslint-plugin-import": "^2.26.0", "eslint-plugin-node": "^11.1.0", - "eslint-plugin-promise": "^5.2.0", - "eslint-plugin-react": "^7.30.0", - "prettier": "^2.6.2", - "ts-loader": "^9.3.0", + "eslint-plugin-promise": "^6.0.0", + "eslint-plugin-react": "^7.30.1", + "prettier": "^2.7.1", + "ts-loader": "^9.3.1", "typelevel-ts": "^0.4.0", "webpack": "^5.73.0", "webpack-bundle-analyzer": "^4.5.0", "webpack-cli": "^4.10.0", - "webpack-dev-server": "^4.9.2" + "webpack-dev-server": "^4.9.3" } } diff --git a/packages/taboule/src/config/index.tsx b/packages/taboule/src/config/index.tsx index 1dd7a9245..e24709ca5 100644 --- a/packages/taboule/src/config/index.tsx +++ b/packages/taboule/src/config/index.tsx @@ -11,7 +11,7 @@ import { TikTokPSearchMetadata, // SummaryMetadata, } from '@shared/models/contributor/ContributorPersonalSummary'; -import { SearchMetaData as TikTokSearchMetadata } from '@tktrex/shared/models/MetaData'; +import { SearchMetadata as TikTokSearchMetadata } from '@tktrex/shared/models/Metadata'; import { Metadata } from '@shared/models/Metadata'; import { GuardoniExperiment } from '@shared/models/Experiment'; import * as React from 'react'; diff --git a/packages/taboule/src/state/queries.ts b/packages/taboule/src/state/queries.ts index 84c849d28..b9032a614 100644 --- a/packages/taboule/src/state/queries.ts +++ b/packages/taboule/src/state/queries.ts @@ -13,7 +13,7 @@ import { } from '@shared/models/contributor/ContributorPersonalSummary'; import { GuardoniExperiment } from '@shared/models/Experiment'; import { SearchQuery } from '@shared/models/http/SearchQuery'; -import { SearchMetaData as TKSearchMetadata } from '@tktrex/shared/models/MetaData'; +import { SearchMetadata as TKSearchMetadata } from '@tktrex/shared/models/Metadata'; import { Metadata } from '@shared/models/Metadata'; import { MakeAPIClient } from '@shared/providers/api.provider'; import { available, queryStrict } from 'avenger'; @@ -230,7 +230,7 @@ export const GetTabouleQueries = ({ TK_API.v2.Public.GetSearchByQuery(input), TE.map((content) => ({ total: content.length, - content: content, + content, })) ), available diff --git a/platforms/guardoni/README.md b/platforms/guardoni/README.md index 9c0541174..7f58048b8 100644 --- a/platforms/guardoni/README.md +++ b/platforms/guardoni/README.md @@ -40,7 +40,7 @@ Options: --version Show version number [boolean] --headless Run guardoni in headless mode. [boolean] [default: false] - --evidenceTag The evidence related tag. [string] + --researchTag The evidence related tag. [string] --profile The current user profile [string] --backend The API endpoint for server requests [string] --proxy Socket proxy for puppeteer. [string] diff --git a/platforms/guardoni/__tests__/cli/cli-tk.spec.ts b/platforms/guardoni/__tests__/cli/cli-tk.spec.ts index cbddca350..fa3893a31 100644 --- a/platforms/guardoni/__tests__/cli/cli-tk.spec.ts +++ b/platforms/guardoni/__tests__/cli/cli-tk.spec.ts @@ -22,7 +22,7 @@ const publicKey = process.env.PUBLIC_KEY; const secretKey = process.env.SECRET_KEY; describe('CLI', () => { - const evidenceTag = 'test-tag'; + const researchTag = 'test-tag'; let experimentId: string; let guardoni: GuardoniCLI; @@ -103,7 +103,7 @@ describe('CLI', () => { proxy: undefined, }, loadFor: 3000, - evidenceTag, + researchTag, advScreenshotDir: undefined, excludeURLTag: undefined, }, @@ -299,44 +299,6 @@ describe('CLI', () => { }); }); - test('fails when receive an error during experiment conclusion', async () => { - const data = tests.fc.sample(CommonDirectiveArb, 2).map((d) => ({ - ...d, - loadFor: 1000, - watchFor: '2s', - })); - - // return directive - axiosMock.request.mockResolvedValueOnce({ - data, - }); - - axiosMock.request.mockResolvedValueOnce({ - data: { - acknowledged: false, - }, - }); - - const start = new Date(); - const result: any = await guardoni.run({ - run: 'experiment', - experiment: experimentId as any, - })(); - const end = new Date(); - - expect(result).toMatchObject({ - _tag: 'Left', - left: { - message: "Can't conclude the experiment", - details: [], - }, - }); - - expect(differenceInMilliseconds(end, start)).toBeGreaterThan( - (2 + 3) * 2 * 100 - ); - }); - test('succeed when experimentId has valid "tk" directive', async () => { // return directive axiosMock.request.mockResolvedValueOnce({ @@ -347,12 +309,6 @@ describe('CLI', () => { })), }); - axiosMock.request.mockResolvedValueOnce({ - data: { - acknowledged: true, - }, - }); - const start = new Date(); const result: any = await guardoni.run({ run: 'experiment', @@ -367,7 +323,7 @@ describe('CLI', () => { values: [ { experimentId, - evidenceTag, + researchTag, profileName, }, ], @@ -389,12 +345,6 @@ describe('CLI', () => { })), }); - axiosMock.request.mockResolvedValueOnce({ - data: { - acknowledged: true, - }, - }); - const result: any = await guardoni.run({ run: 'experiment', experiment: experimentId as any, diff --git a/platforms/guardoni/__tests__/cli/cli-yt.spec.ts b/platforms/guardoni/__tests__/cli/cli-yt.spec.ts index dd7cbec59..9485beab9 100644 --- a/platforms/guardoni/__tests__/cli/cli-yt.spec.ts +++ b/platforms/guardoni/__tests__/cli/cli-yt.spec.ts @@ -7,6 +7,7 @@ import { } from '@shared/arbitraries/Directive.arb'; import * as tests from '@shared/test'; import differenceInMilliseconds from 'date-fns/differenceInMilliseconds'; +import { pipe } from 'fp-ts/lib/function'; import * as fs from 'fs'; import * as path from 'path'; import { GetGuardoniCLI, GuardoniCLI } from '../../src/guardoni/cli'; @@ -22,7 +23,7 @@ const publicKey = process.env.PUBLIC_KEY as string; const secretKey = process.env.SECRET_KEY as string; describe('CLI', () => { - const evidenceTag = 'test-tag'; + const researchTag = 'test-tag'; let experimentId: string; let guardoni: GuardoniCLI; @@ -50,7 +51,6 @@ describe('CLI', () => { comparisonCSVContent.right, 'utf-8' ); - }); afterAll(() => { @@ -89,7 +89,7 @@ describe('CLI', () => { proxy: undefined, }, loadFor: 3000, - evidenceTag, + researchTag: researchTag, advScreenshotDir: undefined, excludeURLTag: undefined, }, @@ -120,10 +120,7 @@ describe('CLI', () => { await expect( guardoni.run({ run: 'register-csv', - file: path.resolve( - basePath, - 'experiments/tk-experiment.csv' - ) as any, + file: path.resolve(basePath, 'experiments/tk-experiment.csv') as any, type: 'comparison', })() ).resolves.toMatchObject({ @@ -144,10 +141,7 @@ describe('CLI', () => { const result: any = await guardoni.run({ run: 'register-csv', type: 'comparison', - file: path.resolve( - basePath, - 'experiments/yt-experiment.csv' - ) as any, + file: path.resolve(basePath, 'experiments/yt-experiment.csv') as any, })(); expect(result).toMatchObject({ @@ -170,10 +164,7 @@ describe('CLI', () => { const result: any = await guardoni.run({ run: 'register-csv', type: 'comparison', - file: path.resolve( - basePath, - 'experiments/yt-experiment.csv' - ) as any, + file: path.resolve(basePath, 'experiments/yt-experiment.csv') as any, })(); expect(result).toMatchObject({ @@ -198,10 +189,7 @@ describe('CLI', () => { const result: any = await guardoni.run({ run: 'register-csv', type: 'comparison', - file: path.resolve( - basePath, - 'experiments/yt-experiment.csv' - ) as any, + file: path.resolve(basePath, 'experiments/yt-experiment.csv') as any, })(); expect(result).toMatchObject({ @@ -273,44 +261,6 @@ describe('CLI', () => { }); }); - test('fails when receive an error during experiment conclusion', async () => { - const data = tests.fc.sample(CommonDirectiveArb, 2).map((d) => ({ - ...d, - loadFor: 1000, - watchFor: '2s', - })); - - // return directive - axiosMock.request.mockResolvedValueOnce({ - data, - }); - - axiosMock.request.mockResolvedValueOnce({ - data: { - acknowledged: false, - }, - }); - - const start = new Date(); - const result: any = await guardoni.run({ - run: 'experiment', - experiment: experimentId as any, - })(); - const end = new Date(); - - expect(result).toMatchObject({ - _tag: 'Left', - left: { - message: "Can't conclude the experiment", - details: [], - }, - }); - - expect(differenceInMilliseconds(end, start)).toBeGreaterThan( - (2 + 3) * 2 * 100 - ); - }); - test('succeed when experimentId has valid "yt" directives', async () => { // return directive axiosMock.request.mockResolvedValueOnce({ @@ -321,17 +271,27 @@ describe('CLI', () => { })), }); - axiosMock.request.mockResolvedValueOnce({ - data: { - acknowledged: true, - }, - }); - const result: any = await guardoni.run({ run: 'experiment', experiment: experimentId as any, + opts: { publicKey, secretKey }, })(); + // check guardoni has written proper settings.json + // inside extension folder + + const settingJson = pipe( + path.resolve(ytExtensionDir, 'settings.json'), + (p) => fs.readFileSync(p, 'utf-8'), + JSON.parse + ); + + expect(settingJson).toMatchObject({ + publicKey, + secretKey, + experimentId, + }); + expect(result).toMatchObject({ _tag: 'Right', right: { @@ -339,8 +299,8 @@ describe('CLI', () => { values: [ { profileName, - evidenceTag, - directiveType: 'comparison', + researchTag, + experimentId, }, ], }, @@ -357,16 +317,10 @@ describe('CLI', () => { })), }); - axiosMock.request.mockResolvedValueOnce({ - data: { - acknowledged: true, - }, - }); - const result: any = await guardoni.run({ run: 'experiment', experiment: experimentId as any, - opts: { publicKey, secretKey} + opts: { publicKey, secretKey }, })(); expect(result).toMatchObject({ @@ -377,8 +331,8 @@ describe('CLI', () => { { publicKey, experimentId, - evidenceTag, - profileName + researchTag, + profileName, }, ], }, @@ -397,12 +351,6 @@ describe('CLI', () => { })), }); - axiosMock.request.mockResolvedValueOnce({ - data: { - acknowledged: true, - }, - }); - const result: any = await guardoni.run({ run: 'auto', value: '1' })(); expect(result).toMatchObject({ @@ -412,7 +360,7 @@ describe('CLI', () => { values: [ { directiveType: 'comparison', - evidenceTag: evidenceTag, + researchTag, }, ], }, @@ -428,12 +376,6 @@ describe('CLI', () => { })), }); - axiosMock.request.mockResolvedValueOnce({ - data: { - acknowledged: true, - }, - }); - const result: any = await guardoni.run({ run: 'auto', value: '2' })(); expect(result).toMatchObject({ diff --git a/platforms/guardoni/__tests__/guardoni.spec.ts b/platforms/guardoni/__tests__/guardoni.spec.ts index 9cf69d743..b146f7d9c 100644 --- a/platforms/guardoni/__tests__/guardoni.spec.ts +++ b/platforms/guardoni/__tests__/guardoni.spec.ts @@ -62,7 +62,7 @@ describe('Guardoni', () => { tosAccepted: undefined, ...keys, profileName: profile, - evidenceTag: 'test-tag', + researchTag: 'test-tag', advScreenshotDir: undefined, excludeURLTag: undefined, loadFor: 3000, @@ -261,15 +261,12 @@ describe('Guardoni', () => { return Promise.resolve({ asElement: () => ({ press: jest.fn().mockResolvedValue(undefined), - evaluate: jest - .fn() - .mockResolvedValueOnce(1) - .mockResolvedValue(0) + evaluate: jest.fn().mockResolvedValueOnce(1).mockResolvedValue(0), }), }); }); - pageMock.$.mockResolvedValue(null) + pageMock.$.mockResolvedValue(null); const guardoni = GetGuardoni({ basePath, diff --git a/platforms/guardoni/bin/check-stealth.ts b/platforms/guardoni/bin/check-stealth.ts index 63c6d8cfb..4afbd0049 100644 --- a/platforms/guardoni/bin/check-stealth.ts +++ b/platforms/guardoni/bin/check-stealth.ts @@ -4,9 +4,8 @@ import { dispatchBrowser } from '../src/guardoni/browser'; import { getDefaultConfig } from '../src/guardoni/config'; import { guardoniLogger } from '../src/logger'; - void (async () => { - const ctx = {} as any + const ctx = {} as any; await dispatchBrowser(ctx)({ version: '1', guardoniConfigFile: 'guardoni.config.json', @@ -25,7 +24,7 @@ void (async () => { profileName: 'default', newProfile: true, execount: 0, - evidencetag: [], + researchTag: [], }, })().then(async (g) => { if (g._tag === 'Right') { diff --git a/platforms/guardoni/docs/cli/usage.md b/platforms/guardoni/docs/cli/usage.md index c58e46edf..5d5bca63c 100644 --- a/platforms/guardoni/docs/cli/usage.md +++ b/platforms/guardoni/docs/cli/usage.md @@ -3,7 +3,6 @@ title: Usage sidebar_position: 2 --- - ```bash guardoni-cli @@ -33,7 +32,7 @@ Options: [string] [default: "guardoni.config.json"] --headless Run guardoni in headless mode. [boolean] [default: false] - --evidenceTag The evidence related tag. [string] + --researchTag The evidence related tag. [string] --profile The current user profile [string] --backend The API endpoint for server requests [string] --proxy Socket proxy for puppeteer. [string] @@ -89,7 +88,7 @@ List succeeded: Experiments List Output values: -d659fc7852b7a2878387773231054d534976bb12: +d659fc7852b7a2878387773231054d534976bb12: when: 2022-05-27T11:50:02.423Z directiveType: comparison @@ -115,7 +114,7 @@ Experiment succeeded: Experiment completed Output values: experimentId: d659fc7852b7a2878387773231054d534976bb12 -evidenceTag: no-tag-25368 +researchTag: no-tag-25368 directiveType: comparison execCount: 3 profileName: profile-test-99 diff --git a/platforms/guardoni/docs/development/test.md b/platforms/guardoni/docs/development/test.md index a77839428..b10a0e03f 100644 --- a/platforms/guardoni/docs/development/test.md +++ b/platforms/guardoni/docs/development/test.md @@ -22,8 +22,10 @@ For this purpose there are two different scripts located at `platforms/guardoni/ ```bash cd ./platforms/guardoni -# run test for youtube platform -./scripts/cli-yt-test.sh +# run test for `platform` "youtube" and `type` "home" +./scripts/cli-yt-test-home.mjs +# run test for `platform` "youtube" and `type` "video" +./scripts/cli-yt-test-videos.mjs # run test for titkok platform ./scripts/cli-tk-test.sh ``` diff --git a/platforms/guardoni/docs/intro.md b/platforms/guardoni/docs/intro.md index 9f4ef9431..746e2f82e 100644 --- a/platforms/guardoni/docs/intro.md +++ b/platforms/guardoni/docs/intro.md @@ -34,7 +34,7 @@ Guardoni is still **alpha** stage, the software is not thoroughly tested by the - **[Installation guide](https://docs.tracking.exposed/guardoni/getting-started/installation)** - Get Scrapy installed on your computer. + Get Guardoni installed on your computer. - **Coming soon:** Guardoni Tutorial diff --git a/platforms/guardoni/package.json b/platforms/guardoni/package.json index 1194e22fa..1bd460f5e 100644 --- a/platforms/guardoni/package.json +++ b/platforms/guardoni/package.json @@ -22,7 +22,9 @@ "postinstall": "electron-builder install-app-deps", "prepack": "yarn build", "check-stealth": "ts-node ./bin/check-stealth.ts", - "tdd": "jest --watch --verbose" + "tdd": "jest --watch --verbose", + "cli-yt-test-videos": "node ./scripts/cli-yt-test-videos.mjs", + "cli-yt-test-home": "node ./scripts/cli-yt-test-home.mjs" }, "bin": { "cli": "./bin/guardoni-cli.js" @@ -56,7 +58,7 @@ "csv-stringify": "^6.0.5", "debug": "^4.3.4", "electron-debug": "^3.2.0", - "electron-log": "^4.4.7", + "electron-log": "^4.4.8", "electron-squirrel-startup": "^1.0.0", "electron-store": "^8.0.2", "electron-unhandled": "^4.0.1", @@ -77,8 +79,8 @@ "puppeteer-in-electron": "^3.0.5", "react": "^17.0.2", "react-dom": "^17.0.2", - "react-router": "^5.2.1", - "react-router-dom": "^5.3.0", + "react-router": "^5.3.3", + "react-router-dom": "^5.3.3", "react-swipeable-views": "^0.14.0", "unzipper": "^0.10.11", "ws": "^8.5.0", @@ -90,45 +92,46 @@ "@types/lodash": "^4.14.182", "@types/nconf": "^0.10.2", "@types/prettier": "^2.6.3", - "@types/react": "^17.0.45", + "@types/react": "^17.0.47", "@types/react-dom": "^17.0.17", "@types/unzipper": "^0.10.5", "@types/webpack-node-externals": "^2.5.3", "@types/ws": "^8.5.3", "@types/yargs": "^17.0.10", - "@typescript-eslint/eslint-plugin": "^5.27.0", - "@typescript-eslint/parser": "^5.27.0", + "@typescript-eslint/eslint-plugin": "^5.30.5", + "@typescript-eslint/parser": "^5.30.5", "bufferutil": "^4.0.6", "canvas": "^2.9.1", "chalk": "^5.0.0", "electron": "16.2.8", - "electron-builder": "^23.2.0", + "electron-builder": "^23.3.0", "electron-reloader": "^1.2.3", - "eslint": "^8.16.0", + "eslint": "^8.19.0", "eslint-config-prettier": "^8.5.0", - "eslint-config-standard-with-typescript": "^21.0.1", + "eslint-config-standard-with-typescript": "^22.0.0", "eslint-plugin-import": "^2.26.0", "eslint-plugin-node": "^11.1.0", - "eslint-plugin-promise": "^5.2.0", + "eslint-plugin-promise": "^6.0.0", "jest": "^27.5.1", "node-loader": "^2.0.0", "node-polyfill-webpack-plugin": "^1.1.4", "pkg": "^5.7.0", - "prettier": "^2.6.2", + "prettier": "^2.7.1", "rpmbuild": "^0.0.23", "string-replace-loader": "^3.1.0", "style-loader": "^3.3.1", "ts-jest": "^27.1.5", - "ts-loader": "^9.3.0", - "ts-node": "^10.8.0", + "ts-loader": "^9.3.1", + "ts-node": "^10.8.2", "tsconfig-paths": "^3.14.1", - "typescript": "^4.7.2", + "typescript": "^4.7.4", "unlazy-loader": "^0.1.3", "utf-8-validate": "^5.0.9", "webpack": "^5.73.0", "webpack-bundle-analyzer": "^4.5.0", "webpack-cli": "^4.10.0", - "webpack-node-externals": "^3.0.0" + "webpack-node-externals": "^3.0.0", + "xvfb": "^0.4.0" }, "pkg": { "scripts": [ diff --git a/platforms/guardoni/scripts/cli-yt-test.mjs b/platforms/guardoni/scripts/cli-yt-test-home.mjs similarity index 61% rename from platforms/guardoni/scripts/cli-yt-test.mjs rename to platforms/guardoni/scripts/cli-yt-test-home.mjs index 685fb034d..3f4e2da67 100755 --- a/platforms/guardoni/scripts/cli-yt-test.mjs +++ b/platforms/guardoni/scripts/cli-yt-test-home.mjs @@ -1,10 +1,11 @@ #!/usr/bin/env node /* eslint-disable camelcase */ -import { $, os } from 'zx'; +import { $, os, fetch } from 'zx'; import { normalizePlatform, getGuardoniCliPkgName } from './utils.mjs'; import dotenv from 'dotenv'; import assert from 'assert'; +// import Xvfb from 'xvfb'; dotenv.config({ path: '.env.development' }); @@ -29,13 +30,29 @@ void (async function () { `--secretKey=${process.env.SECRET_KEY}`, ]; + // start xvfb + // const xvfb = new Xvfb({ + // silent: true, + // reuse: true, + // xvfb_args: [':95'], + // }); + + // process.env.DISPLAY = ':95'; + + // xvfb.start((err) => { + // // eslint-disable-next-line no-console + // if (err) console.error(err); + // }); + + // reject cookie modal + await $`${cli} ${flags} yt-navigate --cookie-modal=reject --exit --headless=false`; + const yt_home_experiment_register_out = await $`${cli} ${flags} yt-register ./experiments/yt-home.csv | grep 'experimentId: '`; - const yt_home_experiment_id = yt_home_experiment_register_out.stdout.replace( - 'experimentId: \t ', - '' - ); + const yt_home_experiment_id = yt_home_experiment_register_out.stdout + .replace('experimentId: \t ', '') + .replace('\n', ''); await $`echo "home experiment id: ${yt_home_experiment_id}"`; @@ -51,28 +68,16 @@ void (async function () { await $`echo ${yt_home_experiment_public_key}`; - assert.strictEqual(yt_home_experiment_public_key, process.env.PUBLIC_KEY); - await $`curl "http://localhost:9000/api/v1/personal/${yt_home_experiment_public_key}"`; - - // // register an experiment for videos - const yt_video_experiment_register_out = - await $`${cli} ${flags} yt-register ./experiments/yt-videos.csv | grep 'experimentId: '`; - const yt_video_experiment_id = - yt_video_experiment_register_out.stdout.replace('experimentId: \t ', ''); + // wait the parser to finish process metadata + await $`sleep 5`; - await $`echo ${yt_video_experiment_id}`; - - // exec the experiment - const ytVideoExperimentRunOut = - await $`(${cli} ${experimentFlags} yt-experiment ${yt_video_experiment_id} | grep 'publicKey: ')`; - - await $`echo ${ytVideoExperimentRunOut}`; - - const ytVideoExperimentPubKey = ytVideoExperimentRunOut.stdout - .replace('publicKey: \t ', '') - .replace('\n', ''); + assert.strictEqual(yt_home_experiment_public_key, process.env.PUBLIC_KEY); + const metadata = await fetch( + `http://localhost:9000/api/v2/metadata?experimentId=${yt_home_experiment_id}` + ).then((r) => r.json()); - assert.strictEqual(ytVideoExperimentPubKey, process.env.PUBLIC_KEY); + assert.strictEqual(metadata[0].experimentId, yt_home_experiment_id); + assert.strictEqual(metadata[0].type, 'home'); - await $`curl "http://localhost:9000/api/v1/personal/${ytVideoExperimentPubKey}"`; + // xvfb.stop(); })(); diff --git a/platforms/guardoni/scripts/cli-yt-test-videos.mjs b/platforms/guardoni/scripts/cli-yt-test-videos.mjs new file mode 100755 index 000000000..33df4e09e --- /dev/null +++ b/platforms/guardoni/scripts/cli-yt-test-videos.mjs @@ -0,0 +1,70 @@ +#!/usr/bin/env node +/* eslint-disable camelcase */ + +import { $, os, fetch } from 'zx'; +import { normalizePlatform, getGuardoniCliPkgName } from './utils.mjs'; +import dotenv from 'dotenv'; +import assert from 'assert'; + +dotenv.config({ path: '.env.development' }); + +// eslint-disable-next-line no-void +void (async function () { + const version = await $`node -p -e "require('./package.json').version"`; + const platform = normalizePlatform(os.type()); + const cli = `./dist/${getGuardoniCliPkgName( + version.stdout.replace('\n', ''), + platform + )}`; + const flags = [ + '--basePath=./', + `--executablePath=${process.env.PUPPETEER_EXEC_PATH}`, + '-c=guardoni.config.json', + '--headless', + '--verbose', + ]; + + const experimentFlags = [ + ...flags, + `--publicKey=${process.env.PUBLIC_KEY}`, + `--secretKey=${process.env.SECRET_KEY}`, + ]; + + // reject cookie modal + await $`${cli} ${flags} yt-navigate --cookie-modal=reject --exit --headless=false` + + // // register an experiment for videos + const yt_video_experiment_register_out = + await $`${cli} ${flags} yt-register ./experiments/yt-videos.csv | grep 'experimentId: '`; + const yt_video_experiment_id = yt_video_experiment_register_out.stdout + .replace('experimentId: \t ', '') + .replace('\n', ''); + + await $`echo ${yt_video_experiment_id}`; + + // exec the experiment + const ytVideoExperimentRunOut = + await $`(${cli} ${experimentFlags} yt-experiment ${yt_video_experiment_id} | grep 'publicKey: ')`; + + await $`echo ${ytVideoExperimentRunOut}`; + + const ytVideoExperimentPubKey = ytVideoExperimentRunOut.stdout + .replace('publicKey: \t ', '') + .replace('\n', ''); + + assert.strictEqual(ytVideoExperimentPubKey, process.env.PUBLIC_KEY); + + // wait the parser to finish process metadata + await $`sleep 5`; + + const metadata = await fetch( + `http://localhost:9000/api/v2/metadata?experimentId=${yt_video_experiment_id}` + ).then((r) => r.json()); + + assert.strictEqual(metadata[0].experimentId, yt_video_experiment_id); + assert.strictEqual(metadata[0].type, 'video'); + assert.strictEqual(metadata[1].experimentId, yt_video_experiment_id); + assert.strictEqual(metadata[1].type, 'video'); + assert.strictEqual(metadata[2].experimentId, yt_video_experiment_id); + assert.strictEqual(metadata[2].type, 'video'); +})(); diff --git a/platforms/guardoni/src/guardoni/browser.ts b/platforms/guardoni/src/guardoni/browser.ts index b6f183b2d..b6dc99859 100644 --- a/platforms/guardoni/src/guardoni/browser.ts +++ b/platforms/guardoni/src/guardoni/browser.ts @@ -13,8 +13,8 @@ export const dispatchBrowser = const commandLineArg = [ '--no-sandbox', '--disabled-setuid-sandbox', - '--load-extension=' + ctx.platform.extensionDir, '--disable-extensions-except=' + ctx.platform.extensionDir, + '--load-extension=' + ctx.platform.extensionDir, ]; if (proxy) { diff --git a/platforms/guardoni/src/guardoni/cli.ts b/platforms/guardoni/src/guardoni/cli.ts index 0f60a6610..7ba633190 100644 --- a/platforms/guardoni/src/guardoni/cli.ts +++ b/platforms/guardoni/src/guardoni/cli.ts @@ -27,10 +27,18 @@ import { DirectiveType } from '@shared/models/Directive'; export const cliLogger = guardoniLogger.extend('cli'); export interface GuardoniCommandOpts { + headless?: boolean; publicKey: string; secretKey: string; } +export interface GuardoniNavigateOpts extends GuardoniCommandOpts { + cookieModal?: { + action: 'reject' | 'accept'; + }; + exit?: boolean; +} + export type GuardoniCommandConfig = | { run: 'register-csv'; @@ -51,7 +59,7 @@ export type GuardoniCommandConfig = } | { run: 'navigate'; - opts: GuardoniCommandOpts; + opts: GuardoniNavigateOpts; }; export interface GuardoniCLI { @@ -157,8 +165,12 @@ export const GetGuardoniCLI: GetGuardoniCLI = ( return g.runExperiment(command.experiment, command.opts); case 'navigate': { return pipe( - g.runBrowser(command.opts), - TE.map(() => ({ type: 'success', values: [], message: '' })) + g.runNavigate(command.opts), + TE.map(() => ({ + type: 'success', + values: [], + message: 'Navigation completed!', + })) ); } case 'auto': @@ -223,9 +235,9 @@ const runGuardoni = ({ const basePath = guardoniConf.basePath ?? DEFAULT_BASE_PATH; if (verbose) { - D.enable('@guardoni*'); + D.enable('@trex*,guardoni*'); - cliLogger.debug('Running guardoni', { config, basePath, guardoniConf }); + cliLogger.debug('Running guardoni', { config, basePath, ...guardoniConf }); if (config) { // eslint-disable-next-line cliLogger.debug(`Configuration loaded from ${config}`, guardoniConf); @@ -281,12 +293,22 @@ const program = yargs(hideBin(process.argv)) type: 'string', desc: 'The secretKey to use to sign the evidences', default: undefined, + }) + .option('cookie-modal', { + type: 'string', + choices: ['accept', 'reject'], + }) + .option('exit', { + type: 'boolean', }), - ({ publicKey, secretKey, ...argv }) => { + (args) => { void runGuardoni({ - ...argv, + ...args, platform: 'youtube', - command: { run: 'navigate', opts: { publicKey, secretKey } }, + command: { + run: 'navigate', + opts: args, + }, }); } ) @@ -310,14 +332,14 @@ const program = yargs(hideBin(process.argv)) desc: 'The secretKey to use to sign the evidences', default: undefined, }), - ({ experiment, publicKey, secretKey, ...argv }) => { + ({ experiment, ...args }) => { void runGuardoni({ - ...argv, + ...args, platform: 'youtube', command: { run: 'experiment', experiment, - opts: { publicKey, secretKey }, + opts: args, }, }); } @@ -522,7 +544,7 @@ const program = yargs(hideBin(process.argv)) desc: 'Run guardoni in headless mode.', default: false, }) - .option('evidenceTag', { + .option('researchTag', { type: 'string', desc: 'The evidence related tag.', }) diff --git a/platforms/guardoni/src/guardoni/config.ts b/platforms/guardoni/src/guardoni/config.ts index 91e44c0c8..3afe688ae 100644 --- a/platforms/guardoni/src/guardoni/config.ts +++ b/platforms/guardoni/src/guardoni/config.ts @@ -38,7 +38,7 @@ export const getDefaultConfig = (basePath: string): GuardoniConfig => { loadFor: DEFAULT_LOAD_FOR, basePath, profileName: 'default', - evidenceTag: randomTag(), + researchTag: randomTag(), advScreenshotDir: undefined, excludeURLTag: undefined, tosAccepted: undefined, @@ -65,9 +65,9 @@ export const checkConfig = basePath: string, { yt, tk, ...conf }: Partial ): Partial> & { basePath: string } => { - const evidenceTag = conf.evidenceTag ?? randomTag(); + const researchTag = conf.researchTag ?? randomTag(); - ctx.logger.debug('EvidenceTag %O', evidenceTag); + ctx.logger.debug('Research Tag %O', researchTag); const absoluteBasePath = path.isAbsolute(basePath) ? basePath @@ -87,7 +87,7 @@ export const checkConfig = return { ...conf, - evidenceTag, + researchTag, basePath: absoluteBasePath, yt: yt ? { diff --git a/platforms/guardoni/src/guardoni/directives/yt.directives.ts b/platforms/guardoni/src/guardoni/directives/yt.directives.ts index d923b0186..b9fd3b6a3 100644 --- a/platforms/guardoni/src/guardoni/directives/yt.directives.ts +++ b/platforms/guardoni/src/guardoni/directives/yt.directives.ts @@ -235,8 +235,9 @@ async function afterWait(page: puppeteer.Page, directive: any): Promise { player: undefined, name: undefined, }; + if (directive.url.match(/\/watch\?v=/)) { - state = await getYTstatus(page); + state = await getYTStatus(page); debug('afterWait status found to be: %s', state.name); await interactWithYT(page, directive, 'playing'); // hasPlayer = true; @@ -252,6 +253,46 @@ async function afterWait(page: puppeteer.Page, directive: any): Promise { await page.screenshot({ path: fullpath, fullPage: true }); } } + +const cookieModal = async ( + page: puppeteer.Page, + opts: { + action: 'reject' | 'accept'; + } +): Promise => { + if (opts) { + debug('Searching for cookie modal...'); + const hasModal = await page.$('ytd-consent-bump-v2-lightbox'); + + if (hasModal) { + debug('Modal present, searching for buttons...'); + + /** + * Modal buttons + * + * buttons[0] -> Language Selector + * buttons[1] -> "Sign In" + * buttons[2] -> "More Options" + * buttons[2] -> "Reject All" + * buttons[3] -> "Accept App" + */ + const buttons = await hasModal.$$('tp-yt-paper-button'); + + debug('Cookie modal action %s', opts.action); + + if (opts.action === 'reject') { + const rejectButton = buttons[3]; + await rejectButton.click(); + } else if (opts.action === 'accept') { + const acceptButton = buttons[4]; + await acceptButton.click(); + } + + debug('Cookie modal close!'); + } + } +}; + const condition = { '-1': 'unstarted', 0: 'ended', @@ -261,7 +302,7 @@ const condition = { 5: 'video cued', }; -async function getYTstatus(page: puppeteer.Page): Promise<{ +async function getYTStatus(page: puppeteer.Page): Promise<{ name: string; player: puppeteer.JSHandle; }> { @@ -289,10 +330,11 @@ async function interactWithYT( const PERIODIC_CHECK_MS = 3000; // consenso all'inizio + // await cookieModal(page); // non voglio loggarmi (24 ore) // non voglio la prova gratuita (random) - let state = await getYTstatus(page); + let state = await getYTStatus(page); if (state.name !== wantedState) { debug( 'State switching necessary (now %s, wanted %s)', @@ -305,7 +347,7 @@ async function interactWithYT( if (state.name === 'unstarted') { await (state.player as any).press('Space'); await page.waitForTimeout(600); - state = await getYTstatus(page); + state = await getYTStatus(page); } else debug( 'DO NOT press [space] please, as the video is in state [%s]', @@ -332,7 +374,7 @@ async function interactWithYT( for (const checktime of _.times(DEFAULT_MAX_TIME / PERIODIC_CHECK_MS)) { await page.waitForTimeout(PERIODIC_CHECK_MS); - const newst = await getYTstatus(page); + const newst = await getYTStatus(page); if (newst.name === 'unstarted') { debug( @@ -359,9 +401,9 @@ async function interactWithYT( debug('Special value format %s (%d ms)', specialwatch, ms); await page.waitForTimeout(ms); - const newst = await getYTstatus(page); + const newst = await getYTStatus(page); await (newst.player as any).press('Space'); - await getYTstatus(page); + await getYTStatus(page); } else if (_.isInteger(specialwatch)) { // eslint-disable-next-line no-console console.log( @@ -384,15 +426,9 @@ async function interactWithYT( type YTHooks = DirectiveHooks< 'youtube.com', { - interactWithYT: ( - page: puppeteer.Page, - directive: OpenURLDirective, - opts: string - ) => Promise; - getYTstatus: ( - page: puppeteer.Page, - directive: OpenURLDirective - ) => Promise<{ name: string; player: any }>; + interactWithYT: typeof interactWithYT; + getYTStatus: typeof getYTStatus; + cookieModal: typeof cookieModal; } >; @@ -412,8 +448,9 @@ export const GetYTHooks: GetYTHooks = (ctx) => { completed, }, customs: { - getYTstatus, + getYTStatus, interactWithYT, + cookieModal, }, DOMAIN_NAME: 'youtube.com' as const, }; diff --git a/platforms/guardoni/src/guardoni/experiment.ts b/platforms/guardoni/src/guardoni/experiment.ts index 06430d0a9..5d9dc53f9 100644 --- a/platforms/guardoni/src/guardoni/experiment.ts +++ b/platforms/guardoni/src/guardoni/experiment.ts @@ -276,7 +276,7 @@ export const saveExperiment = const experimentInfo = { experimentId, - evidenceTag: ctx.config.evidenceTag, + researchTag: ctx.config.researchTag, directiveType, execCount: profile.execount, profileName: profile.profileName, @@ -327,26 +327,3 @@ export const listExperiments = ) ); }; - -export const concludeExperiment = - (ctx: GuardoniContext) => - (experimentInfo: ExperimentInfo): TE.TaskEither => { - // this conclude the API sent by extension remoteLookup, - // a connection to DELETE /api/v3/experiment/:publicKey - - return pipe( - ctx.API.v3.Public.ConcludeExperiment({ - Params: { - testTime: experimentInfo.when.toISOString(), - }, - }), - TE.chain((body) => { - if (!body.acknowledged) { - return TE.left( - new AppError('APIError', "Can't conclude the experiment", []) - ); - } - return TE.right(experimentInfo); - }) - ); - }; diff --git a/platforms/guardoni/src/guardoni/extension.ts b/platforms/guardoni/src/guardoni/extension.ts index 0bdca8d08..69910a11f 100644 --- a/platforms/guardoni/src/guardoni/extension.ts +++ b/platforms/guardoni/src/guardoni/extension.ts @@ -14,7 +14,7 @@ import * as IOE from 'fp-ts/lib/IOEither'; import * as TE from 'fp-ts/lib/TaskEither'; import * as fs from 'fs'; import path from 'path'; -import { GuardoniCommandOpts } from './cli'; +import { UserSettings } from '@shared/extension/models/UserSettings'; import { GuardoniContext, Platform } from './types'; /** @@ -141,7 +141,7 @@ export const downloadExtension = ( export const setLocalSettings = (ctx: GuardoniContext) => - (s?: GuardoniCommandOpts): void => { + (s?: Partial): void => { if (!s?.publicKey && !s?.secretKey) { ctx.logger.debug('No publicKey/secretKey pair given...'); return; diff --git a/platforms/guardoni/src/guardoni/guardoni.ts b/platforms/guardoni/src/guardoni/guardoni.ts index 0e6bba617..7f54377dd 100755 --- a/platforms/guardoni/src/guardoni/guardoni.ts +++ b/platforms/guardoni/src/guardoni/guardoni.ts @@ -32,7 +32,6 @@ import { import { GetTKHooks } from './directives/tk.directives'; import { GetYTHooks } from './directives/yt.directives'; import { - concludeExperiment, getDirective, listExperiments, registerCSV, @@ -58,11 +57,13 @@ import { ProgressDetails, } from './types'; import { getChromePath, getPackageVersion } from './utils'; -import { GuardoniCommandOpts } from './cli'; +import { GuardoniCommandOpts, GuardoniNavigateOpts } from './cli'; const runNavigate = (ctx: GuardoniContext) => - (opts?: GuardoniCommandOpts): TE.TaskEither => { + (opts?: GuardoniNavigateOpts): TE.TaskEither => { + ctx.logger.debug('Running navigate with opts %O', opts); + const home = ctx.platform.name === 'tiktok' ? 'https://www.tiktok.com' @@ -72,7 +73,7 @@ const runNavigate = TE.right(setLocalSettings(ctx)(opts)), TE.chain(() => dispatchBrowser(ctx)({ - headless: false, + headless: opts ? opts.headless : true, }) ), TE.chain((b) => { @@ -83,12 +84,18 @@ const runNavigate = waitUntil: 'networkidle0', }); + await page.waitForTimeout(2000); + + await ctx.hooks.customs.cookieModal(page, { + action: 'reject', + }); + return b; }, toAppError); }), TE.chain((b) => { return TE.tryCatch( - () => + async () => new Promise((resolve, reject) => { ctx.logger.info('Browser is ready at %s', home); b.on('error', (e) => { @@ -103,6 +110,10 @@ const runNavigate = ctx.logger.info('browser closing...'); resolve(); }); + + if (opts?.exit) { + void b.close().then(resolve).catch(reject); + } }), toAppError ); @@ -140,12 +151,10 @@ export const runBrowser = page ) ), - TE.chain((publicKey) => - pipe( - concludeExperiment(ctx)(experiment), - TE.map((exp) => ({ ...exp, publicKey: publicKey ?? opts?.publicKey })) - ) - ) + TE.map((publicKey) => ({ + ...experiment, + publicKey: publicKey ?? opts?.publicKey, + })) ); }; @@ -163,11 +172,13 @@ export const runExperiment = return pipe( sequenceS(TE.ApplicativePar)({ profile: updateGuardoniProfile(ctx)( - ctx.config.evidenceTag, + ctx.config.researchTag, ctx.profile ), expId: TE.fromEither(validateNonEmptyString(experimentId)), - localSettings: TE.right(setLocalSettings(ctx)(opts)), + localSettings: TE.right( + setLocalSettings(ctx)({ ...opts, experimentId }) + ), }), TE.chain(({ profile, expId }) => pipe( @@ -301,29 +312,32 @@ const loadContext = ( return pipe( checkProfile({ logger })(basePath, cnf), TE.chain((p) => readProfile({ logger })(getProfileJsonPath(p))), - TE.chain((profile) => - pipe( + TE.chain((profile) => { + const hooks = + platform === 'youtube' + ? GetYTHooks({ + profile, + logger, + }) + : GetTKHooks({ + profile, + }); + + return pipe( getChromePath(), E.map((chromePath) => ({ ...config, chromePath, })), E.map((c) => ({ + hooks, puppeteer: GetPuppeteer({ - logger: logger, + logger, puppeteer: p, config: { loadFor: config.loadFor, }, - hooks: - platform === 'youtube' - ? GetYTHooks({ - profile, - logger, - }) - : GetTKHooks({ - profile, - }), + hooks, }), API: MakeAPIClient( { @@ -346,8 +360,8 @@ const loadContext = ( })), E.mapLeft(toAppError), TE.fromEither - ) - ), + ); + }), TE.chainFirst(downloadExtension) ); }; @@ -376,7 +390,7 @@ export interface Guardoni { experiment: NonEmptyString, onProgress?: (details: ProgressDetails) => void ) => TE.TaskEither; - runBrowser: (opts?: GuardoniCommandOpts) => TE.TaskEither; + runNavigate: (opts?: GuardoniCommandOpts) => TE.TaskEither; } interface GuardoniLauncher { @@ -432,7 +446,7 @@ export const GetGuardoni: GetGuardoni = ({ registerExperiment: registerExperiment(ctx), registerExperimentFromCSV: registerCSV(ctx), listExperiments: listExperiments(ctx), - runBrowser: runNavigate(ctx), + runNavigate: runNavigate(ctx), }; }) ); @@ -451,7 +465,7 @@ export const GetGuardoni: GetGuardoni = ({ registerExperiment: registerExperiment(ctx), registerExperimentFromCSV: registerCSV(ctx), listExperiments: listExperiments(ctx), - runBrowser: runNavigate(ctx), + runNavigate: runNavigate(ctx), }; }) ); diff --git a/platforms/guardoni/src/guardoni/profile.ts b/platforms/guardoni/src/guardoni/profile.ts index 825257d7a..e4c0bde01 100644 --- a/platforms/guardoni/src/guardoni/profile.ts +++ b/platforms/guardoni/src/guardoni/profile.ts @@ -27,7 +27,7 @@ export const getDefaultProfile = ( udd: getProfileDataDir(basePath, profileName), newProfile: true, profileName, - evidencetag: [], + researchTag: [], execount: 0, }; }; @@ -69,7 +69,7 @@ export const checkProfile = const profileName = conf.profileName ?? lastProfile ?? - conf.evidenceTag ?? + conf.researchTag ?? `guardoni-${format(new Date(), 'yyyy-MM-dd')}`; const profileDir = getProfileDataDir(basePath, profileName); @@ -144,7 +144,7 @@ export const readProfile = export const updateGuardoniProfile = (ctx: GuardoniContext) => ( - evidenceTag: string, + researchTag: string, profile: GuardoniProfile ): TE.TaskEither => { ctx.logger.debug('Updating guardoni config %s', ctx.guardoniConfigFile); @@ -154,7 +154,7 @@ export const updateGuardoniProfile = ...profile, newProfile: execCount === 0, execount: profile.execount + 1, - evidencetag: profile.evidencetag.concat(evidenceTag), + researchTag: profile.researchTag.concat(researchTag), }; ctx.logger.debug('Writing guardoni config %O', updatedProfile); diff --git a/platforms/guardoni/src/guardoni/types.ts b/platforms/guardoni/src/guardoni/types.ts index 046f08ac7..56092e44a 100644 --- a/platforms/guardoni/src/guardoni/types.ts +++ b/platforms/guardoni/src/guardoni/types.ts @@ -5,6 +5,7 @@ import { APIClient } from '@shared/providers/api.provider'; import { PuppeteerProvider } from '@shared/providers/puppeteer/puppeteer.provider'; import * as t from 'io-ts'; import { nonEmptyArray } from 'io-ts-types'; +import { DirectiveHooks } from '@shared/providers/puppeteer/DirectiveHook'; export const Platform = t.union( [t.literal('youtube'), t.literal('tiktok')], @@ -32,7 +33,7 @@ export const GuardoniConfig = t.strict( headless: t.boolean, verbose: t.boolean, profileName: t.string, - evidenceTag: t.string, + researchTag: t.string, loadFor: t.number, advScreenshotDir: t.union([t.string, t.undefined]), excludeURLTag: t.union([t.string, t.undefined]), @@ -72,7 +73,7 @@ export const GuardoniProfile = t.strict( profileName: t.string, newProfile: t.boolean, execount: t.number, - evidencetag: t.array(t.string), + researchTag: t.array(t.string), }, 'Profile' ); @@ -81,6 +82,7 @@ export type GuardoniProfile = t.TypeOf; export interface GuardoniContext { puppeteer: PuppeteerProvider; + hooks: DirectiveHooks; API: APIClient; config: GuardoniConfig; platform: PlatformConfig; @@ -92,7 +94,7 @@ export interface GuardoniContext { export interface ExperimentInfo { experimentId: string; - evidenceTag: string; + researchTag: string; directiveType: DirectiveType; execCount: number; profileName: string; diff --git a/platforms/guardoni/src/guardoni/utils.ts b/platforms/guardoni/src/guardoni/utils.ts index c36676e4b..d227d78bb 100644 --- a/platforms/guardoni/src/guardoni/utils.ts +++ b/platforms/guardoni/src/guardoni/utils.ts @@ -18,6 +18,7 @@ export const CHROME_PATHS = [ 'C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe', 'C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe', '/Applications/Chromium.app/Contents/MacOS/Chromium', + '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome', ]; // this function check for standard chrome executable path and diff --git a/platforms/makhno/backend/.eslintignore b/platforms/makhno/backend/.eslintignore deleted file mode 100644 index 2e609504b..000000000 --- a/platforms/makhno/backend/.eslintignore +++ /dev/null @@ -1,3 +0,0 @@ -lib -routes -jest.config.js diff --git a/platforms/makhno/backend/.eslintrc b/platforms/makhno/backend/.eslintrc deleted file mode 100644 index 315003776..000000000 --- a/platforms/makhno/backend/.eslintrc +++ /dev/null @@ -1,42 +0,0 @@ -{ - "extends": ["standard-with-typescript", "prettier"], - "parser": "@typescript-eslint/parser", - "parserOptions": { - "ecmaVersion": 9, - "ecmaFeatures": { - "jsx": true - }, - "sourceType": "module", - "project": ["./tsconfig.json"] - }, - "plugins": ["@typescript-eslint", "node"], - "env": { - "es6": true, - "node": true, - "jest": true - }, - "globals": { - "_": true - }, - "settings": { - "node": { - "version": "14" - } - }, - "ignorePatterns": ["__tests__"], - "rules": { - "no-console": 1, - "no-use-before-define": "off", - "@typescript-eslint/no-use-before-define": ["error"], - "@typescript-eslint/return-await": ["off"], - "@typescript-eslint/promise-function-async": ["off"], - "@typescript-eslint/no-floating-promises": ["off"], - "@typescript-eslint/no-misused-promises": ["off"], - "@typescript-eslint/no-redeclare": ["off"], - "@typescript-eslint/restrict-plus-operands": ["off"], - "@typescript-eslint/restrict-template-expressions": ["off"], - "@typescript-eslint/strict-boolean-expressions": ["off"], - - "react/jsx-key": "off" - } -} diff --git a/platforms/makhno/backend/README.md b/platforms/makhno/backend/README.md deleted file mode 100644 index 2ebe1f170..000000000 --- a/platforms/makhno/backend/README.md +++ /dev/null @@ -1,5 +0,0 @@ - -* This is only a backend system. For the frontend, you should check: https://github.com/tracking-exposed/tracking.exposed subfolder 'makhno' -* Know more at [Makhno](https://tracking.exposed/makhno). - - diff --git a/platforms/makhno/backend/bin/app.ts b/platforms/makhno/backend/bin/app.ts deleted file mode 100644 index 453d68fbb..000000000 --- a/platforms/makhno/backend/bin/app.ts +++ /dev/null @@ -1,148 +0,0 @@ -import { MakeAppContext } from '@shared/backend/app'; -import express from 'express'; -import apiList from '../lib/api'; -import D from 'debug'; -import cors from 'cors'; -import bodyParser from 'body-parser'; -import _ from 'lodash'; - -export const appLogger = D('makhno'); - -async function iowrapper(fname, req, res): Promise { - try { - const funct = apiList[fname]; - const httpresult = await funct(req, res); - - if (httpresult.headers) - _.each(httpresult.headers, function (value, key) { - appLogger('Setting header %s: %s', key, value); - res.setHeader(key, value); - }); - - if (!httpresult) { - appLogger("API (%s) didn't return anything!?", fname); - res.send('Fatal error: Invalid output'); - res.status(501); - } else if (httpresult.json?.error) { - appLogger('API (%s) failure, returning 500', fname); - res.status(500); - res.json(httpresult.json); - } else if (httpresult.json) { - appLogger( - 'API (%s) success, returning %d bytes JSON', - fname, - _.size(JSON.stringify(httpresult.json)) - ); - res.setHeader('Access-Control-Allow-Origin', '*'); - res.json(httpresult.json); - } else if (httpresult.text) { - appLogger( - 'API (%s) success, returning text (size %d)', - fname, - _.size(httpresult.text) - ); - res.send(httpresult.text); - } else if (httpresult.status) { - appLogger( - 'Returning empty status %d from API (%s)', - httpresult.status, - fname - ); - res.status(httpresult.status); - } else { - appLogger('Undetermined failure in API (%s) → %j', fname, httpresult); - res.status(502); - res.send('Error?'); - } - } catch (error) { - res.status(505); - if (error instanceof Error) { - res.send('Software error: ' + error.message); - appLogger( - 'Error in HTTP handler API(%s): %s %s', - fname, - error.message, - error.stack - ); - } else { - res.send('Unknown software error.'); - appLogger('Unknown error in HTTP handler API(%s): %s', fname, error); - } - } - res.end(); -} - -export const makeApp = async ( - ctx: MakeAppContext -): Promise => { - const app = express(); - /* configuration of express4 */ - app.use(cors()); - app.use(bodyParser.json({ limit: '10mb' })); - app.use( - bodyParser.urlencoded({ limit: '10mb', extended: true, parameterLimit: 10 }) - ); - - /* this API is v0 as it is platform neutral. it might be shared among - * all the trex backends, and should return info on system health, echo OK - * if the system is OK, and the git log of the code running */ - app.get( - '/api/v0/info', - async (req, res) => await iowrapper('systemInfo', req, res) - ); - app.get('/api/v0/health', function (req, res) { - res.send('OK'); - res.status(200); - }); - - /* This is the primary API, to handle submitted URLs */ - app.post( - '/api/v1/submit', - async (req, res) => await iowrapper('submitURL', req, res) - ); - - /* This is the primary alpha API, to retrieve results */ - app.get( - '/api/v1/results/:urlpattern?', - async (req, res) => await iowrapper('getResults', req, res) - ); - - app.get( - '/api/v2/statistics/:name/:unit/:amount', - async (req, res) => await iowrapper('getStatistics', req, res) - ); - - /* debug API */ - app.get( - '/api/v2/debug/:urlpattern', - async (req, res) => await iowrapper('getDebugInfo', req, res) - ); - app.get( - '/api/v1/mirror/:key', - async (req, res) => await iowrapper('getMirror', req, res) - ); - - /* monitor for admin */ - app.get( - '/api/v1/monitor/:minutes?', - async (req, res) => await iowrapper('getMonitor', req, res) - ); - - /* subscription email */ - app.post( - '/api/v1/registerEmail2', - async (req, res) => await iowrapper('registerEmail2', req, res) - ); - app.get( - '/api/v1/listEmails/:key', - async (req, res) => await iowrapper('listEmails', req, res) - ); - - /* Capture All 404 errors */ - app.get('*', async (req, res) => { - appLogger('URL not handled: %s', req.url); - res.status(404); - res.send('URL not found'); - }); - return app; -}; diff --git a/platforms/makhno/backend/bin/count-o-clock.js b/platforms/makhno/backend/bin/count-o-clock.js deleted file mode 100644 index 71cb910b4..000000000 --- a/platforms/makhno/backend/bin/count-o-clock.js +++ /dev/null @@ -1,155 +0,0 @@ -#!/usr/bin/env node -const _ = require('lodash'); -const moment = require('moment'); -const debug = require('debug')('bin:count-o-clock'); -const nconf = require('nconf'); - -const aggregated = require('../lib/aggregated'); -const mongo = require('../lib/mongo3'); - -nconf.argv().env(); -const defaultConf = nconf.get('config') || 'config/settings.json'; -nconf.file({ file: defaultConf }); -const schema = nconf.get('schema'); -nconf.file({ file: 'config/trexstats.json' }); -const statsMap = nconf.get('stats'); -const name = nconf.get('name'); - -async function computeCount(mongoc, statinfo, filter) { - /* here each section of config/stats.json is processed, - * this function is called twice: once with 'hour' filter - * and another with 'day' - * - * the 'selector' is meant to be a selector of function mongo.count(), - * the 'innercount' is implemented to count in lists */ - debug( - 'in [%s] %s [%d variables]', - statinfo.column, - statinfo.name, - _.size(statinfo.variables) - ); - - const counting = []; - for (const v of statinfo.variables) { - if (v.selector) { - const thisfilter = Object.assign({}, filter, v.selector); - const amount = await mongo.count(mongoc, statinfo.column, thisfilter); - counting.push(_.set({}, v.name, amount)); - } else { - /* there is 'innercount' the deep counter - * v.innercount = [ 'related', { 'related.foryou': true } ] - unwind, matchToCount, - projectComposed = { 'related': 1 } - */ - const projectComposed = {}; - projectComposed[v.innercount[0]] = 1; - const amount = await mongo.aggregate(mongoc, statinfo.column, [ - { $match: filter }, - { $project: projectComposed }, - { $unwind: '$' + v.innercount[0] }, - { $match: v.innercount[1] }, - { $count: 'amount' }, - ]); - if (amount && _.size(amount)) - counting.push(_.set({}, v.name, _.first(amount).amount)); - } - } - return counting; -} - -function parseIntNconf(name, def) { - const value = nconf.get(name) ? nconf.get(name) : def; - return _.parseInt(value); -} - -async function start() { - const hoursago = parseIntNconf('hoursago', 0); - const daysago = parseIntNconf('daysago', 0); - const statshour = moment() - .subtract(daysago, 'd') - .subtract(hoursago, 'h') - .format(); - const tobedone = name ? _.filter(statsMap, { name }) : statsMap; - - const mongoc = await mongo.clientConnect(); - - debug( - 'Loaded %d possible statistics%s: %d to be done', - _.size(statsMap), - name ? `, demanded '${name}'` : '', - _.size(tobedone) - ); - - const daily = []; - for (const statinfo of tobedone) { - const dayref = aggregated.dayData(statshour); - const dayfilter = _.set({}, statinfo.timevar, { - $gte: new Date(dayref.reference), - $lt: new Date(dayref.dayOnext), - }); - const dayC = await computeCount(mongoc, statinfo, dayfilter); - debug('Day computed %s: %j', statinfo.name, dayC); - const ready = _.reduce( - dayC, - function (memo, e) { - return _.merge(memo, e); - }, - { - dayId: dayref.dayId, - day: new Date(dayref.dayOnly), - name: statinfo.name, - } - ); - const r = await mongo.upsertOne( - mongoc, - schema.stats, - { dayId: dayref.dayId, name: statinfo.name }, - ready - ); - daily.push(r); - } - - if (nconf.get('dayonly')) { - // eslint-disable-next-line no-console - console.log('--dayonly is present! quitting'); - await mongoc.close(); - return; - } - - const hourly = []; - for (const statinfo of tobedone) { - const hoursref = aggregated.hourData(statshour); - const hourfilter = _.set({}, statinfo.timevar, { - $gte: new Date(hoursref.reference), - $lt: new Date(hoursref.hourOnext), - }); - const hourC = await computeCount(mongoc, statinfo, hourfilter); - debug('Hour computed %s: %j', statinfo.name, hourC); - const entry = _.reduce( - hourC, - function (memo, e) { - return _.merge(memo, e); - }, - { - hourId: hoursref.hourId, - hour: new Date(hoursref.hourOnly), - name: statinfo.name, - } - ); - const r = await mongo.upsertOne( - mongoc, - schema.stats, - { hourId: hoursref.hourId, name: statinfo.name }, - entry - ); - hourly.push(r); - } - - await mongoc.close(); -} - -try { - start(); -} catch (error) { - debug('Unexpected error: %s', error.message); -} diff --git a/platforms/makhno/backend/bin/server.ts b/platforms/makhno/backend/bin/server.ts deleted file mode 100644 index b65f08712..000000000 --- a/platforms/makhno/backend/bin/server.ts +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env node -import { Server } from 'http'; -import nconf from 'nconf'; -import dbUtils from '../lib/dbutils'; -import security from '../lib/security'; -import { makeApp, appLogger } from './app'; -import mongo3 from '../lib/mongo3'; - -const cfgFile = 'config/settings.json'; -nconf.argv().env().file({ file: cfgFile }); - -// eslint-disable-next-line -console.log('ઉ nconf loaded, using ', cfgFile); - -if (!nconf.get('interface') || !nconf.get('port')) - throw new Error( - "check your config/settings.json, config of 'interface' and 'post' missing" - ); - -async function initialSanityChecks(): Promise { - /* security checks = is the password set and is not the default? (more checks might come) */ - security.checkKeyIsSet(); - await dbUtils.checkMongoWorks(true /* if true means that failure is fatal */); - appLogger('Makhno backend is operative!'); -} - -async function start(): Promise { - const mongo = await mongo3.clientConnect(); - - const app = await makeApp({ config: nconf.get(), mongo }); - const server = new Server(app); - /* everything starts here, welcome */ - server.listen(nconf.get('port'), nconf.get('interface')); - // eslint-disable-next-line - console.log( - ' Listening on http://' + nconf.get('interface') + ':' + nconf.get('port') - ); - - initialSanityChecks(); -} - -void start(); diff --git a/platforms/makhno/backend/config/settings.json b/platforms/makhno/backend/config/settings.json deleted file mode 100644 index 2cfafc026..000000000 --- a/platforms/makhno/backend/config/settings.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "schema": { - "urls": "urls", - "stats": "trexstats", - "results": "results", - "runs": "runs", - "emails": "emails" - }, - - "mongoHost": "localhost", - "mongoPort": 27017, - "mongoDb": "makhno", - - "port": 15000, - "interface": "0.0.0.0", - - "key": "invalid_default" -} diff --git a/platforms/makhno/backend/lib/api.js b/platforms/makhno/backend/lib/api.js deleted file mode 100644 index 13e95987d..000000000 --- a/platforms/makhno/backend/lib/api.js +++ /dev/null @@ -1,21 +0,0 @@ - -const apiList = { - submitURL: require('../routes/events').submitURL, - getMirror: require('../routes/events').getMirror, - - /* impact */ - getStatistics: require('../routes/statistics').statistics, - - /* the first two APIs for public consultation */ - getResults: require('../routes/results').getResults, - getDebugInfo: require('../routes/results').getDebugInfo, - - /* admin only interface for realtime debug */ - getMonitor: require('../routes/monitor').getMonitor, - - /* email APIs, opt-in, and who wants to be updated */ - registerEmail2: require('../routes/emails').registerEmail2, - listEmails: require('../routes/emails').listEmails, -}; - -export default apiList; \ No newline at end of file diff --git a/platforms/makhno/backend/lib/mongo3.js b/platforms/makhno/backend/lib/mongo3.js deleted file mode 100644 index 942f1a7a7..000000000 --- a/platforms/makhno/backend/lib/mongo3.js +++ /dev/null @@ -1,181 +0,0 @@ -const _ = require('lodash'); -const MongoClient = require('mongodb').MongoClient; -const debug = require('debug')('lib:mongo'); -const nconf = require('nconf'); - -let savedMongoUri = null; -function mongoUri(forced) { - - // by passing 'null' you'll reset mongoUri - if(_.isNull(forced)) - savedMongoUri = null; - - if(forced && forced.uri) - savedMongoUri = forced.uri; - - if(savedMongoUri) - return savedMongoUri; - - /* if is not yet set, reset using the config */ - const mongoHost = nconf.get('mongoHost'); - const mongoPort = nconf.get('mongoPort'); - const mongoDb = nconf.get('mongoDb'); - - if(!mongoHost || !mongoPort || !mongoDb) - throw new Error("configuration missing"); - - savedMongoUri = `mongodb://${mongoHost}:${mongoPort}/${mongoDb}`; - debug("Initializing mongoUri with %s", savedMongoUri); - return savedMongoUri; -} - -async function clientConnect(config) { - if(!config) config = {}; - - try { - const client = new MongoClient(mongoUri()); - return await client.connect(); - } catch (error) { - if (!(error instanceof Error)) { - debug("Error: %s", error); - throw new Error("Error connecting to mongo"); - } - - debug( - "mongo.clientConnect error in connecting at %s: %s", - mongoUri(), error.message, - ); - - throw error; - } -}; - -async function listCollections(mongoc) { - return mongoc - .db() - .listCollections() - .toArray(); -}; - -async function writeOne(mongoc, cName, doc) { - return mongoc - .db() - .collection(cName) - .insertOne(doc); -}; - -async function insertMany(mongoc, cName, docs, options) { - if(!options) options = {}; - return mongoc - .db() - .collection(cName) - .insertMany(docs, options); -}; - -async function updateOne(mongoc, cName, selector, updated) { - return mongoc - .db() - .collection(cName) - .updateOne(selector, { $set: updated }); -}; - -async function updateMany(mongoc, cName, selector, updated) { - return mongoc - .db() - .collection(cName) - .updateMany(selector, { $set: updated }); -} - -async function upsertOne(mongoc, cName, selector, updated) { - return mongoc - .db() - .collection(cName) - .updateOne(selector, { $set: updated }, { upsert: true}); -}; - -async function read(mongoc, cName, selector, sorter) { - return mongoc - .db() - .collection(cName) - .find(selector) - .sort(sorter || {}) - .toArray(); -}; - -async function readOne(mongoc, cName, selector, sorter) { - const l = await read(mongoc, cName, selector, sorter); - if(_.size(l) > 1) - debug("Warning, readOne %j returned %d docs", selector, _.size(l)); - return _.first(l); -}; - -async function deleteMany(mongoc, cName, selector) { - if(_.size(_.keys(selector)) === 0) - throw new Error("Not in my watch: you can't delete everything with this library"); - return mongoc - .db() - .collection(cName) - .deleteMany(selector); -}; - -async function readLimit(mongoc, cName, selector, sorter, limitN, past) { - if(!limitN) - throw new Error("Not specified the amount of documents expected"); - return mongoc - .db() - .collection(cName) - .find(selector) - .sort(sorter) - .skip(past || 0) - .limit(limitN) - .toArray(); -}; - -async function count(mongoc, cName, selector) { - return mongoc - .db() - .collection(cName) - .countDocuments(selector); -}; - - -async function createIndex(mongoc, cName, index, opt) { - return mongoc - .db() - .createIndex(cName, index, opt); -}; - -async function distinct(mongoc, cName, field, query) { - return mongoc - .db() - .collection(cName) - .distinct(field, query); -}; - -async function aggregate(mongoc, cName, pipeline) { - return mongoc - .db() - .collection(cName) - .aggregate(pipeline) - .toArray(); -}; - -module.exports = { - clientConnect, - mongoUri, - - listCollections, - writeOne, - insertMany, - updateOne, - updateMany, - upsertOne, - readOne, - read, - readLimit, - deleteMany, - count, - createIndex, - distinct, - aggregate, -}; diff --git a/platforms/makhno/backend/package.json b/platforms/makhno/backend/package.json deleted file mode 100644 index dbc632589..000000000 --- a/platforms/makhno/backend/package.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "name": "@makhno/backend", - "version": "2.6.1", - "packageManager": "yarn@3.1.1", - "description": "makhno — resist against automated content takedown", - "scripts": { - "lint": "eslint ./", - "watch": "key=fuffa DEBUG=\"*,-express:*,-body-parser:*,-send\" ts-node-dev -r tsconfig-paths/register --respawn --transpile-only bin/server", - "start": "DEBUG=\"*,-body-parser:*,-express:*,-lib:cache,-send\" ts-node bin/server", - "makhno": "DEBUG=\"*\" ts-node bin/makhno", - "makhno:watch": "DEBUG=\"*\" ts-node-dev bin/makhno" - }, - "author": "Claudio Agosti & Andrea Ascari", - "license": "AGPL-3.0", - "dependencies": { - "body-parser": "^1.19.1", - "bs58": "^3.1.0", - "cookie": "^0.3.1", - "cors": "^2.8.5", - "debug": "^4.3.4", - "express": "^4.17.2", - "food-words": "^1.1.0", - "jquery": "^3.6.0", - "lodash": "^4.17.21", - "moment": "^2.29.2", - "mongodb": "^4.3.1", - "nacl-signature": "^1.0.0", - "nconf": "^0.8.5", - "node-fetch": "^2.6.7", - "tweetnacl": "^0.14.5" - }, - "devDependencies": { - "@types/express": "^4.17.13", - "@types/lodash": "^4.14.182", - "@types/node": "^16.11.36", - "ts-node": "^10.8.0", - "ts-node-dev": "^2.0.0", - "typescript": "^4.7.2" - } -} diff --git a/platforms/makhno/backend/routes/emails.js b/platforms/makhno/backend/routes/emails.js deleted file mode 100644 index 4695b2e82..000000000 --- a/platforms/makhno/backend/routes/emails.js +++ /dev/null @@ -1,71 +0,0 @@ -const nconf = require('nconf'); -const debug = require('debug')('routes:emails'); -const _ = require('lodash'); - -const mongo3 = require('../lib/mongo3'); -const utils = require('../lib/utils'); -const security = require('../lib/security'); - -function validateEmail(email) { - const re = - /^(([^<>()[\]\\.,;:\s@"]+(\.[^<>()[\]\\.,;:\s@"]+)*)|(".+"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/; - return re.test(email); -} - -async function registerEmail2(req) { - const email = req.body.email; - const reason = req.body.reason; - const collection = nconf.get('schema').emails; - - if (!validateEmail(email)) { - debug('Rejected email address %s as invalid', email); - return { status: 403 }; - } - const reasons = ['subscribe-to-ukraine', 'press--list']; - if (reasons.indexOf(reason) === -1) { - debug('Invalid reason %s not found in %j', reason, reasons); - return { status: 403 }; - } - - const subid = utils.hash({ email, reason }); - - const mongoc = await mongo3.clientConnect({ concurrency: 1 }); - const emailAlreadyExists = await mongo3.count(mongoc, collection, { subid }); - - if (emailAlreadyExists === 1) { - debug('Address %s already present', email); - await mongoc.close(); - return { status: 201 }; - } - - await mongo3.writeOne(mongoc, collection, { - subid, - email, - registeredAt: new Date(), - reason, - }); - debug('Registed a new subscriber %s', email); - await mongoc.close(); - return { status: 200 }; -} - -async function listEmails(req) { - if (!security.checkPassword(req)) return { status: 403 }; - - const mongoc = await mongo3.clientConnect({ concurrency: 1 }); - const mails = await mongo3.read(mongoc, nconf.get('schema').emails); - - debug( - 'Fetched %d emails, reasons: %j', - mails.length, - _.countBy(mails, 'reason') - ); - await mongoc.close(); - - return { json: mails }; -} - -module.exports = { - registerEmail2, - listEmails, -}; diff --git a/platforms/makhno/backend/routes/events.js b/platforms/makhno/backend/routes/events.js deleted file mode 100644 index d576395ab..000000000 --- a/platforms/makhno/backend/routes/events.js +++ /dev/null @@ -1,139 +0,0 @@ -const _ = require('lodash'); - -const debug = require('debug')('routes:events'); -const nconf = require('nconf'); -const moment = require('moment'); - -const automo = require('../lib/automo'); -const utils = require('../lib/utils'); -const security = require('../lib/security'); - -let last = null; -function getMirror(req) { - if (!security.checkPassword(req)) return security.authError; - - if (last) { - const retval = Object(last); - last = null; - debug( - 'getMirror: authentication successfull, %d elements in volatile memory', - _.size(retval) - ); - return { json: { content: retval, elements: _.size(retval) } }; - } else debug('getMirror: auth OK, but nothing to be returned'); - - return { json: { content: null } }; -} -function appendLast(req) { - /* this is used by getMirror, to mirror what the server is getting - * used by developers with password */ - const MAX_STORED_CONTENT = 15; - if (!last) last = []; - if (_.size(last) > MAX_STORED_CONTENT) last = _.tail(last); - - last.push(_.pick(req, ['headers', 'body'])); -} - -async function saveInDB(objects, dbcollection) { - if (!objects.length) - throw new Error("Internal Error: no data"); - - try { - await automo.write(dbcollection, objects); - debug( - 'Saved %d objects in [%s] timelineId %j', - objects.length, - dbcollection); - - return { - error: false, - success: objects.length, - subject: dbcollection, - }; - } catch (error) { - if (!(error instanceof Error)) { - debug('Error in saveInDB: %s', error); - return { - error: error, - message: 'error in saveInDB', - subject: dbcollection, - }; - } - - debug( - 'Error in saving %d %s %j', - objects.length, - dbcollection, - error.message - ); - return { error: true, message: error.message }; - } -} - -function scheduleRun(urlo, minutesOffset, i) { - return _.map(['ru', 'ua', 'fr', 'de', 'pl'], function(twoLetterCountryCode) { - const runId = utils.hash({ - urlId: urlo.urlId, - minutesOffset, - twoLetterCountryCode - }); - return { - runAt: moment().add(minutesOffset, 'minutes').toISOString(), - cc: twoLetterCountryCode, - ...urlo, - runId, - state: 'waiting' - } - }); -} - -async function submitURL(req) { - - // this is necessary for the mirror functionality - appendLast(req); - - const url = req.body.url; - - // this function has to: - // 1) evaluate integrity and URL nature - // 2) save in 'urls', and if new - // 3) allocate 'runs' - - const urlId = utils.hash({url}); - debug('[+] received URL (%s)', url); - - const urlo = { - url, - urlId, - savingTime: moment().toISOString() - }; - - const retv = {}; - try { - const urlrv = await saveInDB([ urlo ], nconf.get('schema').urls); - - // this plan the tests in 1, 5 and 15 minutes (because is a test) - const runs = _.flatten(_.map([1, 5, 15], _.partial(scheduleRun, urlo))); - const runrv = await saveInDB(runs, nconf.get('schema').runs); - debug("DB action results: %j %j", urlrv, runrv); - retv.text = "Submission successful"; - retv.urlId = urlId; - retv.success = true; - } catch(error) { - debug("Error in handling new URL submission: %s", error); - retv.message = "Error"; - retv.error = error; - retv.success = false; - retv.urlId = urlId; - } - - /* this is what returns to the website */ - return { - json: retv - }; -} - -module.exports = { - submitURL, - getMirror, -}; diff --git a/platforms/makhno/backend/routes/results.js b/platforms/makhno/backend/routes/results.js deleted file mode 100644 index 18dec38a7..000000000 --- a/platforms/makhno/backend/routes/results.js +++ /dev/null @@ -1,138 +0,0 @@ -const _ = require('lodash'); -const debug = require('debug')('routes:personal'); -const nconf = require('nconf'); - -const utils = require('../lib/utils'); -const mongo = require('../lib/mongo3'); -const CSV = require('../lib/CSV'); - -async function getDebugInfo(req) { - return { json: { - error: true, - messasge: 'Not implemented yet' - }}; -} -async function getResults(req) { - // personal API format is - // /api/v1/results/:urlpattern? - const up = req.params.urlpattern; - const options = { amount: 100, skip: 0 }; - debug('Requested results for [%s]', up); - - try { - - const mongoc = await mongo.clientConnect({concurrency: 1}); - const results = await mongo.readLimit(mongoc, - nconf.get('schema').results, { urlId: up }, { savingTime: -1 }, - options.amount, options.skip); - const runs = await mongo.readLimit(mongoc, - nconf.get('schema').runs, { urlId: up}, { savingTime: -1}, - options.amount, options.skip); - await mongoc.close(); - - if(!results && !runs) - return { - json: { success: false, error: "This test do not seem scheduled for a test" } - } - - debug("%j", runs); - // else we'll have material to display - return { json: { - success: true, - runs, - results: results - } }; - } catch (error) { - const message = error instanceof Error ? error.message : 'unknown error'; - debug('getPersonal handled error: %s', message); - return { json: { - success: false, - message - } }; - } -} - -/* -async function getPersonalCSV(req) { - const CSV_MAX_SIZE = 9000; - const k = req.params.publicKey; - const type = req.params.what; - - if (['foryou', 'search', 'following', 'profile'].indexOf(type) === -1) - return { text: 'Error, nature not supported ' }; - - const data = await automo.getMetadataByFilter( - { publicKey: k, type }, - { amount: CSV_MAX_SIZE, skip: 0 } - ); - - if (!data.length) { - debug("getPersonalCSV didn't found DB entry matching %o", { - publicKey: k, - type, - }); - return { text: 'No data not found in the DB' }; - } - - debug( - 'type [%s] return %d with amount %d skip-zero', - type, - data.length, - CSV_MAX_SIZE - ); - - if (type === 'search') unrolledData = _.reduce(data, flattenSearch, []); - else if (type === 'profile') - unrolledData = _.reduce(data, flattenProfile, []); - else unrolledData = _.map(data, pickFeedFields); - - if (!unrolledData.length) { - debug( - 'getPersonalCSV produced empty data during transformation: investigate parsers and pipeline!' - ); - return { text: 'Data not found, from metadata: ' + data.length }; - } - - const pseudo = utils.string2Food(unrolledData[0].publicKey); - const ready = _.map(unrolledData, function (e) { - e.pseudo = pseudo; - if (_.isString(e.sharen)) e.sharen = 0; - return e; - }); - - // console.table(ready); - const csv = CSV.produceCSVv1(ready); - - debug( - 'getPersonalCSV produced %d entries from %d metadata (type %s), %d bytes (max %d)', - ready.length, - data.length, - type, - csv.length, - CSV_MAX_SIZE - ); - - const filename = - 'tk-' + - type + - '-' + - moment().format('YY-MM-DD') + - '--' + - ready.length + - '.csv'; - - return { - headers: { - 'Content-Type': 'csv/text', - 'Content-Disposition': 'attachment; filename=' + filename, - }, - text: csv, - }; -} -*/ - -module.exports = { - getDebugInfo, - getResults, - // getPersonalCSV, -}; diff --git a/platforms/makhno/backend/routes/statistics.js b/platforms/makhno/backend/routes/statistics.js deleted file mode 100644 index 9e5cbb328..000000000 --- a/platforms/makhno/backend/routes/statistics.js +++ /dev/null @@ -1,80 +0,0 @@ -const _ = require('lodash'); -const moment = require('moment'); -const debug = require('debug')('routes:statistics'); -const nconf = require('nconf'); - -const mongo3 = require('../lib/mongo3'); -const cache = require('../lib/cache'); - -async function statistics(req) { - // the content in 'stats' is saved by count-o-clock and the selector here required is - // specifiy in config/stats.json - const expectedFormat = "/api/v2/statistics/:name/:unit/:amount"; - - const name = req.params.name; - if(cache.allowedNames.indexOf(name) === -1) { - debug("Error! this might not appear in visualization: investigate on why an invalid stat-name is called by c3! (%s)", name); - return { json: { - error: true, - expectedFormat, - allowedNames: cache.allowedNames, - note: `the statistic name you look for was ${name}` - } }; - } - - const unit = req.params.unit; - const allowedRanges = ['hours', 'hour', 'day', 'days']; - if(allowedRanges.indexOf(unit) === -1 ) { - debug("Error! this might not appear in visualization, but the API call has a malformed time-unit!"); - return { json: { error: true, expectedFormat, allowedRanges, note: `the statistic unit you look for was ${unit}` }} - } - - const amount = _.parseInt(req.params.amount); - if(_.isNaN(amount)) { - debug("Error! this might not appear in visualization, but the API call has an invalid number!"); - return { json: { error: true, expectedFormat, invalidNumber: req.params.amount }}; - } - - if(cache.stillValid(name)) { - const content = cache.repullCache(name).content; - debug("Cached [%s] since %d %s ago = %d measures", - name, amount, unit, _.size(content)); - return { - json: content, - } - } - - const filter = { name }; - const refDate = new Date(+moment().subtract(amount, _.nth(unit, 0))); - - if(_.startsWith(unit, 'day')) - _.set(filter, 'day', { '$gt': refDate }); - else - _.set(filter, 'hour', { '$gt': refDate }); - - const mongoc = await mongo3.clientConnect({concurrency: 1}); - const fullc = await mongo3.read(mongoc, nconf.get('schema').stats, filter); - const content = _.map(fullc, function(e) { - /* this is helpful to avoid lines to the bottom, and just - * make the line disappear as missing evidence, that's why - * keys/entry with zero entry get removed. */ - _.each(_.keys(e), function(k) { - if(e[k] === 0 || k === '_id') - _.unset(e, k); - }) - return e; - }); - - debug("Requested [%s] since %d %s ago = %d measures", name, amount, unit, _.size(content)); - cache.setCache(name, content); - await mongoc.close(); - return { json: content, - // headers: { amount, unit, name } - // there is no reason to add info in the header, - // but that was also a standard in all the *trex backends - }; -} - -module.exports = { - statistics -}; diff --git a/platforms/makhno/backend/scripts/build-indexes.js b/platforms/makhno/backend/scripts/build-indexes.js deleted file mode 100644 index 1e60e38ed..000000000 --- a/platforms/makhno/backend/scripts/build-indexes.js +++ /dev/null @@ -1,23 +0,0 @@ -ret = db.urls.createIndex({ urlId: 1 }, { unique: true }); -checkret('urls urlId', ret); -ret = db.urls.createIndex({ savingTime: -1 }); -checkret('urls savingTime', ret); - -ret = db.runs.createIndex({ runId: 1 }, { unique: true }); -checkret('runs runId', ret); -ret = db.runs.createIndex({ savingTime: -1 }); -checkret('runs savingTime', ret); -ret = db.runs.createIndex({ urlId: 1 }); -checkret('runs urlId', ret); -ret = db.runs.createIndex({ runAt: -1 }); -checkret('runs runAt', ret); - -ret = db.results.createIndex({ urlId: 1 }); -checkret('results urlId', ret); -ret = db.results.createIndex({ savingTime: -1 }); -checkret('results savingTime', ret); - -function checkret(info, retval) { - retval.info = info; - printjson(retval); -} diff --git a/platforms/makhno/backend/tsconfig.json b/platforms/makhno/backend/tsconfig.json deleted file mode 100644 index fee5b4699..000000000 --- a/platforms/makhno/backend/tsconfig.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "extends": "../../../tsconfig.json", - "compilerOptions": { - "allowJs": true, - "baseUrl": ".", - "composite": true, - "esModuleInterop": true, - "jsx": "react", - "module": "esnext", - "moduleResolution": "node", - "noImplicitAny": false, - "noEmit": true, - "resolveJsonModule": true, - "strict": true, - "skipLibCheck": true, - "target": "es2015", - "paths": { - "@shared/*": ["../../../packages/shared/src/*"], - "@tktrex/shared/*": ["../shared/src/*"] - } - }, - "references": [ - { - "path": "../../../packages/shared" - }, - { - "path": "../shared" - } - ], - "ts-node": { - "require": ["tsconfig-paths/register"], - "compilerOptions": { - "module": "CommonJS" - } - }, - "include": ["**/*.ts", "**/*.js", "**/*.json"], - "exclude": ["node_modules", "build", "jest.config.js"] -} diff --git a/platforms/tktrex/backend/bin/app.ts b/platforms/tktrex/backend/bin/app.ts index 6a322d5c4..58588ca08 100644 --- a/platforms/tktrex/backend/bin/app.ts +++ b/platforms/tktrex/backend/bin/app.ts @@ -5,73 +5,11 @@ import D from 'debug'; import cors from 'cors'; import path from 'path'; import bodyParser from 'body-parser'; -import _ from 'lodash'; +import { routeHandleMiddleware } from '@shared/backend/utils/routeHandlerMiddleware'; export const appLogger = D('tktrex'); -async function iowrapper(fname, req, res): Promise { - try { - const funct = apiList[fname]; - const httpresult = await funct(req, res); - - if (httpresult.headers) - _.each(httpresult.headers, function (value, key) { - appLogger('Setting header %s: %s', key, value); - res.setHeader(key, value); - }); - - if (!httpresult) { - appLogger("API (%s) didn't return anything!?", fname); - res.send('Fatal error: Invalid output'); - res.status(501); - } else if (httpresult.json?.error) { - appLogger('API (%s) failure, returning 500', fname); - res.status(500); - res.json(httpresult.json); - } else if (httpresult.json) { - appLogger( - 'API (%s) success, returning %d bytes JSON', - fname, - _.size(JSON.stringify(httpresult.json)) - ); - res.setHeader('Access-Control-Allow-Origin', '*'); - res.json(httpresult.json); - } else if (httpresult.text) { - appLogger( - 'API (%s) success, returning text (size %d)', - fname, - _.size(httpresult.text) - ); - res.send(httpresult.text); - } else if (httpresult.status) { - appLogger( - 'Returning empty status %d from API (%s)', - httpresult.status, - fname - ); - res.status(httpresult.status); - } else { - appLogger('Undetermined failure in API (%s) → %j', fname, httpresult); - res.status(502); - res.send('Error?'); - } - } catch (error) { - res.status(505); - if (error instanceof Error) { - res.send('Software error: ' + error.message); - appLogger( - 'Error in HTTP handler API(%s): %s %s', - fname, - error.message, - error.stack - ); - } else { - res.send('Unknown software error.'); - appLogger('Unknown error in HTTP handler API(%s): %s', fname, error); - } - } - res.end(); -} +const iowrapper = routeHandleMiddleware(apiList); export const makeApp = async ( ctx: MakeAppContext @@ -87,140 +25,78 @@ export const makeApp = async ( /* this API is v0 as it is platform neutral. it might be shared among * all the trex backends, and should return info on system health, echo OK * if the system is OK, and the git log of the code running */ - app.get( - '/api/v0/info', - async (req, res) => await iowrapper('systemInfo', req, res) - ); app.get('/api/v0/health', function (req, res) { res.send('OK'); res.status(200); }); /* This is the API meant to receive data donation */ - app.post( - '/api/v2/events', - async (req, res) => await iowrapper('processEvents', req, res) - ); - app.post( - '/api/v2/apiEvents', - async (req, res) => await iowrapper('processAPIEvents', req, res) - ); - app.post( - '/api/v2/handshake', - async (req, res) => await iowrapper('handshake', req, res) - ); + app.post('/api/v2/events', iowrapper('processEvents')); + app.post('/api/v2/apiEvents', iowrapper('processAPIEvents')); + app.post('/api/v2/handshake', iowrapper('handshake')); - app.get( - '/api/v2/recent', - async (req, res) => await iowrapper('getRecent', req, res) - ); + app.get('/api/v2/recent', iowrapper('getRecent')); - app.get( - '/api/v2/statistics/:name/:unit/:amount', - async (req, res) => await iowrapper('getStatistics', req, res) - ); + app.get('/api/v2/statistics/:name/:unit/:amount', iowrapper('getStatistics')); /* debug API */ - app.get( - '/api/v2/debug/html/:htmlId', - async (req, res) => await iowrapper('unitById', req, res) - ); - app.get( - '/api/v1/mirror/:key', - async (req, res) => await iowrapper('getMirror', req, res) - ); + app.get('/api/v2/debug/html/:htmlId', iowrapper('unitById')); + app.get('/api/v1/mirror/:key', iowrapper('getMirror')); /* monitor for admin */ - app.get( - '/api/v2/monitor/:minutes?', - async (req, res) => await iowrapper('getMonitor', req, res) - ); + app.get('/api/v2/monitor/:minutes?', iowrapper('getMonitor')); /* special function for researchers */ app.get( '/api/v2/research/:publicKeyList/:what/csv', - async (req, res) => await iowrapper('getResearcherData', req, res) + iowrapper('getResearcherData') ); /* experiments API: at the moment not implemented for tiktok, only copied by yttrex */ app.get( '/api/v2/guardoni/list/:directiveType/:key?', - async (req, res) => await iowrapper('getAllExperiments', req, res) - ); - app.get('/api/v3/directives/public', async (req, res) => - iowrapper('getPublicDirectives', req, res) - ); - app.post( - '/api/v3/directives/:directiveType', - async (req, res) => await iowrapper('postDirective', req, res) - ); - app.get( - '/api/v3/directives/:experimentId', - async (req, res) => await iowrapper('fetchDirective', req, res) - ); - app.post( - '/api/v2/handshake', - async (req, res) => await iowrapper('experimentChannel3', req, res) - ); - app.delete( - '/api/v3/experiment/:testTime', - async (req, res) => await iowrapper('concludeExperiment3', req, res) - ); - app.get( - '/api/v2/experiment/:experimentId/json', - async (req, res) => await iowrapper('experimentJSON', req, res) + iowrapper('getAllExperiments') ); + app.get('/api/v3/directives/public', iowrapper('getPublicDirectives')); + app.post('/api/v3/directives/:directiveType', iowrapper('postDirective')); + app.get('/api/v3/directives/:experimentId', iowrapper('fetchDirective')); + app.post('/api/v2/handshake', iowrapper('experimentChannel3')); + + app.get('/api/v2/experiment/:experimentId/json', iowrapper('experimentJSON')); app.get( '/api/v2/experiment/:experimentId/csv/:type', - async (req, res) => await iowrapper('experimentCSV', req, res) - ); - app.get( - '/api/v2/experiment/:experimentId/dot', - async (req, res) => await iowrapper('experimentDOT', req, res) + iowrapper('experimentCSV') ); + app.get('/api/v2/experiment/:experimentId/dot', iowrapper('experimentDOT')); /* subscription email */ // Improved from the version initially used in ycai, same payload, different behvior: // apiRouter.post('/v3/registerEmail', iowrapper('registerEmail2')); - app.post( - '/api/v1/registerEmail2', - async (req, res) => await iowrapper('registerEmail2', req, res) - ); - app.get( - '/api/v1/listEmails/:key', - async (req, res) => await iowrapper('listEmails', req, res) - ); + app.post('/api/v1/registerEmail2', iowrapper('registerEmail2')); + app.get('/api/v1/listEmails/:key', iowrapper('listEmails')); /* ============== Documented only the API below ============== */ /* TODO the JSON was v1 and should be fixed in site, the what should be a query string, should be timed params */ app.get( '/api/v2/personal/:publicKey/experiments/:experimentId/:format', - async (req, res) => await iowrapper('getPersonalByExperimentId', req, res) + iowrapper('getPersonalByExperimentId') ); app.get( '/api/v[1-2]/personal/:publicKey/:what/json', - async (req, res) => await iowrapper('getPersonal', req, res) - ); - app.get( - '/api/v2/personal/:publicKey/:what/csv', - async (req, res) => await iowrapper('getPersonalCSV', req, res) + iowrapper('getPersonal') ); + app.get('/api/v2/personal/:publicKey/:what/csv', iowrapper('getPersonalCSV')); /* implemented for DMI winter school, supported in Taboule, trimming in progress */ - app.get( - '/api/v2/public/searches', - async (req, res) => await iowrapper('getSearches', req, res) - ); + app.get('/api/v2/public/searches', iowrapper('getSearches')); app.get( '/api/v2/public/query/:string/:format', - async (req, res) => await iowrapper('getSearchByQuery', req, res) - ); - app.get( - '/api/v2/public/queries/list', - async (req, res) => await iowrapper('getQueryList', req, res) + iowrapper('getSearchByQuery') ); + app.get('/api/v2/public/queries/list', iowrapper('getQueryList')); + app.get('/api/v2/metadata', iowrapper('listMetadata')); /* --------------- end of documented APIs -------------------- */ diff --git a/platforms/tktrex/backend/config/settings.json b/platforms/tktrex/backend/config/settings.json index 9e3b7458e..e5ef60e14 100644 --- a/platforms/tktrex/backend/config/settings.json +++ b/platforms/tktrex/backend/config/settings.json @@ -5,9 +5,8 @@ "htmls": "htmls", "metadata": "metadata", "full": "full", - "directives": "directives", "emails": "emails2", - "experiments": "experiments" + "experiments": "experiments2" }, "mongoHost": "localhost", diff --git a/platforms/tktrex/backend/config/trexstats.json b/platforms/tktrex/backend/config/trexstats.json index 4b037d05c..557e6fce8 100644 --- a/platforms/tktrex/backend/config/trexstats.json +++ b/platforms/tktrex/backend/config/trexstats.json @@ -81,5 +81,11 @@ "selector": { } } ] + }, + { + "name": "experiments", + "column": "experiments2", + "timevar": "when", + "variables": [{ "name": "total", "selector": {} }] }] } diff --git a/platforms/tktrex/backend/ecosystem.config.js b/platforms/tktrex/backend/ecosystem.config.js index 727df73c0..8f23c03db 100644 --- a/platforms/tktrex/backend/ecosystem.config.js +++ b/platforms/tktrex/backend/ecosystem.config.js @@ -1,5 +1,5 @@ const testEnv = { - mongoDb: 'tktrex', + mongoDb: 'tktrex-test', }; const tk = { diff --git a/platforms/tktrex/backend/lib/api.js b/platforms/tktrex/backend/lib/api.js index 90bcbd2e8..7a2460b86 100644 --- a/platforms/tktrex/backend/lib/api.js +++ b/platforms/tktrex/backend/lib/api.js @@ -1,4 +1,5 @@ import * as publicRoutes from '../routes/public'; +import * as metadataRoutes from '../routes/metadata'; const apiList = { processEvents: require('../routes/events').processEvents, @@ -9,12 +10,16 @@ const apiList = { /* for revision */ unitById: require('../routes/htmlunit').unitById, - getVideoId: publicRoutes.getVideoId, + systemInfo: publicRoutes.systemInfo, getRelated: publicRoutes.getRelated, getVideoCSV: publicRoutes.getVideoCSV, + getSearches: publicRoutes.getSearches, + getVideoId: publicRoutes.getVideoId, + getRecent: publicRoutes.getRecent, + + ...metadataRoutes, /* changes made in emergency during the winter school - might be reviewed */ - getSearches: publicRoutes.getSearches, getPersonal: require('../routes/personal').getPersonal, getPersonalCSV: require('../routes/personal').getPersonalCSV, @@ -29,19 +34,21 @@ const apiList = { /* work in progress, admin, tag */ getMonitor: require('../routes/monitor').getMonitor, - /* experiment related APIs -- implemented but not really - integrated with guardoni: don't rely on them, they are a - simple copy, paritally TypeScripted review, from yt */ + // experiment related APIs -- implemented but not really + // integrated with guardoni: don't rely on them, they are a + // simple copy, paritally TypeScripted review, from yt getAllExperiments: require('../routes/experiments').list, // experimentCSV: require('../routes/experiments').csv, experimentDOT: require('../routes/experiments').dot, experimentJSON: require('../routes/experiments').json, + experimentCSV: require('../routes/experiments').csv, experimentChannel3: require('../routes/experiments').channel3, concludeExperiment3: require('../routes/experiments').conclude3, postDirective: require('../routes/directives').post, fetchDirective: require('../routes/directives').get, getPublicDirectives: require('../routes/directives').getPublic, - getPersonalByExperimentId : require('../routes/personal').getPersonalByExperimentId, + getPersonalByExperimentId: + require('../routes/personal').getPersonalByExperimentId, /* and specificly for the email, opt-in, and who wants to be get updated */ registerEmail2: require('../routes/emails').registerEmail2, listEmails: require('../routes/emails').listEmails, diff --git a/platforms/tktrex/backend/lib/experiments.js b/platforms/tktrex/backend/lib/experiments.js index 118e9b47c..69f76aaa4 100644 --- a/platforms/tktrex/backend/lib/experiments.js +++ b/platforms/tktrex/backend/lib/experiments.js @@ -11,7 +11,7 @@ const mongo3 = require('./mongo3'); async function pickDirective(experimentId) { const mongoc = await mongo3.clientConnect({ concurrency: 1 }); - const rb = await mongo3.readOne(mongoc, nconf.get('schema').directives, { + const rb = await mongo3.readOne(mongoc, nconf.get('schema').experiments, { experimentId, }); await mongoc.close(); @@ -25,7 +25,7 @@ async function registerDirective(directives) { directives, }); const mongoc = await mongo3.clientConnect({ concurrency: 1 }); - const exist = await mongo3.readOne(mongoc, nconf.get('schema').directives, { + const exist = await mongo3.readOne(mongoc, nconf.get('schema').experiments, { experimentId, }); @@ -46,7 +46,7 @@ async function registerDirective(directives) { /* else, we don't had such data, hence */ debug('Registering new experiment %s: %j', experimentId, directives); - await mongo3.writeOne(mongoc, nconf.get('schema').directives, { + await mongo3.writeOne(mongoc, nconf.get('schema').experiments, { when: new Date(), directives, experimentId, diff --git a/platforms/tktrex/backend/package.json b/platforms/tktrex/backend/package.json index 380b4fa44..0cbba6dcc 100644 --- a/platforms/tktrex/backend/package.json +++ b/platforms/tktrex/backend/package.json @@ -6,6 +6,7 @@ "scripts": { "lint": "eslint ./", "test": "jest", + "coc": "DEBUG=* ts-node bin/count-o-clock.js", "tto:watch": "key=fuffa DEBUG=\"*,-express:*,-body-parser:*,-send\" ts-node-dev -r tsconfig-paths/register --respawn --transpile-only bin/observatory", "tto:start": "DEBUG=\"*,-express:*,-body-parser:*,-send\" ts-node bin/observatory", "watch": "key=fuffa DEBUG=\"*,-express:*,-body-parser:*,-send\" ts-node-dev -r tsconfig-paths/register --respawn --transpile-only bin/server", @@ -21,7 +22,7 @@ "license": "AGPL-3.0", "dependencies": { "body-parser": "^1.19.1", - "bs58": "^3.1.0", + "bs58": "^4.0.1", "cookie": "^0.3.1", "cors": "^2.8.5", "debug": "^4.3.4", @@ -32,7 +33,7 @@ "moment": "^2.29.2", "mongodb": "^4.3.1", "nacl-signature": "^1.0.0", - "nconf": "^0.8.5", + "nconf": "^0.11.3", "node-fetch": "^2.6.7", "nodemon": "^1.19.4", "tweetnacl": "^0.14.5" @@ -41,8 +42,8 @@ "@types/express": "^4.17.13", "@types/lodash": "^4.14.182", "@types/node": "^16.11.36", - "ts-node": "^10.8.0", + "ts-node": "^10.8.2", "ts-node-dev": "^2.0.0", - "typescript": "^4.7.2" + "typescript": "^4.7.4" } } diff --git a/platforms/tktrex/backend/routes/__tests__/events.e2e.ts b/platforms/tktrex/backend/routes/__tests__/events.e2e.ts index a73d2708e..464176b23 100644 --- a/platforms/tktrex/backend/routes/__tests__/events.e2e.ts +++ b/platforms/tktrex/backend/routes/__tests__/events.e2e.ts @@ -19,6 +19,7 @@ describe('Events', () => { describe('Add events', () => { test('succeeds when payload is correctly signed', async () => { const keys = await foldTEOrThrow(bs58.makeKeypair('')); + const researchTag = 'test-tag'; const data = fc.sample(ContributionEventArb, 5); const signature = await foldTEOrThrow( @@ -31,6 +32,7 @@ describe('Events', () => { .set('x-tktrex-version', version) .set('X-Tktrex-publicKey', keys.publicKey) .set('x-tktrex-signature', signature) + .set('X-Tktrex-NonAuthCookieId', researchTag) .send(data); expect(response.status).toBe(200); diff --git a/platforms/tktrex/backend/routes/__tests__/metadata.e2e.ts b/platforms/tktrex/backend/routes/__tests__/metadata.e2e.ts new file mode 100644 index 000000000..e5faf940f --- /dev/null +++ b/platforms/tktrex/backend/routes/__tests__/metadata.e2e.ts @@ -0,0 +1,152 @@ +/* eslint-disable import/first */ +// mock curly module +jest.mock('fetch-opengraph'); + +// import test utils +import { fc } from '@shared/test'; +import { MetadataArb } from '@tktrex/shared/arbitraries/Metadata.arb'; +import moment from 'moment'; +import { GetTest, Test } from '../../test/Test'; + +describe('Metadata API', () => { + let test: Test; + + beforeAll(async () => { + test = await GetTest(); + }); + + afterAll(async () => { + await test.mongo.close(); + }); + + describe('/v2/metadata', () => { + it('fails with invalid `amount`', async () => { + const { body } = await test.app + .get(`/api/v2/metadata`) + .query({ + amount: 'invalid', + }) + .expect(400); + + expect(body).toMatchObject({ + name: 'Bad Request', + }); + }); + + it('succeeds with valid `researchTag` ', async () => { + const researchTag = 'test-tag'; + const experimentId = fc.sample(fc.uuid(), 1)[0]; + const amount = 10; + + const metadataWithExperimentId = fc.sample(MetadataArb, 100).map((m) => ({ + ...m, + savingTime: new Date(), + experimentId, + })); + + const metadataWithResearchTag = fc.sample(MetadataArb, 100).map((m) => ({ + ...m, + savingTime: new Date(), + researchTag, + })); + + await test.mongo3.insertMany( + test.mongo, + test.config.get('schema').metadata, + [...metadataWithResearchTag, ...metadataWithExperimentId] + ); + + const expectedMetadata = metadataWithResearchTag + .sort((a, b) => b.savingTime.getTime() - a.savingTime.getTime()) + .slice(0, amount) + .map((m) => { + return { + ...m, + clientTime: m.clientTime.toISOString(), + savingTime: m.savingTime.toISOString(), + }; + }); + + const { body } = await test.app + .get(`/api/v2/metadata`) + .query({ + researchTag, + amount, + }) + .expect(200); + + expect(body.length).toBe(amount); + expect(body).toMatchObject(expectedMetadata); + + await test.mongo3.deleteMany( + test.mongo, + test.config.get('schema').metadata, + { + id: { + $in: [...metadataWithExperimentId, ...metadataWithResearchTag].map( + (m) => m.id + ), + }, + } + ); + }); + + it('succeeds with valid `experimentId` ', async () => { + const researchTag = 'test-tag'; + const experimentId = fc.sample(fc.uuid(), 1)[0]; + const amount = 10; + + const metadataWithExperimentId = fc.sample(MetadataArb, 100).map((m) => ({ + ...m, + savingTime: new Date(), + experimentId, + })); + + const metadataWithResearchTag = fc.sample(MetadataArb, 100).map((m) => ({ + ...m, + savingTime: new Date(), + researchTag, + })); + + await test.mongo3.insertMany( + test.mongo, + test.config.get('schema').metadata, + [...metadataWithResearchTag, ...metadataWithExperimentId] + ); + + const expectedMetadata = metadataWithExperimentId + .sort((a, b) => b.savingTime.getTime() - a.savingTime.getTime()) + .slice(0, amount) + .map((m) => { + return { + ...m, + clientTime: m.clientTime.toISOString(), + savingTime: m.savingTime.toISOString(), + }; + }); + + const { body } = await test.app + .get(`/api/v2/metadata`) + .query({ + experimentId, + amount, + }) + .expect(200); + + expect(body.length).toBe(amount); + expect(body).toMatchObject(expectedMetadata); + + await test.mongo3.deleteMany( + test.mongo, + test.config.get('schema').metadata, + { + id: { + $in: [...metadataWithExperimentId, ...metadataWithResearchTag].map( + (m) => m.id + ), + }, + } + ); + }); + }); +}); diff --git a/platforms/tktrex/backend/routes/__tests__/personal.e2e.ts b/platforms/tktrex/backend/routes/__tests__/personal.e2e.ts index c53e86b77..e8729d516 100644 --- a/platforms/tktrex/backend/routes/__tests__/personal.e2e.ts +++ b/platforms/tktrex/backend/routes/__tests__/personal.e2e.ts @@ -16,7 +16,7 @@ describe('Events', () => { appTest = await GetTest(); await appTest.mongo3.insertMany( appTest.mongo, - appTest.config.get('schema').directives, + appTest.config.get('schema').experiments, [experiment] ); }); @@ -29,6 +29,8 @@ describe('Events', () => { describe('GetPersonalByExperimentId', () => { test('succeeds with one metadata', async () => { + const researchTag= "test-tag"; + const keys = await foldTEOrThrow(bs58.makeKeypair('')); const data = fc.sample(ContributionEventArb, 1).map((d) => ({ ...d, @@ -49,7 +51,6 @@ describe('Events', () => { html: '

\n
\n
\n
\n
\n
\n
\n

\n yuuna_1210\n

\n

\n 悠那🌹🌕\n

\n
\n \n
\n 🌹🌕#おすすめ\n #コスプレ\n #制服\n \n
\n

\n \n TJR Eat God See Acid Noslek Milkshake Edit - Даба\n Дамбиев\n

\n
\n
\n
\n \n
\n
\n 🌹🌕#おすすめ #コスプレ #制服\n
\n \n
\n
\n
\n \n \n \n \n
\n
\n
\n \n \n \n
\n
\n
\n

\n \n Report\n

\n
\n
\n
\n \n
\n
\n
\n
', n: [1, 0, 0, 0, 8829], geoip: null, - researchTag: null, experimentId: experiment.experimentId, })); @@ -62,8 +63,9 @@ describe('Events', () => { await appTest.app .post(`/api/v2/events`) .set('x-tktrex-version', version) - .set('X-Tktrex-publicKey', keys.publicKey) + .set('x-tktrex-publicKey', keys.publicKey) .set('x-tktrex-signature', signature) + .set('X-Tktrex-NonAuthCookieId', researchTag ?? '') .send(data) .expect(200); diff --git a/platforms/tktrex/backend/routes/directives.js b/platforms/tktrex/backend/routes/directives.js index 3d831962a..8665942b7 100644 --- a/platforms/tktrex/backend/routes/directives.js +++ b/platforms/tktrex/backend/routes/directives.js @@ -53,7 +53,7 @@ async function getPublic(req) { // TODO this API can also be cached with the same logic of statistics const publicDirectives = await mongo3.readLimit( mongoc, - nconf.get('schema').directives, + nconf.get('schema').experiments, {}, // TODO { visibility: 'public' }, { when: -1 }, 20, diff --git a/platforms/tktrex/backend/routes/experiments.js b/platforms/tktrex/backend/routes/experiments.js index 2d7671641..1b3c3f91c 100644 --- a/platforms/tktrex/backend/routes/experiments.js +++ b/platforms/tktrex/backend/routes/experiments.js @@ -3,25 +3,34 @@ const moment = require('moment'); const debug = require('debug')('routes:experiments'); const nconf = require('nconf'); -const automo = require('../lib/automo'); +const CSV = require('../lib/CSV'); const experlib = require('../lib/experiments'); const params = require('../lib/params'); -const CSV = require('../lib/CSV'); const mongo3 = require('../lib/mongo3'); const security = require('../lib/security'); async function sharedDataPull(filter) { - /* this function is invoked by the various API below */ - const MAX = 3000; - const mongoc = await mongo3.clientConnect({concurrency: 1}); - const metadata = await mongo3 - .readLimit(mongoc, nconf.get('schema').metadata, - filter, { savingTime: -1 }, MAX, 0); - await mongoc.close(); - - debug("Found %d available data by filter %o (max %d) %j", - metadata.length, filter, MAX, _.countBy(metadata, 'type')); - return metadata; + /* this function is invoked by the various API below */ + const MAX = 3000; + const mongoc = await mongo3.clientConnect({ concurrency: 1 }); + const metadata = await mongo3.readLimit( + mongoc, + nconf.get('schema').metadata, + filter, + { savingTime: -1 }, + MAX, + 0 + ); + await mongoc.close(); + + debug( + 'Found %d available data by filter %o (max %d) %j', + metadata.length, + filter, + MAX, + _.countBy(metadata, 'type') + ); + return metadata; } // function dotify(data) { @@ -48,191 +57,228 @@ async function sharedDataPull(filter) { // } async function dot(req) { - - throw new Error("Remind this can't work because metadata has many type"); - - // const experiment = params.getString(req, 'experimentId', true); - // const metadata = await sharedDataPull(experiment); - - // if(!_.size(related)) - // return { json: {error: true, message: "No data found with such parameters"}} - - // const grouped = _.groupBy(related, 'videoName'); - // const dotchain = _.map(grouped, function(vidlist, videoName) { - // return { - // videoName, - // dotted: dotify(vidlist) - // }; - // }) - // return { json: dotchain }; + throw new Error("Remind this can't work because metadata has many type"); + + // const experiment = params.getString(req, 'experimentId', true); + // const metadata = await sharedDataPull(experiment); + + // if(!_.size(related)) + // return { json: {error: true, message: "No data found with such parameters"}} + + // const grouped = _.groupBy(related, 'videoName'); + // const dotchain = _.map(grouped, function(vidlist, videoName) { + // return { + // videoName, + // dotted: dotify(vidlist) + // }; + // }) + // return { json: dotchain }; } async function json(req) { - const experimentId = params.getString(req, 'experimentId'); - const metadata = await sharedDataPull({ - 'experiment.experimentId': experimentId - }); - return { json: metadata} + const experimentId = params.getString(req, 'experimentId'); + const metadata = await sharedDataPull({ + 'experiment.experimentId': experimentId, + }); + return { json: metadata }; } -/* async function csv(req) { - - const type = req.params.type; - if(CSV.allowedTypes.indexOf(type) === -1) { - debug("Invalid requested data type? %s", type); - return { text: "Error, invalid URL composed" }; - } - - const experimentId = params.getString(req, 'experimentId', true); - const metadata = await sharedDataPull({ - 'experiment.experimentId': experimentId, type - }); - - const transformed = CSV.unrollNested(metadata, { - type, experiment: true, private: true - }); - - const textcsv = CSV.produceCSVv1(transformed); - debug("Fetch %d metadata(s), and converted in a %d CSV", - _.size(metadata), _.size(textcsv)); - - const filename = `${experimentId.substr(0, 8)}-${type}-${transformed.length}.csv`; - return { - text: textcsv, - headers: { - "Content-Type": "csv/text", - "Content-Disposition": "attachment; filename=" + filename - } - } -}; -*/ + const type = req.params.type; + if (CSV.allowedTypes.indexOf(type) === -1) { + debug('Invalid requested data type? %s', type); + return { text: 'Error, invalid URL composed' }; + } + + const experimentId = params.getString(req, 'experimentId', true); + const metadata = await sharedDataPull({ + 'experiment.experimentId': experimentId, + type, + }); + + const transformed = CSV.unrollNested(metadata, { + type, + experiment: true, + private: true, + }); + + const textcsv = CSV.produceCSVv1(transformed); + debug( + 'Fetch %d metadata(s), and converted in a %d CSV', + _.size(metadata), + _.size(textcsv) + ); + + const filename = `${experimentId.substr(0, 8)}-${type}-${ + transformed.length + }.csv`; + return { + text: textcsv, + headers: { + 'Content-Type': 'csv/text', + 'Content-Disposition': 'attachment; filename=' + filename, + }, + }; +} async function list(req) { - /* this function pull from the collection "directives" - * and filter by returning only the 'comparison' kind of - * experiment. This is imply req.params.type == 'comparison' */ - const MAX = 400; - const type = req.params.directiveType; + /* this function pull from the collection "directives" + * and filter by returning only the 'comparison' kind of + * experiment. This is imply req.params.type == 'comparison' */ + const MAX = 400; + const type = req.params.directiveType; - if(["comparison", "chiaroscuro"].indexOf(type) === -1) - return { text: "Directive Type not supported! "} + if (['comparison', 'chiaroscuro'].indexOf(type) === -1) + return { text: 'Directive Type not supported! ' }; - if(type === "comparison") { - /* this kind of directive require password for listing, + if (type === 'comparison') { + /* this kind of directive require password for listing, instead the shadowban at the moment is free access */ - if(!security.checkPassword(req)) - return { status: 403 }; - } - - const filter = { directiveType: type }; - const mongoc = await mongo3.clientConnect({concurrency: 1}); - - const configured = await mongo3 - .readLimit(mongoc, nconf.get('schema').directives, - filter, { when: -1 }, MAX, 0); - - const active = await mongo3 - .readLimit(mongoc, nconf.get('schema').experiments, - filter, { testTime: -1 }, MAX, 0); - - const expIdList = _.map(configured, 'experimentId'); - const lastweek = await mongo3 - .readLimit(mongoc, nconf.get('schema').metadata, { - "experiment.experimentId": { "$in": expIdList } - }, { savingTime: -1}, MAX, 0); - - await mongoc.close(); - - const infos = {}; - /* this is the return value, it would contain: + if (!security.checkPassword(req)) return { status: 403 }; + } + + const filter = { directiveType: type }; + const mongoc = await mongo3.clientConnect({ concurrency: 1 }); + + const configured = await mongo3.readLimit( + mongoc, + nconf.get('schema').experiments, + filter, + { when: -1 }, + MAX, + 0 + ); + + const active = await mongo3.readLimit( + mongoc, + nconf.get('schema').experiments, + filter, + { testTime: -1 }, + MAX, + 0 + ); + + const expIdList = _.map(configured, 'experimentId'); + const lastweek = await mongo3.readLimit( + mongoc, + nconf.get('schema').metadata, + { + 'experiment.experimentId': { $in: expIdList }, + }, + { savingTime: -1 }, + MAX, + 0 + ); + + await mongoc.close(); + + const infos = {}; + /* this is the return value, it would contain: .configured (the directive list) .active (eventually non-completed experiments) .recent (activly marked metadata) */ - infos.configured = _.map(configured, function(r) { - r.humanizedWhen = moment(r.when).format("YYYY-MM-DD"); - return _.omit(r, ['_id', 'directiveType']) - }); - - infos.active = _.compact(_.map(active, function(e) { - if(e.status === 'completed') - return null; - _.unset(e, '_id'); - e.publicKey = e.publicKey.substr(0, 8); - return e; - })); - - infos.recent = _.reduce(_.groupBy(_.map(lastweek, function(e) { - return { - publicKey: e.publicKey.substr(0, 8), - evidencetag: e.experiment.evidencetag, - experimentId: e.experiment.experimentId - } - }), 'experimentId'), function(memo, listOf, experimentId) { - memo[experimentId] = { - contributions: _.countBy(listOf, 'evidencetag'), - profiles: _.countBy(listOf, 'publicKey') + infos.configured = _.map(configured, function (r) { + r.humanizedWhen = moment(r.when).format('YYYY-MM-DD'); + return _.omit(r, ['_id', 'directiveType']); + }); + + infos.active = _.compact( + _.map(active, function (e) { + if (e.status === 'completed') return null; + _.unset(e, '_id'); + e.publicKey = e.publicKey.substr(0, 8); + return e; + }) + ); + + infos.recent = _.reduce( + _.groupBy( + _.map(lastweek, function (e) { + return { + publicKey: e.publicKey.substr(0, 8), + researchTag: e.experiment.researchTag, + experimentId: e.experiment.experimentId, }; - return memo; - }, {}); - - debug("Directives found: configured %d active %d (type %s, max %d)", - infos.configured.length, infos.active.length, type, MAX); - - return { - json: infos - }; + }), + 'experimentId' + ), + function (memo, listOf, experimentId) { + memo[experimentId] = { + contributions: _.countBy(listOf, 'researchTag'), + profiles: _.countBy(listOf, 'publicKey'), + }; + return memo; + }, + {} + ); + + debug( + 'Directives found: configured %d active %d (type %s, max %d)', + infos.configured.length, + infos.active.length, + type, + MAX + ); + + return { + json: infos, + }; } async function channel3(req) { - // this is invoked as handshake, and might return information - // helpful for the extension, about the experiment running. - const fields = [ 'href', 'experimentId', - 'evidencetag', 'execount', 'newProfile', 'profileName', - 'directiveType' ]; - const experimentInfo = _.pick(req.body, fields); - - experimentInfo.testName = new Date(req.body.when); - experimentInfo.publicKey = _.get(req.body, 'config.publicKey'); - - debug('Experiment info %O', experimentInfo); - - const retval = await experlib.saveExperiment(experimentInfo); - /* this is the default answer, as normally there is not an - * experiment running */ - if(_.isNull(retval)) - return { json: { experimentId: false }}; - - debug("Marked experiment as 'active' — %j", _.pick(retval, [ - 'evidencetag', 'execount', 'directiveType' - ])); - return { json: retval }; -}; + // this is invoked as handshake, and might return information + // helpful for the extension, about the experiment running. + const fields = [ + 'href', + 'experimentId', + 'researchTag', + 'execount', + 'newProfile', + 'profileName', + 'directiveType', + ]; + const experimentInfo = _.pick(req.body, fields); + + experimentInfo.testName = new Date(req.body.when); + experimentInfo.publicKey = _.get(req.body, 'config.publicKey'); + + debug('Experiment info %O', experimentInfo); + + const retval = await experlib.saveExperiment(experimentInfo); + /* this is the default answer, as normally there is not an + * experiment running */ + if (_.isNull(retval)) return { json: { experimentId: false } }; + + debug( + "Marked experiment as 'active' — %j", + _.pick(retval, ['researchTag', 'execount', 'directiveType']) + ); + return { json: retval }; +} async function conclude3(req) { - const testTime = req.params.testTime - debug("Conclude3 received: %s", testTime); - if(testTime.length < 10) - return { status: 403 }; - - const test = moment(testTime); - if(!test.isValid) - return { status: 403 }; - - const retval = await experlib.concludeExperiment(testTime); - // retval is {"acknowledged":true,"modifiedCount":0,"upsertedId":null,"upsertedCount":0,"matchedCount":0} - return { json: retval }; + const testTime = req.params.testTime; + debug('Conclude3 received: %s', testTime); + if (testTime.length < 10) return { status: 403 }; + + const test = moment(testTime); + if (!test.isValid) return { status: 403 }; + + const retval = await experlib.concludeExperiment(testTime); + // retval is {"acknowledged":true,"modifiedCount":0,"upsertedId":null,"upsertedCount":0,"matchedCount":0} + return { json: retval }; } module.exports = { - /* used by the webapps */ - // csv, -- before supporting this the CSV format should be redefined for tiktok - dot, - json, - list, - - /* used by the browser extension/guardoni */ - channel3, - conclude3, + /* used by the webapps */ + // csv, -- before supporting this the CSV format should be redefined for tiktok + dot, + json, + list, + csv, + + /* used by the browser extension/guardoni */ + channel3, + conclude3, }; diff --git a/platforms/tktrex/backend/routes/metadata.ts b/platforms/tktrex/backend/routes/metadata.ts new file mode 100644 index 000000000..b353b5fe8 --- /dev/null +++ b/platforms/tktrex/backend/routes/metadata.ts @@ -0,0 +1,42 @@ +import { decodeOrThrowRequest } from '@shared/endpoints/helper'; +import endpoints from '@tktrex/shared/endpoints/v2/public.endpoints'; +import createDebug from 'debug'; +import _ from 'lodash'; +import automo from '../lib/automo'; + +const debug = createDebug('routes:public'); + +// This variables is used as cap in every readLimit below +const PUBLIC_AMOUNT_ELEMS = 100; + +const listMetadata = async (req): Promise => { + const { + query: { + researchTag, + experimentId, + amount = PUBLIC_AMOUNT_ELEMS, + skip = 0, + }, + } = decodeOrThrowRequest(endpoints.ListMetadata, req); + + const metadata = await automo.getMetadataByFilter( + { + researchTag, + experimentId, + }, + { + amount, + skip, + } + ); + + debug( + 'Returning metadata by researchTag %s, %d evidences', + researchTag, + _.size(metadata) + ); + + return { json: metadata }; +}; + +export { listMetadata }; diff --git a/platforms/tktrex/backend/routes/personal.ts b/platforms/tktrex/backend/routes/personal.ts index ce5de366d..94b62e335 100644 --- a/platforms/tktrex/backend/routes/personal.ts +++ b/platforms/tktrex/backend/routes/personal.ts @@ -240,31 +240,17 @@ const getPersonalByExperimentId = async ( const supporter = await automo.getSupporterByPublicKey(publicKey); const opts = { amount: 100, skip: 0 }; - const htmls = await automo.getLastHTMLs( - { - publicKey: { - $eq: publicKey, - }, - experimentId: { - $eq: experimentId, - }, - }, - opts.skip, - opts.amount - ); - - const htmlIds = htmls.content.map((h) => h.id); // debug('Html ids %O', htmlIds); const metadata = await automo.getMetadataByFilter( { - id: { - $in: htmlIds, - }, publicKey: { $eq: publicKey, }, + experimentId: { + $eq: experimentId, + }, }, opts ); diff --git a/platforms/tktrex/backend/routes/public.js b/platforms/tktrex/backend/routes/public.ts similarity index 94% rename from platforms/tktrex/backend/routes/public.js rename to platforms/tktrex/backend/routes/public.ts index 3afe2fcae..5591e35bf 100644 --- a/platforms/tktrex/backend/routes/public.js +++ b/platforms/tktrex/backend/routes/public.ts @@ -1,12 +1,9 @@ +import createDebug from 'debug'; import _ from 'lodash'; import moment from 'moment'; -import createDebug from 'debug'; - -import * as params from '../lib/params'; import automo from '../lib/automo'; -// import utils from '../lib/utils'; import CSV from '../lib/CSV'; -// import cache from '../lib/cache'; +import * as params from '../lib/params'; const debug = createDebug('routes:public'); @@ -54,7 +51,7 @@ async function getLast(req) { }; */ -export function ensureRelated(rv) { +function ensureRelated(rv): null | any { /* for each related it is called and only the basic info used in 'compare' * page get returned. return 'null' if content is not complete */ const demanded = [ @@ -75,7 +72,7 @@ export function ensureRelated(rv) { : sele; } -export async function getVideoId(req) { +async function getVideoId(req): Promise { const { amount, skip } = params.optionParsing( req.params.paging, PUBLIC_AMOUNT_ELEMS @@ -112,7 +109,7 @@ export async function getVideoId(req) { return { json: evidences }; } -export async function getRelated(req) { +async function getRelated(req): Promise { const { amount, skip } = params.optionParsing( req.params.paging, PUBLIC_AMOUNT_ELEMS @@ -148,7 +145,7 @@ export async function getRelated(req) { return { json: evidences }; } -export async function getVideoCSV(req) { +async function getVideoCSV(req): Promise { // /api/v1/videoCSV/:videoId/:amount const MAXENTRY = 2800; const { amount, skip } = params.optionParsing(req.params.paging, MAXENTRY); @@ -179,14 +176,14 @@ export async function getVideoCSV(req) { }; } -async function getRecent(req) { +async function getRecent(req): Promise { // this still to be determined why was supposed to be implemented: perhaps 'compare' equivalent? return { json: { fuffa: true } }; } const SEARCH_FIELDS = ['id', 'query', 'publicKey', 'savingTime']; /* this is exported because also used in personal */ -export async function getSearches(req) { +async function getSearches(req): Promise { const amount = _.parseInt(req.query.amount) || 50; const skip = _.parseInt(req.query.skip) || 0; // this support the 'standard' format for Taboule @@ -208,12 +205,11 @@ export async function getSearches(req) { return { json: filtered }; } -module.exports = { - // getLast, +export { getVideoId, getRelated, getVideoCSV, - // getRecent, + getRecent, SEARCH_FIELDS, getSearches, }; diff --git a/platforms/tktrex/backend/scripts/build-indexes.js b/platforms/tktrex/backend/scripts/build-indexes.js index a98f459ca..1e4c54fcb 100644 --- a/platforms/tktrex/backend/scripts/build-indexes.js +++ b/platforms/tktrex/backend/scripts/build-indexes.js @@ -1,24 +1,24 @@ -ret = db.metadata2.createIndex({id: 1}, {unique: true }); checkret('metadata id', ret); -ret = db.metadata2.createIndex({videoId: 1}); checkret('metadata videoId', ret); -ret = db.metadata2.createIndex({"sections.videos.videoId": 1}); checkret('metadata sections.videos.href', ret); -ret = db.metadata2.createIndex({authorName: 1}); checkret('metadata authorName', ret); -ret = db.metadata2.createIndex({publicKey: 1}); checkret('metadata publicKey', ret); -ret = db.metadata2.createIndex({savingTime: -1}); checkret('metadata savingTime', ret); -ret = db.metadata2.createIndex({type: -1}); checkret('metadata type', ret); +ret = db.metadata.createIndex({id: 1}, {unique: true }); checkret('metadata id', ret); +ret = db.metadata.createIndex({videoId: 1}); checkret('metadata videoId', ret); +ret = db.metadata.createIndex({"sections.videos.videoId": 1}); checkret('metadata sections.videos.href', ret); +ret = db.metadata.createIndex({authorName: 1}); checkret('metadata authorName', ret); +ret = db.metadata.createIndex({publicKey: 1}); checkret('metadata publicKey', ret); +ret = db.metadata.createIndex({savingTime: -1}); checkret('metadata savingTime', ret); +ret = db.metadata.createIndex({type: -1}); checkret('metadata type', ret); ret = db.supporters.createIndex({ publicKey: 1 }, { unique: true }); checkret('supporters publicKey:', ret); +ret = db.experiments2.createIndex({ experimentId: -1, unique: true }); +checkret('experiments2 experimentId', ret); +ret = db.experiments2.createIndex({ when: -1 }); +checkret('experiments2 when', ret); + ret = db.htmls.createIndex({ id: 1 }, { unique: true} ); checkret('htmls id', ret); ret = db.htmls.createIndex({ savingTime: -1 }); checkret('htmls savingTime', ret); ret = db.htmls.createIndex({ publicKey: -1 }); checkret('htmls publicKey', ret); ret = db.htmls.createIndex({ metadataId: -1 }); checkret('htmls metadataId', ret); ret = db.htmls.createIndex({ processed: 1 }); checkret('htmls processed', ret); -ret = db.retrieved.createIndex({ videoId: 1 }, { unique: true} ); checkret('retrieved videoId', ret); -ret = db.retrieved.createIndex({ when: -1 }); checkret('retrieved when', ret); - -ret = db.categories.createIndex({ videoId: 1 }, { unique: true} ); checkret('categories videoId', ret); -ret = db.categories.createIndex({ when: -1 }); checkret('categories when', ret); function checkret(info, retval) { retval.info = info; diff --git a/platforms/tktrex/docs/package.json b/platforms/tktrex/docs/package.json index d82df077b..55490dfde 100644 --- a/platforms/tktrex/docs/package.json +++ b/platforms/tktrex/docs/package.json @@ -26,7 +26,7 @@ "devDependencies": { "@docusaurus/module-type-aliases": "2.0.0-beta.14", "@tsconfig/docusaurus": "^1.0.4", - "typescript": "^4.7.2" + "typescript": "^4.7.4" }, "browserslist": { "production": [ diff --git a/platforms/tktrex/extension/__tests__/app.spec.ts b/platforms/tktrex/extension/__tests__/app.spec.ts index 5fd92c0c2..24bdfff59 100644 --- a/platforms/tktrex/extension/__tests__/app.spec.ts +++ b/platforms/tktrex/extension/__tests__/app.spec.ts @@ -15,7 +15,7 @@ import * as handlers from '../src/app/handlers'; import api, { getHeadersForDataDonation } from '../src/background/api'; import tkHub from '../src/handlers/hub'; import { tkLog } from '../src/logger'; -import {sleep} from '@shared/utils/promise.utils' +import { sleep } from '@shared/utils/promise.utils'; const chromeListener = jest.fn(); @@ -67,7 +67,6 @@ chrome.runtime.sendMessage active: true, ux: true, href: tkURL, - evidencetag: 'fake-tag', researchTag: 'test-tag', execount: 1, newProfile: true, diff --git a/platforms/tktrex/extension/__tests__/profile.spec.ts b/platforms/tktrex/extension/__tests__/profile.spec.ts index 60038a314..9854822f1 100644 --- a/platforms/tktrex/extension/__tests__/profile.spec.ts +++ b/platforms/tktrex/extension/__tests__/profile.spec.ts @@ -1,7 +1,7 @@ import { boot } from '@shared/extension/app'; import { handleServerLookup, - initializeKey + initializeKey, } from '@shared/extension/chrome/background/account'; import { load } from '@shared/extension/chrome/background/index'; import { handleSyncMessage } from '@shared/extension/chrome/background/sync'; @@ -17,8 +17,6 @@ import api, { getHeadersForDataDonation } from '../src/background/api'; import tkHub from '../src/handlers/hub'; import { tkLog } from '../src/logger'; - - const chromeListener = jest.fn(); const profileMatcher = app.tkHandlers.profile; @@ -71,7 +69,6 @@ chrome.runtime.sendMessage active: true, ux: true, href: tkURL, - evidencetag: 'fake-tag', researchTag: 'test-tag', execount: 1, newProfile: true, @@ -530,7 +527,7 @@ describe('TK App', () => { }, ], type: 'profile', - } + }, ], }); }); diff --git a/platforms/tktrex/extension/package.json b/platforms/tktrex/extension/package.json index 4dbd4203e..13d2aeb0f 100644 --- a/platforms/tktrex/extension/package.json +++ b/platforms/tktrex/extension/package.json @@ -32,20 +32,20 @@ "@types/debug": "^4.1.7", "@types/jquery": "^3.5.14", "@types/node": "^16.11.36", - "@types/react": "^17.0.45", + "@types/react": "^17.0.47", "@types/react-dom": "^17.0.17", - "@typescript-eslint/eslint-plugin": "^5.27.0", - "@typescript-eslint/parser": "^5.27.0", - "eslint": "^8.16.0", + "@typescript-eslint/eslint-plugin": "^5.30.5", + "@typescript-eslint/parser": "^5.30.5", + "eslint": "^8.19.0", "eslint-config-prettier": "^8.5.0", - "eslint-config-standard-with-typescript": "^21.0.1", + "eslint-config-standard-with-typescript": "^22.0.0", "eslint-plugin-import": "^2.26.0", "eslint-plugin-node": "^11.1.0", - "eslint-plugin-promise": "^5.2.0", - "eslint-plugin-react": "^7.30.0", + "eslint-plugin-promise": "^6.0.0", + "eslint-plugin-react": "^7.30.1", "io-ts": "^2.2.16", "jest-environment-jsdom-global": "^3.1.2", - "typescript": "^4.7.2", + "typescript": "^4.7.4", "webpack": "^5.73.0" } } diff --git a/platforms/tktrex/extension/public/tktrex-logo.png b/platforms/tktrex/extension/public/tktrex-logo.png index b821604bb..e68f93ca7 100644 Binary files a/platforms/tktrex/extension/public/tktrex-logo.png and b/platforms/tktrex/extension/public/tktrex-logo.png differ diff --git a/platforms/tktrex/extension/public/tktrex_logo.png b/platforms/tktrex/extension/public/tktrex_logo.png deleted file mode 100644 index e68f93ca7..000000000 Binary files a/platforms/tktrex/extension/public/tktrex_logo.png and /dev/null differ diff --git a/platforms/tktrex/extension/src/app/app.ts b/platforms/tktrex/extension/src/app/app.ts index 49bd012f8..38dba2882 100644 --- a/platforms/tktrex/extension/src/app/app.ts +++ b/platforms/tktrex/extension/src/app/app.ts @@ -1,15 +1,15 @@ import { ObserverHandler, refreshUUID, - SelectorObserverHandler, RouteObserverHandler, + SelectorObserverHandler, } from '@shared/extension/app'; -import config from '@shared/extension/config'; import log from '@shared/extension/logger'; +import UserSettings from '@shared/extension/models/UserSettings'; import { sizeCheck } from '@shared/providers/dataDonation.provider'; import _ from 'lodash'; -import { INTERCEPTED_ITEM_CLASS } from '../interceptor/constants'; import tkHub from '../handlers/hub'; +import { INTERCEPTED_ITEM_CLASS } from '../interceptor/constants'; const appLog = log.extend('app'); @@ -144,6 +144,7 @@ const handleInterceptedData = (): void => { // experiment in progress; const handleSigi = _.debounce((element: Node): void => { + // eslint-disable-next-line no-console console.log('Sigi', element); }); @@ -209,68 +210,82 @@ const handleSuggested = _.debounce((elem: Node): void => { * that got display in 'following' 'foryou' or 'creator' page */ let videoCounter = 0; -const handleVideo = _.debounce((node: HTMLElement): void => { - /* we should check nature for good, the 'video' handles are triggered also in - * other pages, afterall! */ - if (_.startsWith(window.location.pathname, '/search')) return; - if (profileHandler.match.location.test(window.location.pathname)) return; - - /* this function return a node element that has a size - * lesser than 10k, and stop when find out the parent - * would be more than 10k big. */ - const videoRoot = _.reduce( - _.times(20), - (memo: HTMLElement, iteration: number): HTMLElement => { - if (memo.parentNode instanceof HTMLElement) { - if (memo.parentNode.outerHTML.length > 10000) { - appLog.debug( - 'handleVideo: parentNode > 10000', - memo.parentNode.outerHTML.length, - ); - return memo; +const handleVideo = _.debounce( + (node: HTMLElement, h: any, b: any, config: UserSettings): void => { + /* we should check nature for good, the 'video' handles are triggered also in + * other pages, afterall! */ + if (_.startsWith(window.location.pathname, '/search')) return; + if (profileHandler.match.location.test(window.location.pathname)) return; + + /* this function return a node element that has a size + * lesser than 10k, and stop when find out the parent + * would be more than 10k big. */ + const videoRoot = _.reduce( + _.times(20), + (memo: HTMLElement, iteration: number): HTMLElement => { + if (memo.parentNode instanceof HTMLElement) { + if (memo.parentNode.outerHTML.length > 10000) { + appLog.debug( + 'handleVideo: parentNode > 10000', + memo.parentNode.outerHTML.length, + ); + return memo; + } + return memo.parentNode; } - return memo.parentNode; - } - return memo; - }, - node, - ); - - if (videoRoot.hasAttribute('trex')) { - appLog.info( - 'element already acquired: skipping', - videoRoot.getAttribute('trex'), + return memo; + }, + node, ); - return; - } - - videoCounter++; + if (videoRoot.hasAttribute('trex')) { + appLog.info( + 'element already acquired: skipping', + videoRoot.getAttribute('trex'), + ); - appLog.info('+video', videoRoot, ' acquired, now', videoCounter, 'in total'); + return; + } - videoRoot.setAttribute('trex', `${videoCounter}`); + videoCounter++; - tkHub.dispatch({ - type: 'NewVideo', - payload: { - html: videoRoot.outerHTML, - href: window.location.href, - feedId, - feedCounter, + appLog.info( + '+video', + videoRoot, + ' acquired, now', videoCounter, - rect: videoRoot.getBoundingClientRect(), - }, - }); + 'in total', + ); - if (config.ux) { - videoRoot.style.border = '1px solid green'; - } -}, 300); + videoRoot.setAttribute('trex', `${videoCounter}`); + + tkHub.dispatch({ + type: 'NewVideo', + payload: { + html: videoRoot.outerHTML, + href: window.location.href, + feedId, + feedCounter, + videoCounter, + rect: videoRoot.getBoundingClientRect(), + }, + }); + + if (config.ux) { + videoRoot.style.border = '1px solid green'; + } + }, + 300, +); const handleProfile = _.debounce( - (node: HTMLElement, route: any, _selectorName: string): void => { + ( + node: HTMLElement, + route: any, + _selectorName: string, + s: UserSettings, + ): void => { const profileName = window.location.pathname.match( route.match.location, )?.[1]; @@ -370,7 +385,7 @@ export const tkHandlers: { [key: string]: ObserverHandler } = { type: 'selector', selector: 'a[href^="/@"]', }, - handle: () => undefined, + handle: () => undefined, }, search: searchHandler, apiInterceptor: { diff --git a/platforms/tktrex/shared/__spec__/parser/parser.spec.ts b/platforms/tktrex/shared/__spec__/parser/parser.spec.ts index 7fd0788e8..e99ed8d2f 100644 --- a/platforms/tktrex/shared/__spec__/parser/parser.spec.ts +++ b/platforms/tktrex/shared/__spec__/parser/parser.spec.ts @@ -5,7 +5,7 @@ import { expectToBeIncludedIn, normalizeDeepStrings, } from '../../src/lib/util'; -import { ForYouVideoMetaData } from '../../src/models/MetaData'; +import { ForYouVideoMetadata } from '../../src/models/Metadata'; import createServerSideParser from '../../src/parser/serverSideParser'; import historicData from './fixtures/history.json'; @@ -15,7 +15,7 @@ describe('The TikTok parser for the ForYou feed', () => { // and exclude the example that we know to be wrong const forYouSamples = historicData.filter( (sample) => - isRight(ForYouVideoMetaData.decode(sample.metadata)), + isRight(ForYouVideoMetadata.decode(sample.metadata)), ); const { parseForYouVideo } = createServerSideParser(); @@ -36,7 +36,7 @@ describe('The TikTok parser for the ForYou feed', () => { // inside the expected value, // now we also check that it validates the schema of the // expected value - const validation = ForYouVideoMetaData.decode(actual); + const validation = ForYouVideoMetadata.decode(actual); if (!isRight(validation)) { const report = PathReporter.report(validation); throw new Error([ diff --git a/platforms/tktrex/shared/bin/open-doc-api.ts b/platforms/tktrex/shared/bin/open-doc-api.ts index c5ed87c9c..1a71da701 100644 --- a/platforms/tktrex/shared/bin/open-doc-api.ts +++ b/platforms/tktrex/shared/bin/open-doc-api.ts @@ -18,7 +18,6 @@ swagger.writeOpenDocTo( endpoints, version: packageJson.version, models: { - ...apiModels.Common, ...apiModels.Events, ...apiModels.Personal, ...apiModels.Public, @@ -39,5 +38,5 @@ swagger.writeOpenDocTo( }, ], }, - path.resolve(process.cwd(), './build') + path.resolve(process.cwd(), './build'), ); diff --git a/platforms/tktrex/shared/package.json b/platforms/tktrex/shared/package.json index 35c32fdc8..7eed7fe7a 100644 --- a/platforms/tktrex/shared/package.json +++ b/platforms/tktrex/shared/package.json @@ -11,21 +11,21 @@ "open-doc-api": "ts-node bin/open-doc-api.ts" }, "devDependencies": { - "@types/jest": "^27.5.1", + "@types/jest": "^27.5.2", "@types/node": "^16.11.36", "@types/yargs": "^17.0.10", - "@typescript-eslint/eslint-plugin": "^5.27.0", - "@typescript-eslint/parser": "^5.27.0", - "eslint": "^8.16.0", - "eslint-config-standard-with-typescript": "^21.0.1", + "@typescript-eslint/eslint-plugin": "^5.30.5", + "@typescript-eslint/parser": "^5.30.5", + "eslint": "^8.19.0", + "eslint-config-standard-with-typescript": "^22.0.0", "eslint-plugin-import": "^2.26.0", "eslint-plugin-node": "^11.1.0", - "eslint-plugin-promise": "^5.2.0", - "eslint-plugin-react": "^7.30.0", + "eslint-plugin-promise": "^6.0.0", + "eslint-plugin-react": "^7.30.1", "jest": "^27.5.1", "ts-jest": "^27.1.5", - "ts-node": "^10.8.0", - "typescript": "^4.7.2", + "ts-node": "^10.8.2", + "typescript": "^4.7.4", "yargs": "^17.5.1" }, "dependencies": { diff --git a/platforms/tktrex/shared/src/arbitraries/Metadata.arb.ts b/platforms/tktrex/shared/src/arbitraries/Metadata.arb.ts new file mode 100644 index 000000000..08adc1a97 --- /dev/null +++ b/platforms/tktrex/shared/src/arbitraries/Metadata.arb.ts @@ -0,0 +1,76 @@ +import { getArbitrary } from 'fast-check-io-ts'; +import { + SearchMetadata, + ForYouVideoMetadata, + FollowingVideoMetadata, + MetadataBase, +} from '../models/Metadata'; +import { propsOmitType } from '@shared/utils/arbitrary.utils'; +import { fc } from '@shared/test'; +import { subDays } from 'date-fns'; +import * as t from 'io-ts'; + +const metadataBaseProps = propsOmitType(MetadataBase, ['id', 'savingTime']); + +/** + * SearchMetadata arbitrary + * + **/ + +const searchMetadataProps = propsOmitType(SearchMetadata.types[1], [ + 'results', + 'thumbnails', +]); + +export const SearchMetaDataArb = getArbitrary( + t.intersection([t.type(metadataBaseProps), t.type(searchMetadataProps)]) +).map((ad) => ({ + ...ad, + id: fc.sample(fc.uuid(), 1)[0], + savingTime: subDays(new Date(), fc.sample(fc.nat(), 1)[0]), + clientTime: fc.sample(fc.date(), 1)[0], + results: [], + thumbnails: [], +})); + +/** + * ForYouMetadata arbitrary + * + **/ +const forYouMetadataProps = propsOmitType(ForYouVideoMetadata.types[1], []); + +export const ForYouVideoMetaDataArb = getArbitrary( + t.intersection([t.type(metadataBaseProps), t.type(forYouMetadataProps)]) +).map((ad) => ({ + ...ad, + id: fc.sample(fc.uuid(), 1)[0], + savingTime: subDays(new Date(), fc.sample(fc.nat(), 1)[0]), + clientTime: fc.sample(fc.date(), 1)[0], +})); + +/** + * FollowingVideoMetadata arbitrary + * + **/ +const followingVideoMetadataProps = propsOmitType( + FollowingVideoMetadata.types[1], + [] +); + +export const FollowingVideoMetaDataArb = getArbitrary( + t.intersection([ + t.type(metadataBaseProps), + t.type(followingVideoMetadataProps), + ]) +).map((ad) => ({ + ...ad, + id: fc.sample(fc.uuid(), 1)[0], + savingTime: subDays(new Date(), fc.sample(fc.nat(), 1)[0]), + clientTime: fc.sample(fc.date(), 1)[0], +})); + +export const MetadataArb = fc.oneof( + SearchMetaDataArb, + ForYouVideoMetaDataArb, + FollowingVideoMetaDataArb +); diff --git a/platforms/tktrex/shared/src/endpoints/v2/index.ts b/platforms/tktrex/shared/src/endpoints/v2/index.ts index 68648ffcd..58c5d748d 100644 --- a/platforms/tktrex/shared/src/endpoints/v2/index.ts +++ b/platforms/tktrex/shared/src/endpoints/v2/index.ts @@ -1,7 +1,9 @@ import Public from './public.endpoints'; import Personal from './personal.endpoints'; +import Metadata from './metadata.endpoints'; export default { Public, Personal, + Metadata, }; diff --git a/platforms/tktrex/shared/src/endpoints/v2/metadata.endpoints.ts b/platforms/tktrex/shared/src/endpoints/v2/metadata.endpoints.ts new file mode 100644 index 000000000..09b4472be --- /dev/null +++ b/platforms/tktrex/shared/src/endpoints/v2/metadata.endpoints.ts @@ -0,0 +1,27 @@ +import { DocumentedEndpoint } from '@shared/endpoints'; +import { Format } from '@shared/models/common'; +import * as t from 'io-ts'; +import { NumberFromString } from 'io-ts-types/NumberFromString'; +import * as apiModel from '../../models'; + +const ListMetadata = DocumentedEndpoint({ + title: 'Get metadata by research tag', + description: '', + tags: ['public'], + Method: 'GET', + getPath: () => '/v2/metadata', + Input: { + Query: t.type({ + experimentId: t.union([t.string, t.undefined]), + researchTag: t.union([t.string, t.undefined]), + amount: t.union([NumberFromString, t.undefined]), + skip: t.union([NumberFromString, t.undefined]), + format: t.union([Format, t.undefined]), + }), + }, + Output: t.array(apiModel.Metadata.Metadata), +}); + +export default { + ListMetadata, +}; diff --git a/platforms/tktrex/shared/src/endpoints/v2/personal.endpoints.ts b/platforms/tktrex/shared/src/endpoints/v2/personal.endpoints.ts index 38cba9de3..52b57d3f7 100644 --- a/platforms/tktrex/shared/src/endpoints/v2/personal.endpoints.ts +++ b/platforms/tktrex/shared/src/endpoints/v2/personal.endpoints.ts @@ -1,5 +1,6 @@ import * as t from 'io-ts'; import * as apiModel from '../../models'; +import { Format, What } from '@shared/models/common'; import { DocumentedEndpoint } from '@shared/endpoints'; import { PersonalData } from '../../models/personal'; @@ -9,7 +10,7 @@ const GetPersonalJSON = DocumentedEndpoint({ Input: { Params: t.type({ publicKey: t.string, - what: apiModel.Common.What, + what: What, }), }, Output: apiModel.Personal.PersonalVideoList, @@ -24,7 +25,7 @@ const GetPersonalCSV = DocumentedEndpoint({ Input: { Params: t.type({ publicKey: t.string, - what: apiModel.Common.What, + what: What, }), }, Output: t.string, @@ -44,7 +45,7 @@ const GetPersonalByExperiment = DocumentedEndpoint({ Params: t.type({ publicKey: t.string, experimentId: t.string, - format: apiModel.Common.Format, + format: Format, }), }, Output: PersonalData, diff --git a/platforms/tktrex/shared/src/endpoints/v2/public.endpoints.ts b/platforms/tktrex/shared/src/endpoints/v2/public.endpoints.ts index f3f520f68..1ff92444d 100644 --- a/platforms/tktrex/shared/src/endpoints/v2/public.endpoints.ts +++ b/platforms/tktrex/shared/src/endpoints/v2/public.endpoints.ts @@ -1,7 +1,9 @@ import { DocumentedEndpoint } from '@shared/endpoints'; +import { What, Format } from '@shared/models/common'; import { SearchQuery } from '@shared/models/http/SearchQuery'; import * as t from 'io-ts'; import * as apiModel from '../../models'; +import { NumberFromString } from 'io-ts-types/NumberFromString'; export const Handshake = DocumentedEndpoint({ title: 'Handshake', @@ -47,8 +49,8 @@ const GetSearchByQuery = DocumentedEndpoint({ getPath: ({ query, format }) => `/v2/public/query/${query}/${format}`, Input: { Params: t.type({ - query: apiModel.Common.What, - format: apiModel.Common.Format, + query: What, + format: Format, }), Query: SearchQuery, }, @@ -67,10 +69,29 @@ const GetQueryList = DocumentedEndpoint({ tags: ['searches'], }); +const ListMetadata = DocumentedEndpoint({ + title: 'Get metadata by research tag', + description: '', + tags: ['public'], + Method: 'GET', + getPath: () => '/v2/metadata', + Input: { + Query: t.type({ + experimentId: t.union([t.string, t.undefined]), + researchTag: t.union([t.string, t.undefined]), + amount: t.union([NumberFromString, t.undefined]), + skip: t.union([NumberFromString, t.undefined]), + format: t.union([Format, t.undefined]), + }), + }, + Output: t.array(apiModel.Metadata.Metadata), +}); + export default { AddEvents, Handshake, GetSearches, GetSearchByQuery, GetQueryList, + ListMetadata, }; diff --git a/platforms/tktrex/shared/src/models/MetaData.ts b/platforms/tktrex/shared/src/models/Metadata.ts similarity index 72% rename from platforms/tktrex/shared/src/models/MetaData.ts rename to platforms/tktrex/shared/src/models/Metadata.ts index 484259681..8fce70805 100644 --- a/platforms/tktrex/shared/src/models/MetaData.ts +++ b/platforms/tktrex/shared/src/models/Metadata.ts @@ -46,7 +46,7 @@ const Metrics = t.type( type Metrics = t.TypeOf; -export const MetaDataBase = t.type( +export const MetadataBase = t.type( { id: t.string, savingTime: t.string, @@ -55,11 +55,11 @@ export const MetaDataBase = t.type( 'VideoMetaDataBase', ); -export type MetaDataBase = t.TypeOf; +export type MetadataBase = t.TypeOf; -export const ForYouVideoMetaData = t.intersection( +export const ForYouVideoMetadata = t.intersection( [ - MetaDataBase, + MetadataBase, t.type( { type: t.literal('foryou'), @@ -83,14 +83,14 @@ export const ForYouVideoMetaData = t.intersection( 'foryou', ), ], - 'ForYouVideoMetaData', + 'ForYouVideoMetadata', ); -export type ForYouVideoMetaData = t.TypeOf; +export type ForYouVideoMetadata = t.TypeOf; -export const FollowingVideoMetaData = t.intersection( +export const FollowingVideoMetadata = t.intersection( [ - MetaDataBase, + MetadataBase, t.type( { type: t.literal('following'), @@ -103,11 +103,11 @@ export const FollowingVideoMetaData = t.intersection( 'FollowingVideoMetaData', ); -export type FollowingVideoMetaData = t.TypeOf; +export type FollowingVideoMetadata = t.TypeOf; -export const SearchMetaData = t.intersection( +export const SearchMetadata = t.intersection( [ - MetaDataBase, + MetadataBase, t.type( { type: t.literal('search'), @@ -124,16 +124,16 @@ export const SearchMetaData = t.intersection( 'search', ), ], - 'SearchVideoMetaData', + 'SearchVideoMetadata', ); -export type SearchMetaData = t.TypeOf; +export type SearchMetadata = t.TypeOf; -export const MetaData = t.union( - [ForYouVideoMetaData, FollowingVideoMetaData, SearchMetaData], +export const Metadata = t.union( + [ForYouVideoMetadata, FollowingVideoMetadata, SearchMetadata], 'VideoMetaData', ); -export type MetaData = t.TypeOf; +export type Metadata = t.TypeOf; -export default MetaData; +export default Metadata; diff --git a/platforms/tktrex/shared/src/models/index.ts b/platforms/tktrex/shared/src/models/index.ts index 93d45b6a2..e966a428e 100644 --- a/platforms/tktrex/shared/src/models/index.ts +++ b/platforms/tktrex/shared/src/models/index.ts @@ -1,6 +1,6 @@ -import * as Common from './common'; import * as Events from './events/ContributionEvent'; import * as Personal from './personal'; import * as Public from './public'; +import * as Metadata from './Metadata'; -export { Common, Events, Personal, Public }; +export { Events, Personal, Public, Metadata }; diff --git a/platforms/tktrex/shared/src/models/personal/index.ts b/platforms/tktrex/shared/src/models/personal/index.ts index ee052375a..e9dbb5075 100644 --- a/platforms/tktrex/shared/src/models/personal/index.ts +++ b/platforms/tktrex/shared/src/models/personal/index.ts @@ -1,6 +1,6 @@ import * as t from 'io-ts'; import { DateFromISOString } from 'io-ts-types/lib/DateFromISOString'; -import MetaData from '../MetaData'; +import Metadata from '../Metadata'; import {Supporter} from '@shared/models/Supporter'; export const PersonalVideoFeed = t.strict( @@ -29,7 +29,7 @@ export type PersonalVideoList = t.TypeOf; */ export const PersonalData = t.strict({ supporter: Supporter, - metadata: t.array(MetaData), + metadata: t.array(Metadata), }, 'PersonalData'); export type PersonalData = t.TypeOf diff --git a/platforms/tktrex/shared/src/models/public/index.ts b/platforms/tktrex/shared/src/models/public/index.ts index 3f3954332..36b4f6112 100644 --- a/platforms/tktrex/shared/src/models/public/index.ts +++ b/platforms/tktrex/shared/src/models/public/index.ts @@ -1,16 +1,6 @@ import * as t from 'io-ts'; import { DateFromISOString } from 'io-ts-types/lib/DateFromISOString'; -// still a copy from YT to be converted -export const What = t.union( - [t.literal('foryou'), t.literal('following'), t.literal('search')], - 'What', -); - -export type What = t.TypeOf; - -export const Format = t.union([t.literal('csv'), t.literal('json')], 'Format'); -export type Format = t.TypeOf; export const PersonalVideoFeed = t.strict( { diff --git a/platforms/ycai/studio/package.json b/platforms/ycai/studio/package.json index 90a2704d6..71b510acd 100644 --- a/platforms/ycai/studio/package.json +++ b/platforms/ycai/studio/package.json @@ -26,45 +26,45 @@ }, "devDependencies": { "@testing-library/jest-dom": "^5.16.4", - "@testing-library/react": "^13.2.0", - "@testing-library/user-event": "^14.2.0", + "@testing-library/react": "^13.3.0", + "@testing-library/user-event": "^14.2.1", "@types/bs58": "^4.0.1", "@types/chrome": "^0.0.188", "@types/debug": "^4.1.7", - "@types/jest": "^27.5.1", + "@types/jest": "^27.5.2", "@types/node": "^16.11.36", - "@types/react": "^17.0.45", + "@types/react": "^17.0.47", "@types/react-dom": "^17.0.17", "@types/react-test-renderer": "^17.0.2", "@types/swagger-ui": "^3.52.0", - "@typescript-eslint/eslint-plugin": "^5.27.0", - "@typescript-eslint/parser": "^5.27.0", + "@typescript-eslint/eslint-plugin": "^5.30.5", + "@typescript-eslint/parser": "^5.30.5", "buffer": "^6.0.3", "conventional-changelog-conventionalcommits": "^4.6.3", "css-loader": "^6.7.1", "dotenv": "^14.3.2", - "eslint": "^8.16.0", + "eslint": "^8.19.0", "eslint-config-prettier": "^8.5.0", - "eslint-config-standard-with-typescript": "^21.0.1", + "eslint-config-standard-with-typescript": "^22.0.0", "eslint-plugin-import": "^2.26.0", "eslint-plugin-node": "^11.1.0", - "eslint-plugin-promise": "^5.2.0", - "eslint-plugin-react": "^7.30.0", + "eslint-plugin-promise": "^6.0.0", + "eslint-plugin-react": "^7.30.1", "husky": "^7.0.4", "jest": "^27.5.1", "jest-chrome": "^0.7.2", - "prettier": "^2.6.2", + "prettier": "^2.7.1", "react-test-renderer": "^17.0.2", "style-loader": "^3.3.1", "svgdom": "^0.1.10", "ts-jest": "^27.1.5", - "ts-loader": "^9.3.0", - "ts-node": "^10.8.0", - "typescript": "^4.7.2", + "ts-loader": "^9.3.1", + "ts-node": "^10.8.2", + "typescript": "^4.7.4", "webpack": "^5.73.0", "webpack-bundle-analyzer": "^4.5.0", "webpack-cli": "^4.10.0", - "webpack-dev-server": "^4.9.2", + "webpack-dev-server": "^4.9.3", "zenroom": "^2.2.3" }, "dependencies": { @@ -88,7 +88,7 @@ "date-fns": "^2.28.0", "debug": "^4.3.4", "fp-ts": "^2.11.9", - "i18next": "^21.6.10", + "i18next": "^21.8.13", "io-ts": "^2.2.16", "io-ts-types": "^0.5.16", "lodash": "^4.17.21", @@ -100,8 +100,8 @@ "react-dnd": "^14.0.5", "react-dnd-html5-backend": "^14.1.0", "react-dom": "^17.0.2", - "react-i18next": "^11.15.3", - "react-use-clipboard": "^1.0.7", + "react-i18next": "^11.18.0", + "react-use-clipboard": "^1.0.8", "swagger-ui": "^4.4.1", "ts-endpoint": "^2.0.0", "tweetnacl": "^1.0.3", diff --git a/platforms/ycai/studio/src/components/dashboard/community/ADVChannelStatsBox.tsx b/platforms/ycai/studio/src/components/dashboard/community/ADVChannelStatsBox.tsx index f9006649c..44d9fe5a8 100644 --- a/platforms/ycai/studio/src/components/dashboard/community/ADVChannelStatsBox.tsx +++ b/platforms/ycai/studio/src/components/dashboard/community/ADVChannelStatsBox.tsx @@ -31,7 +31,7 @@ export const ADVChannelStatsBox: React.FC = () => { { if (creatorADVStats.length === 0) { diff --git a/platforms/ycai/studio/src/components/dashboard/community/DonutChart.tsx b/platforms/ycai/studio/src/components/dashboard/community/DonutChart.tsx index 380f731e6..43563909c 100644 --- a/platforms/ycai/studio/src/components/dashboard/community/DonutChart.tsx +++ b/platforms/ycai/studio/src/components/dashboard/community/DonutChart.tsx @@ -43,7 +43,7 @@ export const DonutChart = ({ const chartOpts: c3.ChartConfiguration = { bindto: `#${donutChardId}`, data: { - columns: columns, + columns, type: 'donut', colors, }, diff --git a/platforms/ycai/studio/src/i18n/index.ts b/platforms/ycai/studio/src/i18n/index.ts index 3309cab3f..baa68331d 100644 --- a/platforms/ycai/studio/src/i18n/index.ts +++ b/platforms/ycai/studio/src/i18n/index.ts @@ -5,7 +5,7 @@ import en from './en-US'; export default i18n.use(initReactI18next).init({ resources: { - en: en, + en, }, fallbackLng: 'en', interpolation: { diff --git a/platforms/ycai/studio/src/state/dashboard/creator.queries.ts b/platforms/ycai/studio/src/state/dashboard/creator.queries.ts index 351da23f5..e71d7954a 100644 --- a/platforms/ycai/studio/src/state/dashboard/creator.queries.ts +++ b/platforms/ycai/studio/src/state/dashboard/creator.queries.ts @@ -76,7 +76,7 @@ const throwOnMissingProfile = ( TE.fromPredicate( (s): s is AuthorizedContentCreator => s?.registeredOn !== undefined && s.accessToken !== undefined, - () => new APIError('NotFound', 'Missing Content Creator', []) + () => new APIError(404, 'NotFound', 'Missing Content Creator', []) ) ); diff --git a/platforms/ycai/studio/src/theme.ts b/platforms/ycai/studio/src/theme.ts index 827281dc1..f890d45a2 100644 --- a/platforms/ycai/studio/src/theme.ts +++ b/platforms/ycai/studio/src/theme.ts @@ -124,8 +124,8 @@ export const YCAITheme = createTheme({ secondary: black, }, common: { - white: white, - black: black, + white, + black, }, background: { default: white, diff --git a/platforms/yttrex/backend/bin/app.ts b/platforms/yttrex/backend/bin/app.ts index cd0b41cd4..8a797ec84 100644 --- a/platforms/yttrex/backend/bin/app.ts +++ b/platforms/yttrex/backend/bin/app.ts @@ -6,91 +6,16 @@ import { GetLogger } from '@shared/logger'; import bodyParser from 'body-parser'; import cors from 'cors'; import express from 'express'; -import _ from 'lodash'; import { apiList } from '../lib/api'; import mongo3 from '../lib/mongo3'; import { DeleteRecommendationRoute } from '../routes/youchoose/deleteRecommendation.route'; +import { routeHandleMiddleware } from '@shared/backend/utils/routeHandlerMiddleware'; const logger = GetLogger('api'); -const logAPICount = { requests: {}, responses: {}, errors: {} }; - -function loginc(kind: string, fname: string): void { - logAPICount[kind][fname] = logAPICount[kind][fname] - ? logAPICount[kind][fname]++ - : 1; -} - -const iowrapper = - (fname: string) => - async (req: express.Request, res: express.Response): Promise => { - try { - loginc('requests', fname); - const funct = apiList[fname]; - const httpresult = await funct(req, res); - - if (httpresult.headers) - _.each(httpresult.headers, function (value, key) { - logger.debug('Setting header %s: %s', key, value); - res.setHeader(key, value); - }); - - if (!httpresult) { - logger.debug("API (%s) didn't return anything!?", fname); - loginc('errors', fname); - res.send('Fatal error: Invalid output'); - res.status(501); - } else if (httpresult.json?.error) { - const statusCode = httpresult.json.status ?? 500; - logger.debug('API (%s) failure, returning %d', fname, statusCode); - loginc('errors', fname); - res.status(statusCode); - res.json(httpresult.json); - } else if (httpresult.json) { - // logger("API (%s) success, returning %d bytes JSON", fname, _.size(JSON.stringify(httpresult.json))); - loginc('responses', fname); - res.setHeader('Access-Control-Allow-Origin', '*'); - res.json(httpresult.json); - } else if (httpresult.text) { - // logger("API (%s) success, returning text (size %d)", fname, _.size(httpresult.text)); - loginc('responses', fname); - res.send(httpresult.text); - } else if (httpresult.status) { - // logger("Returning empty status %d from API (%s)", httpresult.status, fname); - loginc('responses', fname); - res.status(httpresult.status); - } else { - logger.debug( - 'Undetermined failure in API (%s) → %j', - fname, - httpresult - ); - loginc('errors', fname); - res.status(502); - res.send('Error?'); - } - } catch (error) { - res.status(502); - res.send('Software error: ' + error.message); - loginc('errors', fname); - logger.debug('Error in HTTP handler API(%s): %o', fname, error); - } - res.end(); - }; + +const iowrapper = routeHandleMiddleware(apiList); /* one log entry per minute about the amount of API absolved */ -setInterval(() => { - let print = false; - _.each(_.keys(logAPICount), function (k) { - if (!_.keys(logAPICount[k]).length) - // eslint-disable-next-line @typescript-eslint/no-dynamic-delete - delete logAPICount[k]; - else print = true; - }); - if (print) logger.debug('%j', logAPICount); - logAPICount.responses = {}; - logAPICount.errors = {}; - logAPICount.requests = {}; -}, 60 * 1000); export const makeApp = async ( ctx: MakeAppContext @@ -270,10 +195,7 @@ export const makeApp = async ( apiRouter.post('/v3/directives/:ignored?', iowrapper('postDirective')); apiRouter.get('/v3/directives/:experimentId', iowrapper('fetchDirective')); apiRouter.post('/v2/handshake', iowrapper('experimentChannel3')); - apiRouter.delete( - '/v3/experiment/:testTime', - iowrapper('concludeExperiment3') - ); + apiRouter.get( '/v2/experiment/:experimentId/json', iowrapper('experimentJSON') @@ -284,6 +206,12 @@ export const makeApp = async ( ); apiRouter.get('/v2/experiment/:experimentId/dot', iowrapper('experimentDOT')); + /** + * Metadata + */ + + apiRouter.get('/v2/metadata', iowrapper('listMetadata')); + router.use('/api/', apiRouter); router.get('*', async (req, res) => { diff --git a/platforms/yttrex/backend/bin/prepare-test-fixtures.ts b/platforms/yttrex/backend/bin/prepare-test-fixtures.ts index bc7771f50..2d69ea3b5 100755 --- a/platforms/yttrex/backend/bin/prepare-test-fixtures.ts +++ b/platforms/yttrex/backend/bin/prepare-test-fixtures.ts @@ -78,7 +78,7 @@ async function main(): Promise { path.resolve(basePath, `${metadata.id}.json`), JSON.stringify({ htmls, - metadata: metadata, + metadata, }) ); }); diff --git a/platforms/yttrex/backend/config/settings.json b/platforms/yttrex/backend/config/settings.json index 02c136d3f..39672a513 100644 --- a/platforms/yttrex/backend/config/settings.json +++ b/platforms/yttrex/backend/config/settings.json @@ -9,8 +9,7 @@ "answers": "answers", "recommendations": "recommendations", "ytvids": "ytvids", - "experiments": "experiments", - "directives": "directives", + "experiments": "experiments2", "emails": "emails", "tokens": "tokens", "creators": "creators", diff --git a/platforms/yttrex/backend/config/trexstats.json b/platforms/yttrex/backend/config/trexstats.json index 3477c0ca7..db230ddf3 100644 --- a/platforms/yttrex/backend/config/trexstats.json +++ b/platforms/yttrex/backend/config/trexstats.json @@ -91,7 +91,7 @@ }, { "name": "experiment", - "selector": { "type": "video", "experiment": { "$exists": true } } + "selector": { "type": "video", "experimentId": { "$exists": true } } } ] }, @@ -129,7 +129,7 @@ }, { "name": "experiment", - "selector": { "experiment": { "$exists": true }, "type": "search" } + "selector": { "experimentId": { "$exists": true }, "type": "search" } } ] }, @@ -145,10 +145,16 @@ { "name": "home", "selector": { "nature.type": "home" } }, { "name": "experiment", - "selector": { "experiment": { "$exists": true } } + "selector": { "experimentId": { "$exists": true } } } ] }, + { + "name": "experiments", + "column": "experiments2", + "timevar": "when", + "variables": [{ "name": "total", "selector": {} }] + }, { "name": "creators", "column": "creators", diff --git a/platforms/yttrex/backend/ecosystem.config.js b/platforms/yttrex/backend/ecosystem.config.js index 12f6c21c0..46952f73b 100644 --- a/platforms/yttrex/backend/ecosystem.config.js +++ b/platforms/yttrex/backend/ecosystem.config.js @@ -18,9 +18,7 @@ const yt = { cwd: __dirname, script: 'yarn leaveserv:watch', watch: false, - env_test: { - ...testEnv, - }, + env_test: testEnv, }, parser: { name: 'yt-parser', @@ -28,9 +26,7 @@ const yt = { cwd: __dirname, script: 'yarn parserv:watch', watch: false, - env_test: { - ...testEnv, - }, + env_test: testEnv, }, }; diff --git a/platforms/yttrex/backend/lib/CSV.js b/platforms/yttrex/backend/lib/CSV.js index b23469ae6..0a160773f 100644 --- a/platforms/yttrex/backend/lib/CSV.js +++ b/platforms/yttrex/backend/lib/CSV.js @@ -184,7 +184,7 @@ function unrollNested(metadata, options) { } if (options.experiment) { shared.experimentId = evidence.experiment.experimentId; - shared.evidencetag = evidence.experiment.evidencetag; + shared.researchTag = evidence.experiment.researchTag; shared.execount = evidence.experiment.execount; // note, this is only present in experiment as sharedDataPull // in routes/experiment perform an aggregate, but we diff --git a/platforms/yttrex/backend/lib/aggregated.js b/platforms/yttrex/backend/lib/aggregated.js index 12979bbe8..501a8ea6c 100644 --- a/platforms/yttrex/backend/lib/aggregated.js +++ b/platforms/yttrex/backend/lib/aggregated.js @@ -28,7 +28,7 @@ function dayData(reference) { dayOnext, dayId, reference: dayOnly, - m: m, + m, }; } @@ -60,7 +60,7 @@ function hourData(reference) { hourOnext, hourId, reference: hourOnly, - m: m, + m, }; } diff --git a/platforms/yttrex/backend/lib/api.js b/platforms/yttrex/backend/lib/api.js index f673f1f3b..208f37f6a 100644 --- a/platforms/yttrex/backend/lib/api.js +++ b/platforms/yttrex/backend/lib/api.js @@ -1,4 +1,7 @@ -const apiList = { +import * as publicRoutes from '../routes/public'; +import * as metadataRoutes from '../routes/metadata'; + +export const apiList = { systemInfo: require('../routes/system').systemInfo, processEvents2: require('../routes/events').processEvents2, getMirror: require('../routes/events').getMirror, @@ -7,11 +10,10 @@ const apiList = { unitById: require('../routes/htmlunit').unitById, /* the three currently used/tested for the public */ - getLast: require('../routes/public').getLast, - getLastHome: require('../routes/public').getLastHome, - getVideoId: require('../routes/public').getVideoId, - getRelated: require('../routes/public').getRelated, - getVideoCSV: require('../routes/public').getVideoCSV, + ...publicRoutes, + + // metadata + ...metadataRoutes, /* searches routes */ getQueries: require('../routes/searches').getQueries, @@ -21,7 +23,6 @@ const apiList = { getSearchKeywords: require('../routes/searches').getSearchKeywords, getSearchDetails: require('../routes/searches').getSearchDetails, - getByAuthor: require('../routes/public').getByAuthor, getPersonalCSV: require('../routes/personal').getPersonalCSV, /* return user' last videos */ @@ -62,7 +63,6 @@ const apiList = { // used from extension experimentChannel3: require('../routes/experiments').channel3, // used by guardoni to close it - concludeExperiment3: require('../routes/experiments').conclude3, /* for survey (with emails) */ recordAnswers: require('../routes/answers').recordAnswers, @@ -82,7 +82,6 @@ const apiList = { repullByCreator: require('../routes/youchoose').repullByCreator, recommendationById: require('../routes/youchoose').getRecommendationById, updateVideoRec: require('../routes/youchoose').updateVideoRec, - getCreatorRelated: require('../routes/public').getCreatorRelated, getCreatorStats: require('../routes/youchoose').getCreatorStats, creatorRegister: require('../routes/youchoose').creatorRegister, creatorVerify: require('../routes/youchoose').creatorVerify, @@ -101,7 +100,3 @@ const apiList = { adsPerChannel: require('../routes/ads').perChannel, adsUnbound: require('../routes/ads').unbound, }; - -module.exports = { - apiList, -}; diff --git a/platforms/yttrex/backend/lib/automo.js b/platforms/yttrex/backend/lib/automo.js index dededbaa0..4898396ad 100644 --- a/platforms/yttrex/backend/lib/automo.js +++ b/platforms/yttrex/backend/lib/automo.js @@ -390,7 +390,7 @@ async function deleteEntry(publicKey, id) { const metadata = await mongo3.deleteMany( mongoc, nconf.get('schema').metadata, - { id: id } + { id } ); await mongoc.close(); return { metadata }; @@ -399,7 +399,7 @@ async function deleteEntry(publicKey, id) { async function getRelatedByVideoId(videoId, options) { const mongoc = await mongo3.clientConnect({ concurrency: 1 }); const related = await mongo3.aggregate(mongoc, nconf.get('schema').metadata, [ - { $match: { videoId: videoId } }, + { $match: { videoId } }, { $sort: { savingTime: -1 } }, { $skip: options.skip }, { $limit: options.amount }, @@ -830,7 +830,7 @@ async function registerDirective(directives, directiveType) { directives, }); const mongoc = await mongo3.clientConnect({ concurrency: 1 }); - const exist = await mongo3.readOne(mongoc, nconf.get('schema').directives, { + const exist = await mongo3.readOne(mongoc, nconf.get('schema').experiments, { experimentId, }); @@ -845,7 +845,7 @@ async function registerDirective(directives, directiveType) { } /* else, we don't had such data, hence */ - await mongo3.writeOne(mongoc, nconf.get('schema').directives, { + await mongo3.writeOne(mongoc, nconf.get('schema').experiments, { when: new Date(), directiveType, directives, @@ -858,7 +858,7 @@ async function registerDirective(directives, directiveType) { async function pickDirective(experimentId) { const mongoc = await mongo3.clientConnect({ concurrency: 1 }); - const rb = await mongo3.readOne(mongoc, nconf.get('schema').directives, { + const rb = await mongo3.readOne(mongoc, nconf.get('schema').experiments, { experimentId, }); await mongoc.close(); diff --git a/platforms/yttrex/backend/lib/parser/html.ts b/platforms/yttrex/backend/lib/parser/html.ts index e1ec7d349..3c4b973e0 100644 --- a/platforms/yttrex/backend/lib/parser/html.ts +++ b/platforms/yttrex/backend/lib/parser/html.ts @@ -82,6 +82,12 @@ export function toMetadata( metadata.id = entry.source.html.metadataId; metadata.publicKey = entry.source.html.publicKey; metadata.timelineId = entry.source.html.timelineId; + if ( + entry.source.html.experimentId && + entry.source.html.experimentId.length > 0 + ) { + metadata.experimentId = entry.source.html.experimentId; + } if (entry.findings.nature.type === 'search') { metadata = { diff --git a/platforms/yttrex/backend/models/HTML.ts b/platforms/yttrex/backend/models/HTML.ts index 1f362bdf1..8530112d1 100644 --- a/platforms/yttrex/backend/models/HTML.ts +++ b/platforms/yttrex/backend/models/HTML.ts @@ -18,6 +18,8 @@ export const HTML = t.strict( processed: t.boolean, timelineId: t.union([t.string, t.undefined]), n: t.union([t.array(t.any), t.undefined]), + researchTag: t.union([t.string, t.undefined]), + experimentId: t.union([t.string, t.undefined]), }, 'HTML' ); diff --git a/platforms/yttrex/backend/package.json b/platforms/yttrex/backend/package.json index 0da00af48..bb172b1d1 100644 --- a/platforms/yttrex/backend/package.json +++ b/platforms/yttrex/backend/package.json @@ -28,7 +28,7 @@ "dependencies": { "abort-controller": "^3.0.0", "body-parser": "^1.19.1", - "bs58": "^3.1.0", + "bs58": "^4.0.1", "chardet": "^1.4.0", "cookie": "^0.3.1", "cors": "^2.8.5", @@ -44,37 +44,37 @@ "moment": "^2.29.2", "mongodb": "^4.3.1", "nacl-signature": "^1.0.0", - "nconf": "^0.8.5", + "nconf": "^0.11.3", "node-fetch": "^2.6.7", "numeral": "^2.0.6", "ts-endpoint": "^2.0.0", "ts-io-error": "^2.0.0", - "ts-node": "^10.8.0", + "ts-node": "^10.8.2", "ts-node-dev": "^2.0.0", "tweetnacl": "^0.14.5" }, "devDependencies": { "@types/body-parser": "^1.19.2", "@types/cors": "^2.8.12", - "@types/jest": "^27.5.1", + "@types/jest": "^27.5.2", "@types/jsdom": "^16.2.14", "@types/nconf": "^0.10.2", "@types/node": "^16.11.36", "@types/supertest": "^2.0.12", "@types/uuid": "^8.3.4", - "@typescript-eslint/eslint-plugin": "^5.27.0", - "@typescript-eslint/parser": "^5.27.0", - "eslint": "^8.16.0", + "@typescript-eslint/eslint-plugin": "^5.30.5", + "@typescript-eslint/parser": "^5.30.5", + "eslint": "^8.19.0", "eslint-config-prettier": "^8.5.0", - "eslint-config-standard-with-typescript": "^21.0.1", + "eslint-config-standard-with-typescript": "^22.0.0", "eslint-plugin-import": "^2.26.0", "eslint-plugin-node": "^11.1.0", - "eslint-plugin-promise": "^5.2.0", + "eslint-plugin-promise": "^6.0.0", "jest": "^27.5.1", - "prettier": "^2.6.2", - "supertest": "^6.2.2", + "prettier": "^2.7.1", + "supertest": "^6.2.4", "ts-jest": "^27.1.5", "tsconfig-paths": "^3.14.1", - "typescript": "^4.7.2" + "typescript": "^4.7.4" } } diff --git a/platforms/yttrex/backend/parsers/home.ts b/platforms/yttrex/backend/parsers/home.ts index af7540e2e..4892e61c4 100644 --- a/platforms/yttrex/backend/parsers/home.ts +++ b/platforms/yttrex/backend/parsers/home.ts @@ -340,7 +340,7 @@ function actualHomeProcess(D: Document, clientTime: Date): HomeProcess { sections.length ); debugSizes(effective); - return { selected: effective as any[], sections: sections }; + return { selected: effective as any[], sections }; /* sections would be removed before being saved in mongodb */ } diff --git a/platforms/yttrex/backend/parsers/shared.js b/platforms/yttrex/backend/parsers/shared.js index 8c51b252a..be97f1623 100644 --- a/platforms/yttrex/backend/parsers/shared.js +++ b/platforms/yttrex/backend/parsers/shared.js @@ -1,53 +1,52 @@ const debug = require('debug')('parser:shared'); const debuge = require('debug')('parser:shared:error'); -const url = require('url'); +const URL = require('url'); /* shared functions used from video and home */ export function getThumbNailHref(e) { - // e is an 'element' from .querySelectorAll('ytd-compact-video-renderer') - let thumbnailHref = null; - try { - const refe = e.querySelector('.ytd-thumbnail-overlay-time-status-renderer'); - if(!refe) - return null; - - const thumbnailSrc = refe.closest('a').querySelector('img').getAttribute('src'); - if(!thumbnailSrc) - return null; - - // eslint-disable-next-line node/no-deprecated-api - const c = url.parse(thumbnailSrc); - thumbnailHref = 'https://' + c.host + c.pathname; - } catch(e) { - debuge("thumbnail mining error: %s", e.message); - } - return thumbnailHref; + // e is an 'element' from .querySelectorAll('ytd-compact-video-renderer') + let thumbnailHref = null; + try { + const refe = e.querySelector('.ytd-thumbnail-overlay-time-status-renderer'); + if (!refe) return null; + + const thumbnailSrc = refe + .closest('a') + .querySelector('img') + .getAttribute('src'); + if (!thumbnailSrc) return null; + + // eslint-disable-next-line node/no-deprecated-api, n/no-deprecated-api + const c = URL.parse(thumbnailSrc); + thumbnailHref = 'https://' + c.host + c.pathname; + } catch (e) { + debuge('thumbnail mining error: %s', e.message); + } + return thumbnailHref; } export function logged(D) { - const avatarN = D.querySelectorAll('button#avatar-btn'); - const loginN = D.querySelectorAll('[href^="https://accounts.google.com/ServiceLogin"]'); - const avalen = avatarN ? avatarN.length : 0; - const logilen = loginN ? loginN.length : 0; - - // login button | avatar button len - if(logilen && !avalen) - return false; - if(avalen && !logilen) - return true; - - debug("Inconsistent condition avatar %d login %d", avalen, logilen); - return null; + const avatarN = D.querySelectorAll('button#avatar-btn'); + const loginN = D.querySelectorAll( + '[href^="https://accounts.google.com/ServiceLogin"]' + ); + const avalen = avatarN ? avatarN.length : 0; + const logilen = loginN ? loginN.length : 0; + + // login button | avatar button len + if (logilen && !avalen) return false; + if (avalen && !logilen) return true; + + debug('Inconsistent condition avatar %d login %d', avalen, logilen); + return null; } export function fixHumanizedTime(inputstr) { - // this function fix the time 0:10, 10:10, in HH:MM:SS - if(inputstr.length === 4) - return '0:0' + inputstr; - if(inputstr.length === 5) - return '0:' + inputstr; - if(inputstr.length > 9) - debug("Warning this is weird in fixHumanizedTime: [%s]", inputstr); - return inputstr; + // this function fix the time 0:10, 10:10, in HH:MM:SS + if (inputstr.length === 4) return '0:0' + inputstr; + if (inputstr.length === 5) return '0:' + inputstr; + if (inputstr.length > 9) + debug('Warning this is weird in fixHumanizedTime: [%s]', inputstr); + return inputstr; } diff --git a/platforms/yttrex/backend/parsers/video.ts b/platforms/yttrex/backend/parsers/video.ts index 60b3347f1..312fcf327 100644 --- a/platforms/yttrex/backend/parsers/video.ts +++ b/platforms/yttrex/backend/parsers/video.ts @@ -243,7 +243,7 @@ function relatedMetadata(e: any, i: number): ParsedInfo | null { verified, foryou, videoId, - params: params, + params, recommendedSource: source, recommendedTitle: mined ? mined.title : title || null, recommendedLength, @@ -508,7 +508,7 @@ export function processVideo( return { title, type: 'video', - params: params, + params, videoId, login, publicationString, diff --git a/platforms/yttrex/backend/routes/__tests__/metadata.e2e.ts b/platforms/yttrex/backend/routes/__tests__/metadata.e2e.ts new file mode 100644 index 000000000..901673248 --- /dev/null +++ b/platforms/yttrex/backend/routes/__tests__/metadata.e2e.ts @@ -0,0 +1,175 @@ +/* eslint-disable import/first */ +// mock curly module +jest.mock('../../lib/curly'); +jest.mock('fetch-opengraph'); + +// import test utils +import { fc } from '@shared/test'; +import moment from 'moment'; +import _ from 'lodash'; +import { v4 as uuid } from 'uuid'; +import { + ParsedInfoArb, + VideoMetadataArb, +} from '../../tests/arbitraries/Metadata.arb'; +import { GetTest, Test } from '../../tests/Test'; + +describe('Metadata API', () => { + let test: Test; + + beforeAll(async () => { + test = await GetTest(); + }); + + afterAll(async () => { + await test.mongo.close(); + }); + + describe('GET /v2/metadata', () => { + const researchTag = 'test-tag'; + + it('fails when amount is not valid ', async () => { + const { body } = await test.app + .get(`/api/v2/metadata`) + .query({ + amount: 'not-valid', + }) + .expect(400); + + expect(body).toMatchObject({ + name: 'Bad Request', + }); + }); + + it('succeeds with metadata when filtering by "researchTag"', async () => { + const amount = 10; + + const metadata = fc.sample(VideoMetadataArb, 100).map((m) => ({ + ...m, + savingTime: new Date(), + researchTag, + })); + + await test.mongo3.insertMany( + test.mongo, + test.config.get('schema').metadata, + metadata + ); + + const expectedMetadata = metadata + .filter( + (a) => + a.savingTime.getTime() > + new Date(moment().startOf('day').toISOString()).getTime() + ) + .sort((a, b) => b.savingTime.getTime() - a.savingTime.getTime()) + .slice(0, amount) + .map((m) => { + return { + ...m, + clientTime: m.clientTime.toISOString(), + publicationTime: m.publicationTime.toISOString(), + savingTime: m.savingTime.toISOString(), + related: m.related.map((r) => { + return { + ...r, + elems: r.elems ?? null, + thumbnailHref: r.thumbnailHref ?? null, + publicationTime: r.publicationTime.toISOString(), + recommendedPubTime: r.recommendedPubTime + ? r.recommendedPubTime.toISOString() + : null, + }; + }), + }; + }); + + const { body } = await test.app + .get(`/api/v2/metadata`) + .query({ + researchTag, + amount, + }) + .expect(200); + + expect(body.length).toBe(expectedMetadata.length); + expect(body).toMatchObject(expectedMetadata); + + await test.mongo3.deleteMany( + test.mongo, + test.config.get('schema').metadata, + { + id: { + $in: expectedMetadata.map((m) => m.id), + }, + } + ); + }); + + it('succeeds with metadata', async () => { + const experimentId = fc.sample(fc.uuid(), 1)[0]; + const amount = 10; + + const metadata = fc.sample(VideoMetadataArb, 100).map((m) => ({ + ...m, + savingTime: new Date(), + experimentId, + })); + + await test.mongo3.insertMany( + test.mongo, + test.config.get('schema').metadata, + metadata + ); + + const expectedMetadata = metadata + .filter( + (a) => + a.savingTime.getTime() > + new Date(moment().startOf('day').toISOString()).getTime() + ) + .sort((a, b) => b.savingTime.getTime() - a.savingTime.getTime()) + .slice(0, amount) + .map((m) => { + return { + ...m, + clientTime: m.clientTime.toISOString(), + publicationTime: m.publicationTime.toISOString(), + savingTime: m.savingTime.toISOString(), + related: m.related.map((r) => { + return { + ...r, + elems: r.elems ?? null, + thumbnailHref: r.thumbnailHref ?? null, + publicationTime: r.publicationTime.toISOString(), + recommendedPubTime: r.recommendedPubTime + ? r.recommendedPubTime.toISOString() + : null, + }; + }), + }; + }); + + const { body } = await test.app + .get(`/api/v2/metadata`) + .query({ + experimentId, + amount, + }) + .expect(200); + + expect(body.length).toBe(expectedMetadata.length); + expect(body).toMatchObject(expectedMetadata); + + await test.mongo3.deleteMany( + test.mongo, + test.config.get('schema').metadata, + { + id: { + $in: expectedMetadata.map((m) => m.id), + }, + } + ); + }); + }); +}); diff --git a/platforms/yttrex/backend/routes/__tests__/personal.e2e.ts b/platforms/yttrex/backend/routes/__tests__/personal.e2e.ts index 629ebf9b2..462ec5ec8 100644 --- a/platforms/yttrex/backend/routes/__tests__/personal.e2e.ts +++ b/platforms/yttrex/backend/routes/__tests__/personal.e2e.ts @@ -30,7 +30,7 @@ describe('Events', () => { appTest = await GetTest(); await appTest.mongo3.insertMany( appTest.mongo, - appTest.config.get('schema').directives, + appTest.config.get('schema').experiments, [experiment] ); }); @@ -43,6 +43,8 @@ describe('Events', () => { describe('GetPersonalByExperimentId', () => { test('succeeds with one metadata', async () => { + const researchTag = 'test-tag'; + const keys = await foldTEOrThrow(bs58.makeKeypair('')); const fixture = pipe( @@ -60,6 +62,7 @@ describe('Events', () => { ...d, ...fixture.sources[0], experimentId: experiment.experimentId, + researchTag, })); // create a signature @@ -71,8 +74,10 @@ describe('Events', () => { await appTest.app .post(`/api/v2/events`) .set('x-yttrex-version', version) + .set('x-yttrex-build', '') .set('X-yttrex-publicKey', keys.publicKey) .set('x-yttrex-signature', signature) + .set('x-yttrex-nonAuthCookieId', 'local') .set('accept-language', 'en') .send(data) .expect(200); @@ -105,20 +110,20 @@ describe('Events', () => { // run parser await GetParserProvider('htmls', { - db, - parsers: parsers, - getContributions: getLastHTMLs({ db }), - saveResults: updateMetadataAndMarkHTML({ db }), - getEntryDate: (e) => e.html.savingTime, - getEntryNatureType: (e) => e.html.nature.type, + db, + parsers: parsers, + getContributions: getLastHTMLs({ db }), + saveResults: updateMetadataAndMarkHTML({ db }), + getEntryDate: (e) => e.html.savingTime, + getEntryNatureType: (e) => e.html.nature.type, + }) + .run({ + singleUse: true, + stop: 1, + htmlAmount: 100, + backInTime: 10, }) - .run({ - singleUse: true, - stop: 1, - htmlAmount: 100, - backInTime: 10, - }) - .then((r) => r.payload.metadata); + .then((r) => r.payload.metadata); // wait for the parser to process the html await sleep(5 * 1000); @@ -138,7 +143,7 @@ describe('Events', () => { href: 'https://www.youtube.com/', publicKey: keys.publicKey, clientTime: fixture.sources[0].clientTime, - timelineId: null, + experimentId: experiment.experimentId, }, ], }); diff --git a/platforms/yttrex/backend/routes/__tests__/public.e2e.ts b/platforms/yttrex/backend/routes/__tests__/public.e2e.ts index 2e7a30145..bdf290061 100644 --- a/platforms/yttrex/backend/routes/__tests__/public.e2e.ts +++ b/platforms/yttrex/backend/routes/__tests__/public.e2e.ts @@ -4,9 +4,6 @@ jest.mock('../../lib/curly'); jest.mock('fetch-opengraph'); // import test utils -import { ContentCreator } from '@shared/models/ContentCreator'; -import { Recommendation } from '@shared/models/Recommendation'; -import { Video } from '@shared/models/Video'; import { fc } from '@shared/test'; import _ from 'lodash'; import { v4 as uuid } from 'uuid'; diff --git a/platforms/yttrex/backend/routes/directives.js b/platforms/yttrex/backend/routes/directives.js index e8e2fa63d..b65492aae 100644 --- a/platforms/yttrex/backend/routes/directives.js +++ b/platforms/yttrex/backend/routes/directives.js @@ -187,7 +187,7 @@ async function getPublic(req) { const publicDirectives = await mongo3.readLimit( mongoc, - nconf.get('schema').directives, + nconf.get('schema').experiments, filter, { when: -1 }, 20, diff --git a/platforms/yttrex/backend/routes/events.js b/platforms/yttrex/backend/routes/events.js index 81330175e..f6f5d9109 100644 --- a/platforms/yttrex/backend/routes/events.js +++ b/platforms/yttrex/backend/routes/events.js @@ -154,6 +154,8 @@ async function processEvents2(req) { const headers = processHeaders(req.headers, EXPECTED_HDRS); if (headers.error) return headerError(headers); + // toodo: this should be returned as 400/500 error + if (!utils.verifyRequestSignature(req)) { debug('Verification fail (signature %s)', headers.signature); return { diff --git a/platforms/yttrex/backend/routes/experiments.js b/platforms/yttrex/backend/routes/experiments.js index db51554be..49547112c 100644 --- a/platforms/yttrex/backend/routes/experiments.js +++ b/platforms/yttrex/backend/routes/experiments.js @@ -248,7 +248,7 @@ async function list(req) { const configured = await mongo3.readLimit( mongoc, - nconf.get('schema').directives, + nconf.get('schema').experiments, filter, { when: -1 }, options.amount, @@ -264,7 +264,7 @@ async function list(req) { const total = await mongo3.count( mongoc, - nconf.get('schema').directives, + nconf.get('schema').experiments, filter ); @@ -327,7 +327,7 @@ async function emergency(req) { const experimentId = params.getString(req, 'experimentId', true); const directive = await mongo3.readOne( mongoc, - nconf.get('schema').directives, + nconf.get('schema').experiments, { experimentId, directiveType: 'comparison', @@ -399,7 +399,7 @@ async function channel3(req) { const fields = [ 'href', 'experimentId', - 'evidencetag', + 'researchTag', 'execount', 'newProfile', 'profileName', @@ -410,8 +410,7 @@ async function channel3(req) { experimentInfo.testName = req.body.testTime ?? new Date().toISOString(); experimentInfo.publicKey = _.get(req.body, 'config.publicKey'); - if(!experimentInfo.experimentId) - return { json: { ignored: true }}; + if (!experimentInfo.experimentId) return { json: { ignored: true } }; debug('Experiment info %O', experimentInfo); @@ -422,24 +421,11 @@ async function channel3(req) { debug( "Marked experiment as 'active' — %j", - _.pick(retval, ['evidencetag', 'execount', 'directiveType']) + _.pick(retval, ['researchTag', 'execount', 'directiveType']) ); return { json: retval }; } -async function conclude3(req) { - const testTime = req.params.testTime; - debug('Conclude3 received: %s', testTime); - if (testTime.length < 10) return { status: 403 }; - - const test = moment(testTime); - if (!test.isValid) return { status: 403 }; - - const retval = await automo.concludeExperiment(testTime); - // retval is {"acknowledged":true,"modifiedCount":0,"upsertedId":null,"upsertedCount":0,"matchedCount":0} - return { json: retval }; -} - module.exports = { /* used by the webapps */ csv, @@ -450,5 +436,4 @@ module.exports = { /* used by the browser extension/guardoni */ channel3, - conclude3, }; diff --git a/platforms/yttrex/backend/routes/metadata.ts b/platforms/yttrex/backend/routes/metadata.ts new file mode 100644 index 000000000..52792939b --- /dev/null +++ b/platforms/yttrex/backend/routes/metadata.ts @@ -0,0 +1,65 @@ +import * as endpoints from '@shared/endpoints/helper'; +import { v2 } from '@yttrex/shared/endpoints'; +import D from 'debug'; +import _ from 'lodash'; +import automo from '../lib/automo'; +import moment from 'moment'; +import CSV from '../lib/CSV'; + +const debug = D('routes:metadata'); + +// This variables is used as cap in every readLimit below +const PUBLIC_AMOUNT_ELEMS = 100; + +const listMetadata = async (req: Express.Request): Promise => { + const { + query: { + experimentId, + researchTag, + amount = PUBLIC_AMOUNT_ELEMS, + skip = 0, + format, + }, + } = endpoints.decodeOrThrowRequest(v2.Metadata.ListMetadata, req); + + const metadata = await automo.getMetadataByFilter( + { + experimentId, + researchTag, + }, + { + amount, + skip, + } + ); + + debug( + 'Returning metadata by experimentId %s, %d evidences', + experimentId, + _.size(metadata) + ); + + if (format === 'csv') { + const csv = CSV.produceCSVv1(metadata); + let filename = `metadata`; + filename += experimentId ? `-experiment-${experimentId}` : ''; + filename += researchTag ? `-research_tag-${researchTag}` : ''; + filename += '-' + moment().format('YY-MM-DD') + '.csv'; + + debug('VideoCSV: produced %d bytes, returning %s', _.size(csv), filename); + + // if (!_.size(csv)) return { text: 'Error, Zorry: 🤷' }; + + return { + headers: { + 'Content-Type': 'csv/text', + 'Content-Disposition': `attachment; filename=${filename}`, + }, + text: csv, + }; + } + + return { json: metadata }; +}; + +export { listMetadata }; diff --git a/platforms/yttrex/backend/routes/personal.ts b/platforms/yttrex/backend/routes/personal.ts index df79db9c9..8c1390dcc 100644 --- a/platforms/yttrex/backend/routes/personal.ts +++ b/platforms/yttrex/backend/routes/personal.ts @@ -63,7 +63,7 @@ async function getPersonalCSV(req): Promise { const data = await automo.getMetadataByPublicKey(k, { amount: CSV_MAX_SIZE, skip: 0, - type: type, + type, }); /* this return of videos or homepage, they generated slightly different CSV formats */ diff --git a/platforms/yttrex/backend/routes/profile.js b/platforms/yttrex/backend/routes/profile.js index 9850a22ac..1e3c43dea 100644 --- a/platforms/yttrex/backend/routes/profile.js +++ b/platforms/yttrex/backend/routes/profile.js @@ -73,7 +73,7 @@ async function createAndOrJoinTag(req) { }); const ret = {json: {}}; - const exists = await mongo3.readOne(mongoc, nconf.get('schema').groups, { id: id }); + const exists = await mongo3.readOne(mongoc, nconf.get('schema').groups, { id }); if(_.get(exists, 'id')) { ret.json.group = Object(exists); ret.created = false; diff --git a/platforms/yttrex/backend/routes/public.js b/platforms/yttrex/backend/routes/public.ts similarity index 89% rename from platforms/yttrex/backend/routes/public.js rename to platforms/yttrex/backend/routes/public.ts index 3498718d9..2756baf3c 100644 --- a/platforms/yttrex/backend/routes/public.js +++ b/platforms/yttrex/backend/routes/public.ts @@ -1,20 +1,20 @@ -const _ = require('lodash'); -const moment = require('moment'); -const debug = require('debug')('routes:public'); - -const params = require('../lib/params'); -const automo = require('../lib/automo'); -const utils = require('../lib/utils'); -const CSV = require('../lib/CSV'); -const cache = require('../lib/cache'); -const endpoints = require('../lib/endpoint'); -const { v1 } = require('@shared/endpoints'); -const structured = require('../lib/structured'); +import _ from 'lodash'; +import moment from 'moment'; +import D from 'debug'; +import params from '../lib/params'; +import automo from '../lib/automo'; +import utils from '../lib/utils'; +import CSV from '../lib/CSV'; +import cache from '../lib/cache'; +import * as endpoints from '@shared/endpoints/helper'; +import { v1 } from '@yttrex/shared/endpoints'; +import structured from '../lib/structured'; +const debug = D('routes:public'); // This variables is used as cap in every readLimit below const PUBLIC_AMOUNT_ELEMS = 100; -async function getLast(req) { +async function getLast(req: Express.Request): Promise { if (cache.stillValid('last')) return { json: cache.repullCache('last') }; // if not initialized or if the cache time is expired: do the query @@ -38,7 +38,7 @@ async function getLast(req) { /* the complex entry has nested metadata */ const reduction = _.map(last, function (ce) { const lst = _.last(_.orderBy(ce.info, 'savingTime')).savingTime; - const d = moment.duration(moment(lst) - moment()); + const d = moment.duration((moment(lst) as any) - (moment() as any)); const timeago = d.humanize(); return { title: ce.info[0].title, @@ -59,7 +59,7 @@ async function getLast(req) { }; } -async function getLastHome() { +async function getLastHome(): Promise { const DEFMAX = 100; const homelist = await automo.getMetadataByFilter( @@ -92,7 +92,7 @@ async function getLastHome() { }); return memo; }, - [] + [] as any[] ); debug( @@ -103,7 +103,7 @@ async function getLastHome() { return { json: rv }; } -function ensureRelated(rv) { +function ensureRelated(rv): any { /* for each related it is called and only the basic info used in 'compare' * page get returned. return 'null' if content is not complete */ const demanded = [ @@ -124,7 +124,7 @@ function ensureRelated(rv) { : sele; } -async function getVideoId(req) { +async function getVideoId(req): Promise { const { amount, skip } = params.optionParsing( req.params.paging, PUBLIC_AMOUNT_ELEMS @@ -161,7 +161,7 @@ async function getVideoId(req) { return { json: evidences }; } -async function getRelated(req) { +async function getRelated(req): Promise { const { amount, skip } = params.optionParsing( req.params.paging, PUBLIC_AMOUNT_ELEMS @@ -186,7 +186,9 @@ async function getRelated(req) { 'videoId', ]); }); - meta.timeago = moment.duration(meta.savingTime - moment()).humanize(); + meta.timeago = moment + .duration(meta.savingTime - (moment() as any)) + .humanize(); return _.omit(meta, ['_id', 'publicKey']); }); debug( @@ -197,7 +199,7 @@ async function getRelated(req) { return { json: evidences }; } -async function getVideoCSV(req) { +async function getVideoCSV(req): Promise { // /api/v1/videoCSV/:videoId/:amount const MAXENTRY = 2800; const { amount, skip } = params.optionParsing(req.params.paging, MAXENTRY); @@ -228,7 +230,7 @@ async function getVideoCSV(req) { }; } -async function getByAuthor(req) { +async function getByAuthor(req): Promise { /* this API do not return the standard format with videos and related inside, * but a data format ready for the visualization provided - this has been * temporarly suspended: https://github.com/tracking-exposed/youtube.tracking.exposed/issues/18 */ @@ -297,7 +299,7 @@ async function getByAuthor(req) { return { json: retval.result }; } -async function getCreatorRelated(req) { +async function getCreatorRelated(req): Promise { /* this is the route invoked by API /api/v3/creator/:channelId/related/:amount? and differs from the others because take as an input a @@ -332,7 +334,7 @@ async function getCreatorRelated(req) { } } -module.exports = { +export { getLast, getLastHome, getVideoId, diff --git a/platforms/yttrex/backend/routes/youchoose.js b/platforms/yttrex/backend/routes/youchoose.js index 9461265d9..586db239d 100644 --- a/platforms/yttrex/backend/routes/youchoose.js +++ b/platforms/yttrex/backend/routes/youchoose.js @@ -5,7 +5,7 @@ const fetchOpengraph = require('fetch-opengraph'); const ycai = require('../lib/ycai'); const curly = require('../lib/curly'); -const endpoints = require('../lib/endpoint'); +const endpoints = require('@shared/endpoints/helper'); const { v3 } = require('@yttrex/shared/endpoints'); const structured = require('../lib/structured'); diff --git a/platforms/yttrex/backend/scripts/build-indexes.js b/platforms/yttrex/backend/scripts/build-indexes.js index 8e106c04e..46a3842fd 100644 --- a/platforms/yttrex/backend/scripts/build-indexes.js +++ b/platforms/yttrex/backend/scripts/build-indexes.js @@ -52,6 +52,21 @@ checkret('errors id', ret); ret = db.errors.createIndex({ when: -1 }); checkret('errors when', ret); + +ret = db.ads.createIndex({ metadataId: 1 }); +checkret('ads metadataId', ret); +ret = db.ads.createIndex({ savingTime: -1 }); +checkret('ads savingTime', ret); +ret = db.ads.createIndex({ id: 1 }, { unique: true }); +checkret('ads id', ret); + +ret = db.experiments2.createIndex({ experimentId: -1, unique: true }); +checkret('experiments2 experimentId', ret); +ret = db.experiments2.createIndex({ when: -1 }); +checkret('experiments2 when', ret); + + +/* below this the collections are for youchoose */ ret = db.recommendations.createIndex({ urlId: 1 }, { unique: true }); checkret('recommendations urlId', ret); ret = db.recommendations.createIndex({ when: -1 }); @@ -62,16 +77,6 @@ checkret('ytvids videoId', ret); ret = db.ytvids.createIndex({ creatorId: -1 }); checkret('ytvids creatorId', ret); -ret = db.directives.createIndex({ experimentId: -1 }, { unique: true }); -checkret('directives experimentId', ret); - -ret = db.experiments.createIndex({ experimentId: -1 }); -checkret('experiments experimentId', ret); -ret = db.experiments.createIndex({ publicKey: -1 }); -checkret('experiments publicKey', ret); -ret = db.experiments.createIndex({ testTime: -1 }); -checkret('experiments testTime', ret); - ret = db.tokens.createIndex({ channelId: 1 }, { unique: true }); checkret('tokens unique channelId', ret); ret = db.tokens.createIndex({ verificationToken: 1 }); @@ -87,12 +92,6 @@ checkret('creators channelId', ret); ret = db.creators.createIndex({ accessToken: 1 }); checkret('creators accessToken', ret); -ret = db.ads.createIndex({ metadataId: 1 }); -checkret('ads metadataId', ret); -ret = db.ads.createIndex({ savingTime: -1 }); -checkret('ads savingTime', ret); -ret = db.ads.createIndex({ id: 1 }, { unique: true }); -checkret('ads id', ret); function checkret(info, retval) { retval.info = info; diff --git a/platforms/yttrex/backend/tests/Test.ts b/platforms/yttrex/backend/tests/Test.ts index e71a16d5a..105d70f84 100644 --- a/platforms/yttrex/backend/tests/Test.ts +++ b/platforms/yttrex/backend/tests/Test.ts @@ -28,6 +28,7 @@ export const GetTest = async (): Promise => { config.set('mongoHost', '0.0.0.0'); config.set('key', 'test-key'); config.set('storage', '_test_htmls'); + config.set('mongoDb', 'yttrex-test'); const mongo = await mongo3.clientConnect({ concurrency: 1 }); diff --git a/platforms/yttrex/backend/tests/arbitraries/Metadata.arb.ts b/platforms/yttrex/backend/tests/arbitraries/Metadata.arb.ts index 948049ed8..06ecd00e8 100644 --- a/platforms/yttrex/backend/tests/arbitraries/Metadata.arb.ts +++ b/platforms/yttrex/backend/tests/arbitraries/Metadata.arb.ts @@ -5,6 +5,7 @@ import { ParsedInfo, VideoMetadata, } from '@yttrex/shared/models/Metadata'; +import { subDays } from 'date-fns'; import { getArbitrary } from 'fast-check-io-ts'; import * as t from 'io-ts'; @@ -51,7 +52,7 @@ export const VideoMetadataArb = getArbitrary( ).map((ad) => ({ ...ad, id: fc.sample(fc.uuid(), 1)[0], - savingTime: fc.sample(fc.date(), 1)[0], + savingTime: subDays(new Date(), fc.sample(fc.nat(), 1)[0]), clientTime: fc.sample(fc.date(), 1)[0], publicationTime: fc.sample(fc.date(), 1)[0], params: fc.sample(fc.dictionary(fc.string(), fc.string()), 1)[0], diff --git a/platforms/yttrex/docs/package.json b/platforms/yttrex/docs/package.json index 1f34e6688..c16d1a2b2 100644 --- a/platforms/yttrex/docs/package.json +++ b/platforms/yttrex/docs/package.json @@ -26,7 +26,7 @@ "devDependencies": { "@docusaurus/module-type-aliases": "2.0.0-beta.14", "@tsconfig/docusaurus": "^1.0.4", - "typescript": "^4.7.2" + "typescript": "^4.7.4" }, "browserslist": { "production": [ diff --git a/platforms/yttrex/extension/.env.development b/platforms/yttrex/extension/.env.development index fd1864f3d..92c8d4b98 100644 --- a/platforms/yttrex/extension/.env.development +++ b/platforms/yttrex/extension/.env.development @@ -4,3 +4,6 @@ WEB_ROOT=http://localhost:1313 DATA_CONTRIBUTION_ENABLED=true FLUSH_INTERVAL=4000 DEBUG=@trex* +PUBLIC_KEY=H7AsuUszehN4qKTj2GYYwNNzkJVqUQBRo2wgKevzeUwx +SECRET_KEY=v6pyCC8TzKd1uq7LnGaLQDaZ4qmJwKCne7moXwr1EJthxne4sqBWPkHwqyH8QH5n9pNQGAGBeA9hZ1jwy4hUyeW + diff --git a/platforms/yttrex/extension/__tests__/app.spec.ts b/platforms/yttrex/extension/__tests__/app.spec.ts index 2f8610350..958461d55 100644 --- a/platforms/yttrex/extension/__tests__/app.spec.ts +++ b/platforms/yttrex/extension/__tests__/app.spec.ts @@ -1,8 +1,5 @@ -import { boot } from '@shared/extension/app'; -import { - handleServerLookup, - initializeKey, -} from '@shared/extension/chrome/background/account'; +import { boot, BootOpts } from '@shared/extension/app'; +import { handleServerLookup } from '@shared/extension/chrome/background/account'; import { load } from '@shared/extension/chrome/background/index'; import { handleSyncMessage } from '@shared/extension/chrome/background/sync'; import config from '@shared/extension/config'; @@ -43,7 +40,7 @@ const backgroundOpts = { const keys = { publicKey: process.env.PUBLIC_KEY, - secretKey: process.env.SECRET_KEY + secretKey: process.env.SECRET_KEY, }; let supporterId: string, ytURL: string; @@ -58,12 +55,10 @@ const getConfig = () => ({ active: true, ux: true, href: (() => ytURL)(), - evidencetag: 'fake-tag', - researchTag: '', + researchTag: 'fake-tag', experimentId: '1', execount: 1, newProfile: false, - testTime: new Date().toISOString(), directiveType: 'comparison', }); @@ -98,7 +93,7 @@ chrome.storage.local.set.mockImplementation((obj, cb: any) => { chrome.runtime.onMessage.addListener(chromeListener); -const bootConfig = () => ({ +const bootConfig = (): BootOpts => ({ payload: { config: keys, href: window.location.href, @@ -157,6 +152,8 @@ describe('YT App', () => { await sleep(4000); + appContext.destroy(); + // custom events should be registered on booting expect(eventsRegisterSpy).toHaveBeenCalled(); @@ -169,6 +166,7 @@ describe('YT App', () => { const { handle: _handle, ..._videoOpts } = videoMatcher; expect(handleHomeSpy).toHaveBeenCalledTimes(1); + expect(handleVideoSpy).not.toHaveBeenCalled(); // banner match // const { handle: _bannerHandle, ...bannerOpts } = leafMatcherBanner; @@ -182,7 +180,8 @@ describe('YT App', () => { i + 1, expect.any(HTMLElement), channel1Opts, - 'channel1' + 'channel1', + expect.any(Object) ); }); @@ -195,7 +194,8 @@ describe('YT App', () => { i + 1, expect.any(HTMLElement), channel2Opts, - 'channel2' + 'channel2', + expect.any(Object) ); }); @@ -207,7 +207,8 @@ describe('YT App', () => { i + 1, expect.any(HTMLElement), channel3Opts, - 'channel3' + 'channel3', + expect.any(Object) ); }); @@ -219,7 +220,7 @@ describe('YT App', () => { }); expect(response.status).toBe(200); - expect(response.data.ads.length).toBe(71); + expect(response.data.ads.length).toBe(100); expect(response.data).toMatchObject({ supporter: { @@ -231,12 +232,9 @@ describe('YT App', () => { home: 1, }, }); - - appContext.destroy(); }); test('Collect evidence from video page', async () => { - handleLeafBannerSpy.mockClear(); const appContext = await boot(bootConfig()); ytURL = 'https://www.youtube.com/watch?v=55ud4_Cdbrc'; @@ -252,6 +250,8 @@ describe('YT App', () => { await sleep(4000); + appContext.destroy(); + // custom events should be registered on booting expect(eventsRegisterSpy).toHaveBeenCalled(); @@ -266,7 +266,8 @@ describe('YT App', () => { expect(handleVideoSpy).toHaveBeenCalledWith( window.document.body, videoOpts, - 'video' + 'video', + { ...appContext.config, href: ytURL } ); expect(handleVideoSpy).toBeCalledTimes(1); @@ -276,44 +277,51 @@ describe('YT App', () => { 1, expect.any(HTMLElement), bannerOpts, - 'banner' + 'banner', + { ...appContext.config, href: ytURL } ); expect(handleLeafBannerSpy).toHaveBeenCalledTimes(2); // channel3 match + const leafChannel1Count = 132; const { handle: _channel1Handle, ...channel1Opts } = leafMatcherChannel1; - expect(handleLeafChannel1Spy).toHaveBeenCalledTimes(132); - Array.from({ length: 132 }).map((n, i) => { + expect(handleLeafChannel1Spy).toHaveBeenCalledTimes(leafChannel1Count); + Array.from({ length: leafChannel1Count }).map((n, i) => { expect(handleLeafChannel1Spy).toHaveBeenNthCalledWith( i + 1, expect.any(HTMLElement), channel1Opts, - 'channel1' + 'channel1', + { ...appContext.config, href: ytURL } ); }); - // channel2 match + // channel2 + const leafChannel2Count = 204; const { handle: _channel2Handle, ...channel2Opts } = leafMatcherChannel2; - expect(handleLeafChannel2Spy).toHaveBeenCalledTimes(204); + expect(handleLeafChannel2Spy).toHaveBeenCalledTimes(leafChannel2Count); - Array.from({ length: 204 }).map((n, i) => { + Array.from({ length: leafChannel2Count }).map((n, i) => { expect(handleLeafChannel2Spy).toHaveBeenNthCalledWith( i + 1, expect.any(HTMLElement), channel2Opts, - 'channel2' + 'channel2', + { ...appContext.config, href: ytURL } ); }); // channel3 match + const leafChannel3Count = 22; const { handle: _channel3Handle, ...channel3Opts } = leafMatcherChannel3; - expect(handleLeafChannel3Spy).toHaveBeenCalledTimes(22); - Array.from({ length: 22 }).map((n, i) => { + expect(handleLeafChannel3Spy).toHaveBeenCalledTimes(leafChannel3Count); + Array.from({ length: leafChannel3Count }).map((n, i) => { expect(handleLeafChannel3Spy).toHaveBeenNthCalledWith( i + 1, expect.any(HTMLElement), channel3Opts, - 'channel3' + 'channel3', + { ...appContext.config, href: ytURL } ); }); @@ -346,7 +354,5 @@ describe('YT App', () => { home: 1, }, }); - - appContext.destroy(); }); }); diff --git a/platforms/yttrex/extension/jest.setup.ts b/platforms/yttrex/extension/jest.setup.ts index b22a95861..d7df56cef 100644 --- a/platforms/yttrex/extension/jest.setup.ts +++ b/platforms/yttrex/extension/jest.setup.ts @@ -1,3 +1,7 @@ +import path from 'path'; + +require('dotenv').config({ path: path.resolve(__dirname, '.env.development') }); + process.env.BUILD_DATE = new Date().toISOString(); process.env.VERSION = '0.1-TEST'; process.env.API_ROOT = 'http://localhost:9000/api'; diff --git a/platforms/yttrex/extension/package.json b/platforms/yttrex/extension/package.json index 8d64e5d7c..c19e55550 100644 --- a/platforms/yttrex/extension/package.json +++ b/platforms/yttrex/extension/package.json @@ -25,17 +25,17 @@ }, "devDependencies": { "@types/mini-css-extract-plugin": "^2.5.1", - "@typescript-eslint/eslint-plugin": "^5.27.0", + "@typescript-eslint/eslint-plugin": "^5.30.5", "autoprefixer": "^6.7.7", "copy-webpack-plugin": "^11.0.0", "cross-env": "^2.0.1", "css-loader": "^6.7.1", "dotenv": "^14.3.2", - "eslint": "^8.16.0", - "eslint-config-standard-with-typescript": "^21.0.1", + "eslint": "^8.19.0", + "eslint-config-standard-with-typescript": "^22.0.0", "eslint-plugin-import": "^2.26.0", - "eslint-plugin-promise": "^5.2.0", - "eslint-plugin-react": "^7.30.0", + "eslint-plugin-promise": "^6.0.0", + "eslint-plugin-react": "^7.30.1", "glob": "^7.2.0", "jest": "^27.5.1", "jest-chrome": "^0.7.2", @@ -44,20 +44,20 @@ "mini-css-extract-plugin": "^2.6.1", "sass-loader": "^12.6.0", "style-loader": "^3.3.1", - "ts-loader": "^9.3.0", - "ts-node": "^10.8.0", - "typescript": "^4.7.2", + "ts-loader": "^9.3.1", + "ts-node": "^10.8.2", + "typescript": "^4.7.4", "uglifyjs-webpack-plugin": "^2.2.0", "webpack": "^5.73.0", "webpack-combine-loaders": "^2.0.4", - "webpack-dev-server": "^4.9.2", + "webpack-dev-server": "^4.9.3", "webpack-notifier": "^1.15.0" }, "dependencies": { "@material-ui/core": "^4.12.4", "@material-ui/icons": "^4.11.3", "@material-ui/lab": "^4.0.0-alpha.61", - "bs58": "^3.1.0", + "bs58": "^4.0.1", "classnames": "^2.3.1", "fp-ts": "^2.11.9", "html2canvas": "^1.4.1", diff --git a/platforms/yttrex/extension/src/app/app.ts b/platforms/yttrex/extension/src/app/app.ts index 5fc28bbc6..2d78c8691 100644 --- a/platforms/yttrex/extension/src/app/app.ts +++ b/platforms/yttrex/extension/src/app/app.ts @@ -33,6 +33,7 @@ import { } from '@shared/extension/app'; import config from '@shared/extension/config'; import logger from '@shared/extension/logger'; +import UserSettings from '@shared/extension/models/UserSettings'; import { sizeCheck } from '@shared/providers/dataDonation.provider'; import { consideredURLs, @@ -162,7 +163,8 @@ export function onLocationChange( export const handleLeaf = ( node: HTMLElement, opts: Omit, - selectorName: string + selectorName: string, + config: UserSettings ): void => { // command has .selector .parents .preserveInvisible (this might be undefined) ytLogger.info('Handle "leaf" type: %s', selectorName); @@ -256,11 +258,12 @@ export const handleLeaf = ( } }; -export function handleRoute( +export const handleRoute = ( node: HTMLElement, selector: RouteObserverHandler, - route: string -): void { + route: string, + s: UserSettings +): void => { ytLogger.info(`Handle route ${route}`, selector); const sendableNode = document.querySelector('ytd-app'); @@ -285,7 +288,7 @@ export function handleRoute( }, }); updateUI('video.send'); -} +}; export const watchedPaths = { home: { diff --git a/platforms/yttrex/extension/src/app/index.ts b/platforms/yttrex/extension/src/app/index.ts index 93034494d..2aeb11d68 100644 --- a/platforms/yttrex/extension/src/app/index.ts +++ b/platforms/yttrex/extension/src/app/index.ts @@ -17,17 +17,13 @@ bo.runtime.sendMessage({ type: 'chromeConfig' }, (config) => { newProfile: settings.isNew, href: window.location.href, execount: settings.execount ?? 0, - testTime: new Date().toISOString(), } as any, mapLocalConfig: (c, { href, ...p }) => { return { - experimentId: '', - evidencetag: '', - directiveType: 'comparison', ...c, ...p, href, - } ; + }; }, observe: { handlers: watchedPaths as any, diff --git a/platforms/yttrex/extension/src/chrome/background/api.ts b/platforms/yttrex/extension/src/chrome/background/api.ts index ccb9b822a..826898433 100644 --- a/platforms/yttrex/extension/src/chrome/background/api.ts +++ b/platforms/yttrex/extension/src/chrome/background/api.ts @@ -1,10 +1,13 @@ -import * as endpoints from '@yttrex/shared/endpoints'; import { SyncReq } from '@shared/extension/chrome/background/sync'; import db from '@shared/extension/chrome/db'; import config from '@shared/extension/config'; import { MakeAPIClient } from '@shared/providers/api.provider'; -import { decodeFromBase58, decodeString } from '@shared/utils/decode.utils'; -import bs58 from 'bs58'; +import { + decodeFromBase58, + decodeString, + encodeToBase58, +} from '@shared/utils/decode.utils'; +import * as endpoints from '@yttrex/shared/endpoints'; import nacl from 'tweetnacl'; import ytLog from '../../logger'; @@ -26,8 +29,9 @@ export const getHeadersForDataDonation = async (req: SyncReq): Promise => { decodeString(JSON.stringify(payload)), decodeFromBase58(keypair.secretKey) ); + const sign = encodeToBase58(signature); - // ytLog.info('Signature %s', signature); + ytLog.info('Signature %O (%s)', signature, sign); const headers = { 'Content-Type': 'application/json', @@ -35,7 +39,7 @@ export const getHeadersForDataDonation = async (req: SyncReq): Promise => { 'X-YTtrex-Build': config.BUILD, 'X-YTtrex-NonAuthCookieId': cookieId, 'X-YTtrex-PublicKey': keypair.publicKey, - 'X-YTtrex-Signature': bs58.encode(signature), + 'X-YTtrex-Signature': sign, }; return headers; diff --git a/platforms/yttrex/extension/src/handlers/events.ts b/platforms/yttrex/extension/src/handlers/events.ts index 38555840b..abad761ad 100644 --- a/platforms/yttrex/extension/src/handlers/events.ts +++ b/platforms/yttrex/extension/src/handlers/events.ts @@ -1,6 +1,7 @@ import { Hub } from '@shared/extension/hub'; import _ from 'lodash'; import config from '@shared/extension/config'; +import { UserSettings } from '@shared/extension/models/UserSettings'; import { getTimeISO8601 } from '@shared/extension/utils/common.utils'; import { bo } from '@shared/extension/utils/browser.utils'; import { NewLeafEvent, NewVideoEvent, YTHubEvent } from '../models/HubEvent'; @@ -38,7 +39,7 @@ export function handleLeaf(e: NewLeafEvent): void { state.incremental++; } -export function sync(hub: Hub): void { +export function sync(hub: Hub, config: UserSettings): void { if (state.content.length) { const uuids = _.size(_.uniq(_.map(state.content, 'randomUUID'))); ytLog.debug( @@ -46,10 +47,16 @@ export function sync(hub: Hub): void { _.countBy(state.content, 'type') )} with ${uuids} randomUUID(s)` ); + + const payload = state.content.map((c) => ({ + ...c, + researchTag: config.researchTag, + experimentId: config.experimentId, + })); // Send timelines to the page handling the communication with the API. // This might be refactored using something compatible to the HUB architecture. bo.runtime.sendMessage( - { type: 'sync', payload: state.content, userId: 'local' }, + { type: 'sync', payload, userId: 'local' }, (response: any) => { ytLog.info('Sync response %j', response); hub.dispatch({ type: 'SyncResponse', payload: response }); @@ -61,15 +68,15 @@ export function sync(hub: Hub): void { } } -export function register(hub: Hub, config: any): void { +export function register(hub: Hub, config: UserSettings): void { if (config.active) { hub.on('NewVideo', handleVideo); hub.on('leaf', handleLeaf); - hub.on('WindowUnload', () => sync(hub)); + hub.on('WindowUnload', () => sync(hub, config)); window.setInterval(() => { // ytLog.debug('Sync at interval %s', INTERVAL); - sync(hub); + sync(hub, config); }, INTERVAL); } } diff --git a/platforms/yttrex/shared/package.json b/platforms/yttrex/shared/package.json index 361310504..cbadef34f 100644 --- a/platforms/yttrex/shared/package.json +++ b/platforms/yttrex/shared/package.json @@ -24,22 +24,22 @@ "ts-io-error": "^2.0.0" }, "devDependencies": { - "@types/jest": "^27.5.1", + "@types/jest": "^27.5.2", "@types/node": "^16.11.36", "@types/uuid": "^8.3.4", - "@typescript-eslint/eslint-plugin": "^5.27.0", - "@typescript-eslint/parser": "^5.27.0", - "eslint": "^8.16.0", + "@typescript-eslint/eslint-plugin": "^5.30.5", + "@typescript-eslint/parser": "^5.30.5", + "eslint": "^8.19.0", "eslint-config-prettier": "^8.5.0", - "eslint-config-standard-with-typescript": "^21.0.1", + "eslint-config-standard-with-typescript": "^22.0.0", "eslint-plugin-import": "^2.26.0", "eslint-plugin-node": "^11.1.0", - "eslint-plugin-promise": "^5.2.0", + "eslint-plugin-promise": "^6.0.0", "jest": "^27.5.1", - "prettier": "^2.6.2", + "prettier": "^2.7.1", "ts-jest": "^27.1.5", - "ts-node": "^10.8.0", + "ts-node": "^10.8.2", "tsconfig-paths": "^3.14.1", - "typescript": "^4.7.2" + "typescript": "^4.7.4" } } diff --git a/platforms/yttrex/shared/src/endpoints/v0/index.ts b/platforms/yttrex/shared/src/endpoints/v0/index.ts index 7e91fb5e6..bc2994e00 100644 --- a/platforms/yttrex/shared/src/endpoints/v0/index.ts +++ b/platforms/yttrex/shared/src/endpoints/v0/index.ts @@ -1,5 +1,5 @@ -import * as t from 'io-ts'; import { DocumentedEndpoint } from '@shared/endpoints/DocumentedEndpoint'; +import { HealthResponse } from '../../models/Health'; const GetHealth = DocumentedEndpoint({ title: 'GET health', @@ -7,7 +7,7 @@ const GetHealth = DocumentedEndpoint({ tags: ['health'], Method: 'GET', getPath: () => `/v0/health`, - Output: t.strict({ data: t.literal('OK') }), + Output: HealthResponse, }); export default { diff --git a/platforms/yttrex/shared/src/endpoints/v2/index.ts b/platforms/yttrex/shared/src/endpoints/v2/index.ts index e54298bed..7953f9598 100644 --- a/platforms/yttrex/shared/src/endpoints/v2/index.ts +++ b/platforms/yttrex/shared/src/endpoints/v2/index.ts @@ -1,185 +1,9 @@ -import * as t from 'io-ts'; -import { Endpoint } from 'ts-endpoint'; -import { AddEventsBody } from '@shared/models/ContributionEvent'; -import { ContributorPublicKeyResponse } from '@shared/models/contributor/ContributorPublicKey'; -import { GetExperimentListOutput } from '@shared/models/Experiment'; -import { HandshakeBody, HandshakeResponse } from '@shared/models/HandshakeBody'; -import { PublicKeyParams } from '@shared/models/http/params/PublicKey'; -import { SearchQuery } from '@shared/models/http/SearchQuery'; -import { Metadata } from '@shared/models/Metadata'; -import { ChannelADVStats } from '@shared/models/stats/ChannelADV'; -import { DocumentedEndpoint } from '@shared/endpoints'; +import Metadata from './metadata.endpoints'; +import Public from './public.endpoints'; -export const Handshake = Endpoint({ - Method: 'POST', - getPath: () => `/v2/handshake`, - Input: { - Body: HandshakeBody, - }, - Output: HandshakeResponse, -}); - -const CompareVideo = Endpoint({ - Method: 'GET', - getPath: ({ videoId }) => `/v2/compare/${videoId}`, - Input: { - Params: t.type({ videoId: t.string }), - }, - Output: t.any, -}); - -const VideoRelated = Endpoint({ - Method: 'GET', - getPath: ({ videoId }) => `/v2/related/${videoId}`, - Input: { - Params: t.type({ videoId: t.string }), - }, - Output: t.any, -}); - -const VideoAuthor = Endpoint({ - Method: 'GET', - getPath: ({ videoId }) => `/v2/author/${videoId}`, - Input: { - Params: t.type({ videoId: t.string }), - }, - Output: t.any, -}); - -const Searches = DocumentedEndpoint({ - title: 'Search by type', - description: 'Search description', - tags: ['searches'], - Method: 'GET', - getPath: ({ queryString }) => `/v2/searches/${queryString}`, - Input: { - Params: t.type({ queryString: t.string }), - }, - Output: t.any, -}); - -const SearchesAsCSV = Endpoint({ - Method: 'GET', - getPath: ({ queryString }) => `/v2/searches/${queryString}/csv`, - Input: { - Params: t.type({ queryString: t.string }), - }, - Output: t.any, -}); - -const GetPersonalCSV = Endpoint({ - Method: 'GET', - getPath: ({ publicKey, type }) => `/v2/personal/${publicKey}/${type}/csv`, - Input: { - Params: t.type({ - ...PublicKeyParams.props, - type: t.union([ - t.literal('home'), - t.literal('video'), - t.literal('search'), - ]), - }), - }, - Output: t.any, -}); - -const AddEvents = Endpoint({ - Method: 'POST', - getPath: () => `/v2/events`, - Input: { - Headers: t.type({ - 'X-YTtrex-Version': t.string, - 'X-YTtrex-Build': t.string, - 'X-YTtrex-PublicKey': t.string, - 'X-YTtrex-Signature': t.string, - }), - Body: AddEventsBody, - }, - Output: t.any, -}); - -const AddAPIEvents = Endpoint({ - Method: 'POST', - getPath: () => `/v2/apiEvents`, - Input: { - Headers: t.type({ - 'X-TrEx-Version': t.string, - 'X-TrEx-Build': t.string, - 'X-TrEx-PublicKey': t.string, - 'X-TrEx-Signature': t.string, - }), - Body: AddEventsBody, - }, - Output: t.any, -}); - -const GetChannelADVStats = Endpoint({ - Method: 'GET', - getPath: ({ channelId }) => `/v2/ad/channel/${channelId}`, - Input: { - Query: t.type({ - since: t.string, - till: t.string, - }), - Params: t.type({ - channelId: t.string, - }), - }, - Output: t.array(ChannelADVStats), -}); - -const GetExperimentList = Endpoint({ - Method: 'GET', - getPath: ({ type, publicKey }) => `/v2/guardoni/list/${type}/${publicKey}`, - Input: { - Query: SearchQuery, - Params: t.type({ - type: t.union([t.literal('comparison'), t.literal('chiaroscuro')]), - ...PublicKeyParams.props, - }), - }, - Output: GetExperimentListOutput, -}); - -const GetExperimentById = Endpoint({ - Method: 'GET', - getPath: ({ experimentId }) => `/v2/experiment/${experimentId}/json`, - Input: { - Query: SearchQuery, - Params: t.type({ - experimentId: t.string, - }), - }, - Output: t.array(Metadata), -}); - -const DeletePersonalContributionByPublicKey = Endpoint({ - Method: 'DELETE', - getPath: ({ publicKey, selector }) => - `/v2/personal/${publicKey}/selector/id/${selector}`, - Input: { - Params: t.type({ - ...PublicKeyParams.props, - selector: t.union([t.string, t.undefined]), - }), - }, - Output: ContributorPublicKeyResponse, -}); - -export default { - Public: { - Handshake, - CompareVideo, - VideoRelated, - VideoAuthor, - Searches, - SearchesAsCSV, - AddEvents, - AddAPIEvents, - GetChannelADVStats, - GetExperimentList, - GetExperimentById, - GetPersonalCSV, - DeletePersonalContributionByPublicKey, - }, +const endpoints = { + Public, + Metadata, }; + +export default endpoints; diff --git a/platforms/yttrex/shared/src/endpoints/v2/metadata.endpoints.ts b/platforms/yttrex/shared/src/endpoints/v2/metadata.endpoints.ts new file mode 100644 index 000000000..1b67fcca8 --- /dev/null +++ b/platforms/yttrex/shared/src/endpoints/v2/metadata.endpoints.ts @@ -0,0 +1,24 @@ +import { Format } from '@shared/models/common'; +import * as t from 'io-ts'; +import { NumberFromString } from 'io-ts-types'; +import { Endpoint } from 'ts-endpoint'; +import { MetadataList } from '../../models/Metadata'; + +const ListMetadata = Endpoint({ + Method: 'GET', + getPath: () => `/v2/metadata`, + Input: { + Query: t.type({ + experimentId: t.union([t.string, t.undefined]), + researchTag: t.union([t.string, t.undefined]), + amount: t.union([NumberFromString, t.undefined]), + skip: t.union([NumberFromString, t.undefined]), + format: t.union([Format, t.undefined]), + }), + }, + Output: MetadataList, +}); + +export default { + ListMetadata, +}; diff --git a/platforms/yttrex/shared/src/endpoints/v2/public.endpoints.ts b/platforms/yttrex/shared/src/endpoints/v2/public.endpoints.ts new file mode 100644 index 000000000..0fb66556e --- /dev/null +++ b/platforms/yttrex/shared/src/endpoints/v2/public.endpoints.ts @@ -0,0 +1,185 @@ +import * as t from 'io-ts'; +import { Endpoint } from 'ts-endpoint'; +import { AddEventsBody } from '@shared/models/ContributionEvent'; +import { ContributorPublicKeyResponse } from '@shared/models/contributor/ContributorPublicKey'; +import { GetExperimentListOutput } from '@shared/models/Experiment'; +import { HandshakeBody, HandshakeResponse } from '@shared/models/HandshakeBody'; +import { PublicKeyParams } from '@shared/models/http/params/PublicKey'; +import { SearchQuery } from '@shared/models/http/SearchQuery'; +import { Metadata } from '@shared/models/Metadata'; +import { ChannelADVStats } from '@shared/models/stats/ChannelADV'; +import { DocumentedEndpoint } from '@shared/endpoints'; + +export const Handshake = Endpoint({ + Method: 'POST', + getPath: () => `/v2/handshake`, + Input: { + Body: HandshakeBody, + }, + Output: HandshakeResponse, +}); + +const CompareVideo = Endpoint({ + Method: 'GET', + getPath: ({ videoId }) => `/v2/compare/${videoId}`, + Input: { + Params: t.type({ videoId: t.string }), + }, + Output: t.any, +}); + +const VideoRelated = Endpoint({ + Method: 'GET', + getPath: ({ videoId }) => `/v2/related/${videoId}`, + Input: { + Params: t.type({ videoId: t.string }), + }, + Output: t.any, +}); + +const VideoAuthor = Endpoint({ + Method: 'GET', + getPath: ({ videoId }) => `/v2/author/${videoId}`, + Input: { + Params: t.type({ videoId: t.string }), + }, + Output: t.any, +}); + +const Searches = DocumentedEndpoint({ + title: 'Search by type', + description: 'Search description', + tags: ['searches'], + Method: 'GET', + getPath: ({ queryString }) => `/v2/searches/${queryString}`, + Input: { + Params: t.type({ queryString: t.string }), + }, + Output: t.any, +}); + +const SearchesAsCSV = Endpoint({ + Method: 'GET', + getPath: ({ queryString }) => `/v2/searches/${queryString}/csv`, + Input: { + Params: t.type({ queryString: t.string }), + }, + Output: t.any, +}); + +const GetPersonalCSV = Endpoint({ + Method: 'GET', + getPath: ({ publicKey, type }) => `/v2/personal/${publicKey}/${type}/csv`, + Input: { + Params: t.type({ + ...PublicKeyParams.props, + type: t.union([ + t.literal('home'), + t.literal('video'), + t.literal('search'), + ]), + }), + }, + Output: t.any, +}); + +const AddEvents = Endpoint({ + Method: 'POST', + getPath: () => `/v2/events`, + Input: { + Headers: t.type({ + 'X-YTtrex-Version': t.string, + 'X-YTtrex-Build': t.string, + 'X-YTtrex-PublicKey': t.string, + 'X-YTtrex-Signature': t.string, + }), + Body: AddEventsBody, + }, + Output: t.any, +}); + +const AddAPIEvents = Endpoint({ + Method: 'POST', + getPath: () => `/v2/apiEvents`, + Input: { + Headers: t.type({ + 'X-TrEx-Version': t.string, + 'X-TrEx-Build': t.string, + 'X-TrEx-PublicKey': t.string, + 'X-TrEx-Signature': t.string, + }), + Body: AddEventsBody, + }, + Output: t.any, +}); + +const GetChannelADVStats = Endpoint({ + Method: 'GET', + getPath: ({ channelId }) => `/v2/ad/channel/${channelId}`, + Input: { + Query: t.type({ + since: t.string, + till: t.string, + }), + Params: t.type({ + channelId: t.string, + }), + }, + Output: t.array(ChannelADVStats), +}); + +const GetExperimentList = Endpoint({ + Method: 'GET', + getPath: ({ type, publicKey }) => `/v2/guardoni/list/${type}/${publicKey}`, + Input: { + Query: SearchQuery, + Params: t.type({ + type: t.union([t.literal('comparison'), t.literal('chiaroscuro')]), + ...PublicKeyParams.props, + }), + }, + Output: GetExperimentListOutput, +}); + +const GetExperimentById = Endpoint({ + Method: 'GET', + getPath: ({ experimentId }) => `/v2/experiment/${experimentId}/json`, + Input: { + Query: SearchQuery, + Params: t.type({ + experimentId: t.string, + }), + }, + Output: t.array(Metadata), +}); + +const DeletePersonalContributionByPublicKey = Endpoint({ + Method: 'DELETE', + getPath: ({ publicKey, selector }) => + `/v2/personal/${publicKey}/selector/id/${selector}`, + Input: { + Params: t.type({ + ...PublicKeyParams.props, + selector: t.union([t.string, t.undefined]), + }), + }, + Output: ContributorPublicKeyResponse, +}); + +const Public = { + Handshake, + CompareVideo, + VideoRelated, + VideoAuthor, + Searches, + SearchesAsCSV, + AddEvents, + AddAPIEvents, + GetChannelADVStats, + GetExperimentList, + GetExperimentById, + GetPersonalCSV, + DeletePersonalContributionByPublicKey, +}; + +export default Public; diff --git a/platforms/yttrex/shared/src/endpoints/v3/public.endpoints.ts b/platforms/yttrex/shared/src/endpoints/v3/public.endpoints.ts index cabc32bdb..96da0c365 100644 --- a/platforms/yttrex/shared/src/endpoints/v3/public.endpoints.ts +++ b/platforms/yttrex/shared/src/endpoints/v3/public.endpoints.ts @@ -1,21 +1,18 @@ -import * as t from 'io-ts'; -import { Endpoint } from 'ts-endpoint'; import { CreateDirectiveBody, Directive, DirectiveType, - PostDirectiveResponse, + PostDirectiveResponse } from '@shared/models/Directive'; -import { - ConcludeGuardoniExperimentOutput, - GetPublicDirectivesOutput, -} from '@shared/models/Experiment'; +import { GetPublicDirectivesOutput } from '@shared/models/Experiment'; import { HandshakeBody } from '@shared/models/HandshakeBody'; import { GetRecommendationsParams, GetRecommendationsQuery, - RecommendationList, + RecommendationList } from '@shared/models/Recommendation'; +import * as t from 'io-ts'; +import { Endpoint } from 'ts-endpoint'; const Handshake = Endpoint({ Method: 'POST', @@ -82,21 +79,11 @@ const GetPublicDirectives = Endpoint({ Output: GetPublicDirectivesOutput, }); -const ConcludeExperiment = Endpoint({ - Method: 'DELETE', - getPath: ({ testTime }) => `/v3/experiment/${testTime}`, - Input: { - Params: t.type({ testTime: t.string }), - }, - Output: ConcludeGuardoniExperimentOutput, -}); - export const endpoints = { Handshake, GetRecommendations, VideoRecommendations, GetDirective, PostDirective, - ConcludeExperiment, GetPublicDirectives, }; diff --git a/platforms/yttrex/shared/src/models/Health.ts b/platforms/yttrex/shared/src/models/Health.ts new file mode 100644 index 000000000..4229e25dc --- /dev/null +++ b/platforms/yttrex/shared/src/models/Health.ts @@ -0,0 +1,8 @@ +import * as t from 'io-ts'; + +export const HealthResponse = t.strict( + { data: t.literal('OK') }, + 'GetHealthResponse' +); + +export type HealthResponse = t.TypeOf; diff --git a/platforms/yttrex/shared/src/models/Metadata.ts b/platforms/yttrex/shared/src/models/Metadata.ts index 1476abf40..dbcf2d0c6 100644 --- a/platforms/yttrex/shared/src/models/Metadata.ts +++ b/platforms/yttrex/shared/src/models/Metadata.ts @@ -120,7 +120,10 @@ export type SearchMetadata = t.TypeOf; export const Metadata = t.union( [VideoMetadata, HomeMetadata, SearchMetadata], - 'MetadataDB' + 'Metadata' ); export type Metadata = t.TypeOf; + +export const MetadataList = t.array(Metadata, 'Metadata[]'); +export type MetadataList = t.TypeOf; diff --git a/yarn.lock b/yarn.lock index b816d9795..909dbfe1f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3083,34 +3083,6 @@ __metadata: languageName: node linkType: hard -"@makhno/backend@workspace:platforms/makhno/backend": - version: 0.0.0-use.local - resolution: "@makhno/backend@workspace:platforms/makhno/backend" - dependencies: - "@types/express": ^4.17.13 - "@types/lodash": ^4.14.182 - "@types/node": ^16.11.36 - body-parser: ^1.19.1 - bs58: ^3.1.0 - cookie: ^0.3.1 - cors: ^2.8.5 - debug: ^4.3.4 - express: ^4.17.2 - food-words: ^1.1.0 - jquery: ^3.6.0 - lodash: ^4.17.21 - moment: ^2.29.2 - mongodb: ^4.3.1 - nacl-signature: ^1.0.0 - nconf: ^0.8.5 - node-fetch: ^2.6.7 - ts-node: ^10.8.0 - ts-node-dev: ^2.0.0 - tweetnacl: ^0.14.5 - typescript: ^4.7.2 - languageName: unknown - linkType: soft - "@malept/cross-spawn-promise@npm:^1.1.0": version: 1.1.1 resolution: "@malept/cross-spawn-promise@npm:1.1.1" @@ -3403,16 +3375,6 @@ __metadata: languageName: node linkType: hard -"@mrmlnc/readdir-enhanced@npm:^2.2.1": - version: 2.2.1 - resolution: "@mrmlnc/readdir-enhanced@npm:2.2.1" - dependencies: - call-me-maybe: ^1.0.1 - glob-to-regexp: ^0.3.0 - checksum: d3b82b29368821154ce8e10bef5ccdbfd070d3e9601643c99ea4607e56f3daeaa4e755dd6d2355da20762c695c1b0570543d9f84b48f70c211ec09c4aaada2e1 - languageName: node - linkType: hard - "@mui/base@npm:5.0.0-alpha.87": version: 5.0.0-alpha.87 resolution: "@mui/base@npm:5.0.0-alpha.87" @@ -3611,13 +3573,6 @@ __metadata: languageName: node linkType: hard -"@nodelib/fs.stat@npm:^1.1.2": - version: 1.1.3 - resolution: "@nodelib/fs.stat@npm:1.1.3" - checksum: 318deab369b518a34778cdaa0054dd28a4381c0c78e40bbd20252f67d084b1d7bf9295fea4423de2c19ac8e1a34f120add9125f481b2a710f7068bcac7e3e305 - languageName: node - linkType: hard - "@nodelib/fs.walk@npm:^1.2.3": version: 1.2.8 resolution: "@nodelib/fs.walk@npm:1.2.8" @@ -4294,7 +4249,7 @@ __metadata: languageName: node linkType: hard -"@testing-library/react@npm:^13.2.0": +"@testing-library/react@npm:^13.3.0": version: 13.3.0 resolution: "@testing-library/react@npm:13.3.0" dependencies: @@ -4308,7 +4263,7 @@ __metadata: languageName: node linkType: hard -"@testing-library/user-event@npm:^14.2.0": +"@testing-library/user-event@npm:^14.2.1": version: 14.2.1 resolution: "@testing-library/user-event@npm:14.2.1" peerDependencies: @@ -4325,7 +4280,7 @@ __metadata: "@types/lodash": ^4.14.182 "@types/node": ^16.11.36 body-parser: ^1.19.1 - bs58: ^3.1.0 + bs58: ^4.0.1 cookie: ^0.3.1 cors: ^2.8.5 debug: ^4.3.4 @@ -4336,13 +4291,13 @@ __metadata: moment: ^2.29.2 mongodb: ^4.3.1 nacl-signature: ^1.0.0 - nconf: ^0.8.5 + nconf: ^0.11.3 node-fetch: ^2.6.7 nodemon: ^1.19.4 - ts-node: ^10.8.0 + ts-node: ^10.8.2 ts-node-dev: ^2.0.0 tweetnacl: ^0.14.5 - typescript: ^4.7.2 + typescript: ^4.7.4 languageName: unknown linkType: soft @@ -4359,7 +4314,7 @@ __metadata: prism-react-renderer: ^1.2.1 react: ^17.0.2 react-dom: ^17.0.2 - typescript: ^4.7.2 + typescript: ^4.7.4 languageName: unknown linkType: soft @@ -4374,19 +4329,19 @@ __metadata: "@types/jquery": ^3.5.14 "@types/lodash": ^4.14.182 "@types/node": ^16.11.36 - "@types/react": ^17.0.45 + "@types/react": ^17.0.47 "@types/react-dom": ^17.0.17 - "@typescript-eslint/eslint-plugin": ^5.27.0 - "@typescript-eslint/parser": ^5.27.0 + "@typescript-eslint/eslint-plugin": ^5.30.5 + "@typescript-eslint/parser": ^5.30.5 create-react-class: ^15.7.0 debug: ^4.3.4 - eslint: ^8.16.0 + eslint: ^8.19.0 eslint-config-prettier: ^8.5.0 - eslint-config-standard-with-typescript: ^21.0.1 + eslint-config-standard-with-typescript: ^22.0.0 eslint-plugin-import: ^2.26.0 eslint-plugin-node: ^11.1.0 - eslint-plugin-promise: ^5.2.0 - eslint-plugin-react: ^7.30.0 + eslint-plugin-promise: ^6.0.0 + eslint-plugin-react: ^7.30.1 fp-ts: ^2.11.9 io-ts: ^2.2.16 jest-environment-jsdom-global: ^3.1.2 @@ -4396,7 +4351,7 @@ __metadata: nacl: ^0.1.3 react: ^17.0.2 react-dom: ^17.0.2 - typescript: ^4.7.2 + typescript: ^4.7.4 webpack: ^5.73.0 languageName: unknown linkType: soft @@ -4405,17 +4360,17 @@ __metadata: version: 0.0.0-use.local resolution: "@tktrex/shared@workspace:platforms/tktrex/shared" dependencies: - "@types/jest": ^27.5.1 + "@types/jest": ^27.5.2 "@types/node": ^16.11.36 "@types/yargs": ^17.0.10 - "@typescript-eslint/eslint-plugin": ^5.27.0 - "@typescript-eslint/parser": ^5.27.0 - eslint: ^8.16.0 - eslint-config-standard-with-typescript: ^21.0.1 + "@typescript-eslint/eslint-plugin": ^5.30.5 + "@typescript-eslint/parser": ^5.30.5 + eslint: ^8.19.0 + eslint-config-standard-with-typescript: ^22.0.0 eslint-plugin-import: ^2.26.0 eslint-plugin-node: ^11.1.0 - eslint-plugin-promise: ^5.2.0 - eslint-plugin-react: ^7.30.0 + eslint-plugin-promise: ^6.0.0 + eslint-plugin-react: ^7.30.1 fp-ts: ^2.11.9 io-ts: ^2.2.16 jest: ^27.5.1 @@ -4423,8 +4378,8 @@ __metadata: mongodb: ^4.3.1 ts-endpoint: ^2.0.0 ts-jest: ^27.1.5 - ts-node: ^10.8.0 - typescript: ^4.7.2 + ts-node: ^10.8.2 + typescript: ^4.7.4 yargs: ^17.5.1 languageName: unknown linkType: soft @@ -4457,7 +4412,7 @@ __metadata: prism-react-renderer: ^1.2.1 react: ^17.0.2 react-dom: ^17.0.2 - typescript: ^4.7.2 + typescript: ^4.7.4 languageName: unknown linkType: soft @@ -4473,27 +4428,27 @@ __metadata: "@types/dotenv-webpack": ^7.0.3 "@types/geoip-lite": ^1.4.1 "@types/node": ^16.11.36 - "@types/react": ^17.0.45 + "@types/react": ^17.0.47 "@types/react-dom": ^17.0.17 "@types/webpack-bundle-analyzer": ^4.4.1 - "@typescript-eslint/eslint-plugin": ^5.27.0 - "@typescript-eslint/parser": ^5.27.0 + "@typescript-eslint/eslint-plugin": ^5.30.5 + "@typescript-eslint/parser": ^5.30.5 avenger: ^7.0.1 axios: ^0.24.0 canvas: ^2.9.1 copy-webpack-plugin: ^11.0.0 debug: ^4.3.4 dotenv-webpack: ^7.1.0 - eslint: ^8.16.0 + eslint: ^8.19.0 eslint-config-prettier: ^8.5.0 - eslint-config-standard-with-typescript: ^21.0.1 + eslint-config-standard-with-typescript: ^22.0.0 eslint-plugin-import: ^2.26.0 eslint-plugin-node: ^11.1.0 - eslint-plugin-promise: ^5.2.0 - eslint-plugin-react: ^7.30.0 + eslint-plugin-promise: ^6.0.0 + eslint-plugin-react: ^7.30.1 fast-check: ^2.25.0 fast-check-io-ts: ^0.5.0 - filemanager-webpack-plugin: ^6.1.7 + filemanager-webpack-plugin: ^7.0.0 fp-ts: ^2.11.9 geoip-lite: ^1.4.4 html-webpack-plugin: ^5.5.0 @@ -4503,12 +4458,12 @@ __metadata: mini-css-extract-plugin: ^2.6.1 node-loader: ^2.0.0 node-sass: ^7.0.1 - prettier: ^2.6.2 + prettier: ^2.7.1 react: ^17.0.2 react-dom: ^17.0.2 - react-i18next: ^11.15.3 + react-i18next: ^11.18.0 react-refresh: ^0.12.0 - react-refresh-typescript: ^2.0.5 + react-refresh-typescript: ^2.0.7 sass-loader: ^12.6.0 style-loader: ^3.3.1 ts-endpoint: ^2.0.0 @@ -4517,7 +4472,7 @@ __metadata: tsconfig-paths-webpack-plugin: ^3.5.2 type-fest: ^2.11.1 typelevel-ts: ^0.4.0 - typescript: ^4.7.2 + typescript: ^4.7.4 webpack: ^5.73.0 webpack-bundle-analyzer: ^4.5.0 webpack-cli: ^4.10.0 @@ -4540,37 +4495,37 @@ __metadata: "@mui/x-data-grid": ^5.11.1 "@types/node": ^16.11.36 "@types/prettier": ^2.6.3 - "@types/react": ^17.0.45 + "@types/react": ^17.0.47 "@types/react-dom": ^17.0.17 "@types/webpack-bundle-analyzer": ^4.4.1 - "@typescript-eslint/eslint-plugin": ^5.27.0 - "@typescript-eslint/parser": ^5.27.0 + "@typescript-eslint/eslint-plugin": ^5.30.5 + "@typescript-eslint/parser": ^5.30.5 avenger: ^7.0.1 axios: ^0.24.0 date-fns: ^2.28.0 debug: ^4.3.4 - eslint: ^8.16.0 + eslint: ^8.19.0 eslint-config-prettier: ^8.5.0 - eslint-config-standard-with-typescript: ^21.0.1 + eslint-config-standard-with-typescript: ^22.0.0 eslint-plugin-import: ^2.26.0 eslint-plugin-node: ^11.1.0 - eslint-plugin-promise: ^5.2.0 - eslint-plugin-react: ^7.30.0 + eslint-plugin-promise: ^6.0.0 + eslint-plugin-react: ^7.30.1 fp-ts: ^2.11.9 io-ts: ^2.2.16 io-ts-types: ^0.5.16 - prettier: ^2.6.2 + prettier: ^2.7.1 react: ^17.0.2 react-dom: ^17.0.2 ts-endpoint: ^2.0.0 ts-io-error: ^2.0.0 - ts-loader: ^9.3.0 + ts-loader: ^9.3.1 typelevel-ts: ^0.4.0 - typescript: ^4.7.2 + typescript: ^4.7.4 webpack: ^5.73.0 webpack-bundle-analyzer: ^4.5.0 webpack-cli: ^4.10.0 - webpack-dev-server: ^4.9.2 + webpack-dev-server: ^4.9.3 languageName: unknown linkType: soft @@ -4582,21 +4537,21 @@ __metadata: "@material-ui/icons": ^4.11.3 "@material-ui/lab": ^4.0.0-alpha.61 "@testing-library/jest-dom": ^5.16.4 - "@testing-library/react": ^13.2.0 - "@testing-library/user-event": ^14.2.0 + "@testing-library/react": ^13.3.0 + "@testing-library/user-event": ^14.2.1 "@types/bs58": ^4.0.1 "@types/c3": ^0.7.8 "@types/chrome": ^0.0.188 "@types/debug": ^4.1.7 - "@types/jest": ^27.5.1 + "@types/jest": ^27.5.2 "@types/lodash": ^4.14.182 "@types/node": ^16.11.36 - "@types/react": ^17.0.45 + "@types/react": ^17.0.47 "@types/react-dom": ^17.0.17 "@types/react-test-renderer": ^17.0.2 "@types/swagger-ui": ^3.52.0 - "@typescript-eslint/eslint-plugin": ^5.27.0 - "@typescript-eslint/parser": ^5.27.0 + "@typescript-eslint/eslint-plugin": ^5.30.5 + "@typescript-eslint/parser": ^5.30.5 "@vx/group": 0.0.199 "@vx/hierarchy": 0.0.199 "@vx/mock-data": 0.0.199 @@ -4615,16 +4570,16 @@ __metadata: date-fns: ^2.28.0 debug: ^4.3.4 dotenv: ^14.3.2 - eslint: ^8.16.0 + eslint: ^8.19.0 eslint-config-prettier: ^8.5.0 - eslint-config-standard-with-typescript: ^21.0.1 + eslint-config-standard-with-typescript: ^22.0.0 eslint-plugin-import: ^2.26.0 eslint-plugin-node: ^11.1.0 - eslint-plugin-promise: ^5.2.0 - eslint-plugin-react: ^7.30.0 + eslint-plugin-promise: ^6.0.0 + eslint-plugin-react: ^7.30.1 fp-ts: ^2.11.9 husky: ^7.0.4 - i18next: ^21.6.10 + i18next: ^21.8.13 io-ts: ^2.2.16 io-ts-types: ^0.5.16 jest: ^27.5.1 @@ -4634,30 +4589,30 @@ __metadata: nacl: ^0.1.3 newtype-ts: ^0.3.5 postcss: ^8.4.14 - prettier: ^2.6.2 + prettier: ^2.7.1 react: ^17.0.2 react-dnd: ^14.0.5 react-dnd-html5-backend: ^14.1.0 react-dom: ^17.0.2 - react-i18next: ^11.15.3 + react-i18next: ^11.18.0 react-test-renderer: ^17.0.2 - react-use-clipboard: ^1.0.7 + react-use-clipboard: ^1.0.8 style-loader: ^3.3.1 svgdom: ^0.1.10 swagger-ui: ^4.4.1 ts-endpoint: ^2.0.0 ts-jest: ^27.1.5 - ts-loader: ^9.3.0 - ts-node: ^10.8.0 + ts-loader: ^9.3.1 + ts-node: ^10.8.2 tweetnacl: ^1.0.3 tweetnacl-util: ^0.13.5 - typescript: ^4.7.2 + typescript: ^4.7.4 uuid: ^2.0.3 web-vitals: ^2.1.4 webpack: ^5.73.0 webpack-bundle-analyzer: ^4.5.0 webpack-cli: ^4.10.0 - webpack-dev-server: ^4.9.2 + webpack-dev-server: ^4.9.3 zenroom: ^2.2.3 languageName: unknown linkType: soft @@ -4704,7 +4659,7 @@ __metadata: languageName: node linkType: hard -"@types/archiver@npm:^5.1.1": +"@types/archiver@npm:^5.3.1": version: 5.3.1 resolution: "@types/archiver@npm:5.3.1" dependencies: @@ -5375,7 +5330,7 @@ __metadata: languageName: node linkType: hard -"@types/jest@npm:^27.5.1": +"@types/jest@npm:^27.5.2": version: 27.5.2 resolution: "@types/jest@npm:27.5.2" dependencies: @@ -5976,13 +5931,13 @@ __metadata: languageName: node linkType: hard -"@typescript-eslint/eslint-plugin@npm:^5.27.0": - version: 5.30.0 - resolution: "@typescript-eslint/eslint-plugin@npm:5.30.0" +"@typescript-eslint/eslint-plugin@npm:^5.30.5": + version: 5.30.5 + resolution: "@typescript-eslint/eslint-plugin@npm:5.30.5" dependencies: - "@typescript-eslint/scope-manager": 5.30.0 - "@typescript-eslint/type-utils": 5.30.0 - "@typescript-eslint/utils": 5.30.0 + "@typescript-eslint/scope-manager": 5.30.5 + "@typescript-eslint/type-utils": 5.30.5 + "@typescript-eslint/utils": 5.30.5 debug: ^4.3.4 functional-red-black-tree: ^1.0.1 ignore: ^5.2.0 @@ -5995,69 +5950,42 @@ __metadata: peerDependenciesMeta: typescript: optional: true - checksum: f2fe96082c69f2871439303947d6dfcd8c142e8ddcc3d1ea52cc30046f4b64280a02aec1a0d2f8344034bc7ed4fb09eecf4764df93ece467cc284225d8adaa14 - languageName: node - linkType: hard - -"@typescript-eslint/parser@npm:^4.0.0": - version: 4.33.0 - resolution: "@typescript-eslint/parser@npm:4.33.0" - dependencies: - "@typescript-eslint/scope-manager": 4.33.0 - "@typescript-eslint/types": 4.33.0 - "@typescript-eslint/typescript-estree": 4.33.0 - debug: ^4.3.1 - peerDependencies: - eslint: ^5.0.0 || ^6.0.0 || ^7.0.0 - peerDependenciesMeta: - typescript: - optional: true - checksum: 102457eae1acd516211098fea081c8a2ed728522bbda7f5a557b6ef23d88970514f9a0f6285d53fca134d3d4d7d17822b5d5e12438d5918df4d1f89cc9e67d57 + checksum: cf763fb091dcdfd6c25843251a220b654ca83968b17266e0f343771f489085c6afc4e41fcf2187b4c72c4d12a787070c64b5e5367069460f95a8174573f48905 languageName: node linkType: hard -"@typescript-eslint/parser@npm:^5.27.0": - version: 5.30.0 - resolution: "@typescript-eslint/parser@npm:5.30.0" +"@typescript-eslint/parser@npm:^5.0.0, @typescript-eslint/parser@npm:^5.30.5": + version: 5.30.5 + resolution: "@typescript-eslint/parser@npm:5.30.5" dependencies: - "@typescript-eslint/scope-manager": 5.30.0 - "@typescript-eslint/types": 5.30.0 - "@typescript-eslint/typescript-estree": 5.30.0 + "@typescript-eslint/scope-manager": 5.30.5 + "@typescript-eslint/types": 5.30.5 + "@typescript-eslint/typescript-estree": 5.30.5 debug: ^4.3.4 peerDependencies: eslint: ^6.0.0 || ^7.0.0 || ^8.0.0 peerDependenciesMeta: typescript: optional: true - checksum: 7067ba4edea702137de3d5c866f2e4a22032bfa82556351eb8cc1088ef45cdd747cbf2a73d3904583303899de2983ef41050f3a5b5d648a2f88e687318bb6738 + checksum: 6c16821e122b891420a538f200f6e576ad1167855a67e87f9a7d3a08c0513fe26006f6411b8ba6f4662a81526bd0339ae37c47dd88fa5943e6f27ff70da9f989 languageName: node linkType: hard -"@typescript-eslint/scope-manager@npm:4.33.0": - version: 4.33.0 - resolution: "@typescript-eslint/scope-manager@npm:4.33.0" +"@typescript-eslint/scope-manager@npm:5.30.5": + version: 5.30.5 + resolution: "@typescript-eslint/scope-manager@npm:5.30.5" dependencies: - "@typescript-eslint/types": 4.33.0 - "@typescript-eslint/visitor-keys": 4.33.0 - checksum: 9a25fb7ba7c725ea7227a24d315b0f6aacbad002e2549a049edf723c1d3615c22f5c301f0d7d615b377f2cdf2f3519d97e79af0c459de6ef8d2aaf0906dff13e + "@typescript-eslint/types": 5.30.5 + "@typescript-eslint/visitor-keys": 5.30.5 + checksum: 509bee6d62cca1716e8f4792d9180c189974992ba13d8103ca04423a64006cf184c4b2c606d55c776305458140c798a3a9a414d07a60790b83dd714f56c457b0 languageName: node linkType: hard -"@typescript-eslint/scope-manager@npm:5.30.0": - version: 5.30.0 - resolution: "@typescript-eslint/scope-manager@npm:5.30.0" +"@typescript-eslint/type-utils@npm:5.30.5": + version: 5.30.5 + resolution: "@typescript-eslint/type-utils@npm:5.30.5" dependencies: - "@typescript-eslint/types": 5.30.0 - "@typescript-eslint/visitor-keys": 5.30.0 - checksum: 51246d0f6c497ad98fcfb02a9da92e855cd5916cf6ea117b1f8a511e4f62d1eae28f2c7278dfe29cc823c36f3b1fe87ff56681654b68faac5dfd1b897c3c58da - languageName: node - linkType: hard - -"@typescript-eslint/type-utils@npm:5.30.0": - version: 5.30.0 - resolution: "@typescript-eslint/type-utils@npm:5.30.0" - dependencies: - "@typescript-eslint/utils": 5.30.0 + "@typescript-eslint/utils": 5.30.5 debug: ^4.3.4 tsutils: ^3.21.0 peerDependencies: @@ -6065,48 +5993,23 @@ __metadata: peerDependenciesMeta: typescript: optional: true - checksum: 6185117638ca2111b8c10fe156f7de51847f784140906ba518198d056608c7eb812248d033ac0d0b46f73647dc1202f02d9aab626ee013d273fc03df11d1f6e9 - languageName: node - linkType: hard - -"@typescript-eslint/types@npm:4.33.0": - version: 4.33.0 - resolution: "@typescript-eslint/types@npm:4.33.0" - checksum: 3baae1ca35872421b4eb60f5d3f3f32dc1d513f2ae0a67dee28c7d159fd7a43ed0d11a8a5a0f0c2d38507ffa036fc7c511cb0f18a5e8ac524b3ebde77390ec53 + checksum: 080cc1231729c34b778395658374e32d034474056f9b777dbc89d20d15eb93d93d0959328ad47c2a6623d40c6552364ababadce439842a944bce001f55b731b3 languageName: node linkType: hard -"@typescript-eslint/types@npm:5.30.0": - version: 5.30.0 - resolution: "@typescript-eslint/types@npm:5.30.0" - checksum: f83a506880d78419283a86e8aeb6c744b1d1a7fc3a366625125805daf0f9a7640a778537113b8865a4cdd985dcde53066820ea044a750126bc8b478eb93d4d12 +"@typescript-eslint/types@npm:5.30.5": + version: 5.30.5 + resolution: "@typescript-eslint/types@npm:5.30.5" + checksum: c70420618cb875d4e964a20a3fa4cf40cb97a8ad3123e24860e3d829edf3b081c77fa1fe25644700499d27e44aee5783abc7765deee61e2ef59a928db96b2175 languageName: node linkType: hard -"@typescript-eslint/typescript-estree@npm:4.33.0": - version: 4.33.0 - resolution: "@typescript-eslint/typescript-estree@npm:4.33.0" +"@typescript-eslint/typescript-estree@npm:5.30.5": + version: 5.30.5 + resolution: "@typescript-eslint/typescript-estree@npm:5.30.5" dependencies: - "@typescript-eslint/types": 4.33.0 - "@typescript-eslint/visitor-keys": 4.33.0 - debug: ^4.3.1 - globby: ^11.0.3 - is-glob: ^4.0.1 - semver: ^7.3.5 - tsutils: ^3.21.0 - peerDependenciesMeta: - typescript: - optional: true - checksum: 2566984390c76bd95f43240057215c068c69769e406e27aba41e9f21fd300074d6772e4983fa58fe61e80eb5550af1548d2e31e80550d92ba1d051bb00fe6f5c - languageName: node - linkType: hard - -"@typescript-eslint/typescript-estree@npm:5.30.0": - version: 5.30.0 - resolution: "@typescript-eslint/typescript-estree@npm:5.30.0" - dependencies: - "@typescript-eslint/types": 5.30.0 - "@typescript-eslint/visitor-keys": 5.30.0 + "@typescript-eslint/types": 5.30.5 + "@typescript-eslint/visitor-keys": 5.30.5 debug: ^4.3.4 globby: ^11.1.0 is-glob: ^4.0.3 @@ -6115,43 +6018,33 @@ __metadata: peerDependenciesMeta: typescript: optional: true - checksum: cf8caea435c4346fae9c81635864efa17ea106e3dea5cd226c096145958ff6e7918e40cdb2af06526ca43d44717eb088869f1c1db51a52aa9f72b6bf65e11db2 + checksum: 19dce426c826cddd4aadf2fa15be943c6ad7d2038685cc2665749486a5f44a47819aab5d260b54f8a4babf6acf2500e9f62e709d61fce337b12d5468ff285277 languageName: node linkType: hard -"@typescript-eslint/utils@npm:5.30.0": - version: 5.30.0 - resolution: "@typescript-eslint/utils@npm:5.30.0" +"@typescript-eslint/utils@npm:5.30.5": + version: 5.30.5 + resolution: "@typescript-eslint/utils@npm:5.30.5" dependencies: "@types/json-schema": ^7.0.9 - "@typescript-eslint/scope-manager": 5.30.0 - "@typescript-eslint/types": 5.30.0 - "@typescript-eslint/typescript-estree": 5.30.0 + "@typescript-eslint/scope-manager": 5.30.5 + "@typescript-eslint/types": 5.30.5 + "@typescript-eslint/typescript-estree": 5.30.5 eslint-scope: ^5.1.1 eslint-utils: ^3.0.0 peerDependencies: eslint: ^6.0.0 || ^7.0.0 || ^8.0.0 - checksum: 176eda46292398c0fe069d96f51d0083a9b978d4c6e2ca92f10e9ebad83910d67754bcb2c1667cf4c40e06c112558ff1ad973d6f82719cfe4de7c72f89a3df29 - languageName: node - linkType: hard - -"@typescript-eslint/visitor-keys@npm:4.33.0": - version: 4.33.0 - resolution: "@typescript-eslint/visitor-keys@npm:4.33.0" - dependencies: - "@typescript-eslint/types": 4.33.0 - eslint-visitor-keys: ^2.0.0 - checksum: 59953e474ad4610c1aa23b2b1a964445e2c6201521da6367752f37939d854352bbfced5c04ea539274065e012b1337ba3ffa49c2647a240a4e87155378ba9873 + checksum: 12f68cb34a150d39708f4e09a54964360f29589885cd50f119a2061660011752ec72eff3d90111f0e597575d32aae7250a6e2c730a84963e5e30352759d5f1f4 languageName: node linkType: hard -"@typescript-eslint/visitor-keys@npm:5.30.0": - version: 5.30.0 - resolution: "@typescript-eslint/visitor-keys@npm:5.30.0" +"@typescript-eslint/visitor-keys@npm:5.30.5": + version: 5.30.5 + resolution: "@typescript-eslint/visitor-keys@npm:5.30.5" dependencies: - "@typescript-eslint/types": 5.30.0 + "@typescript-eslint/types": 5.30.5 eslint-visitor-keys: ^3.3.0 - checksum: bf2219cbb910d284e2f224aaa701932287b25fe99312f9abf6406a8f52a3a0b852c96276cd740ae3071b2561b253a6cfa30b0b8ab1d199e08f9550402f8fbf1f + checksum: c0de9ae48378eec2682b860a059518bed213ea29575aad538d8d2f8137875e7279e375a7f23d38c1c183466fdd9cf1ca1db4ed5a1d374968f9460d83e48b2437 languageName: node linkType: hard @@ -6477,28 +6370,28 @@ __metadata: dependencies: "@types/body-parser": ^1.19.2 "@types/cors": ^2.8.12 - "@types/jest": ^27.5.1 + "@types/jest": ^27.5.2 "@types/jsdom": ^16.2.14 "@types/nconf": ^0.10.2 "@types/node": ^16.11.36 "@types/supertest": ^2.0.12 "@types/uuid": ^8.3.4 - "@typescript-eslint/eslint-plugin": ^5.27.0 - "@typescript-eslint/parser": ^5.27.0 + "@typescript-eslint/eslint-plugin": ^5.30.5 + "@typescript-eslint/parser": ^5.30.5 abort-controller: ^3.0.0 body-parser: ^1.19.1 - bs58: ^3.1.0 + bs58: ^4.0.1 chardet: ^1.4.0 cookie: ^0.3.1 cors: ^2.8.5 date-fns: ^2.28.0 debug: ^4.3.4 - eslint: ^8.16.0 + eslint: ^8.19.0 eslint-config-prettier: ^8.5.0 - eslint-config-standard-with-typescript: ^21.0.1 + eslint-config-standard-with-typescript: ^22.0.0 eslint-plugin-import: ^2.26.0 eslint-plugin-node: ^11.1.0 - eslint-plugin-promise: ^5.2.0 + eslint-plugin-promise: ^6.0.0 express: ^4.17.2 fetch-opengraph: ^1.0.35 food-words: ^1.1.0 @@ -6510,19 +6403,19 @@ __metadata: moment: ^2.29.2 mongodb: ^4.3.1 nacl-signature: ^1.0.0 - nconf: ^0.8.5 + nconf: ^0.11.3 node-fetch: ^2.6.7 numeral: ^2.0.6 - prettier: ^2.6.2 - supertest: ^6.2.2 + prettier: ^2.7.1 + supertest: ^6.2.4 ts-endpoint: ^2.0.0 ts-io-error: ^2.0.0 ts-jest: ^27.1.5 - ts-node: ^10.8.0 + ts-node: ^10.8.2 ts-node-dev: ^2.0.0 tsconfig-paths: ^3.14.1 tweetnacl: ^0.14.5 - typescript: ^4.7.2 + typescript: ^4.7.4 languageName: unknown linkType: soft @@ -6539,7 +6432,7 @@ __metadata: prism-react-renderer: ^1.2.1 react: ^17.0.2 react-dom: ^17.0.2 - typescript: ^4.7.2 + typescript: ^4.7.4 languageName: unknown linkType: soft @@ -6551,19 +6444,19 @@ __metadata: "@material-ui/icons": ^4.11.3 "@material-ui/lab": ^4.0.0-alpha.61 "@types/mini-css-extract-plugin": ^2.5.1 - "@typescript-eslint/eslint-plugin": ^5.27.0 + "@typescript-eslint/eslint-plugin": ^5.30.5 autoprefixer: ^6.7.7 - bs58: ^3.1.0 + bs58: ^4.0.1 classnames: ^2.3.1 copy-webpack-plugin: ^11.0.0 cross-env: ^2.0.1 css-loader: ^6.7.1 dotenv: ^14.3.2 - eslint: ^8.16.0 - eslint-config-standard-with-typescript: ^21.0.1 + eslint: ^8.19.0 + eslint-config-standard-with-typescript: ^22.0.0 eslint-plugin-import: ^2.26.0 - eslint-plugin-promise: ^5.2.0 - eslint-plugin-react: ^7.30.0 + eslint-plugin-promise: ^6.0.0 + eslint-plugin-react: ^7.30.1 fp-ts: ^2.11.9 glob: ^7.2.0 html2canvas: ^1.4.1 @@ -6580,16 +6473,16 @@ __metadata: react-dom: ^17.0.2 sass-loader: ^12.6.0 style-loader: ^3.3.1 - ts-loader: ^9.3.0 - ts-node: ^10.8.0 + ts-loader: ^9.3.1 + ts-node: ^10.8.2 tweetnacl: ^0.14.5 tweetnacl-util: ^0.13.5 - typescript: ^4.7.2 + typescript: ^4.7.4 uglifyjs-webpack-plugin: ^2.2.0 uuid: ^2.0.3 webpack: ^5.73.0 webpack-combine-loaders: ^2.0.4 - webpack-dev-server: ^4.9.2 + webpack-dev-server: ^4.9.3 webpack-notifier: ^1.15.0 languageName: unknown linkType: soft @@ -6598,31 +6491,31 @@ __metadata: version: 0.0.0-use.local resolution: "@yttrex/shared@workspace:platforms/yttrex/shared" dependencies: - "@types/jest": ^27.5.1 + "@types/jest": ^27.5.2 "@types/node": ^16.11.36 "@types/uuid": ^8.3.4 - "@typescript-eslint/eslint-plugin": ^5.27.0 - "@typescript-eslint/parser": ^5.27.0 + "@typescript-eslint/eslint-plugin": ^5.30.5 + "@typescript-eslint/parser": ^5.30.5 date-fns: ^2.28.0 debug: ^4.3.4 - eslint: ^8.16.0 + eslint: ^8.19.0 eslint-config-prettier: ^8.5.0 - eslint-config-standard-with-typescript: ^21.0.1 + eslint-config-standard-with-typescript: ^22.0.0 eslint-plugin-import: ^2.26.0 eslint-plugin-node: ^11.1.0 - eslint-plugin-promise: ^5.2.0 + eslint-plugin-promise: ^6.0.0 express: ^4.17.2 fp-ts: ^2.11.9 io-ts: ^2.2.16 io-ts-types: ^0.5.16 jest: ^27.5.1 - prettier: ^2.6.2 + prettier: ^2.7.1 ts-endpoint: ^2.0.0 ts-io-error: ^2.0.0 ts-jest: ^27.1.5 - ts-node: ^10.8.0 + ts-node: ^10.8.2 tsconfig-paths: ^3.14.1 - typescript: ^4.7.2 + typescript: ^4.7.4 languageName: unknown linkType: soft @@ -7097,9 +6990,9 @@ __metadata: languageName: node linkType: hard -"app-builder-lib@npm:23.2.0": - version: 23.2.0 - resolution: "app-builder-lib@npm:23.2.0" +"app-builder-lib@npm:23.3.0": + version: 23.3.0 + resolution: "app-builder-lib@npm:23.3.0" dependencies: 7zip-bin: ~5.1.1 "@develar/schema-utils": ~2.6.5 @@ -7107,13 +7000,13 @@ __metadata: "@malept/flatpak-bundler": ^0.4.0 async-exit-hook: ^2.0.1 bluebird-lst: ^1.0.9 - builder-util: 23.0.9 - builder-util-runtime: 9.0.2 + builder-util: 23.3.0 + builder-util-runtime: 9.0.3 chromium-pickle-js: ^0.2.0 debug: ^4.3.4 ejs: ^3.1.7 electron-osx-sign: ^0.6.0 - electron-publish: 23.0.9 + electron-publish: 23.3.0 form-data: ^4.0.0 fs-extra: ^10.1.0 hosted-git-info: ^4.1.0 @@ -7127,7 +7020,7 @@ __metadata: semver: ^7.3.7 tar: ^6.1.11 temp-file: ^3.4.0 - checksum: b16da4f536a0d50a27378f753e176e0125668685f8dc23d300732e5d21f1c760e02548cab657660822d062c68f1ee16d2f03862293d4fd005cbac2d6698d868d + checksum: c85f098d25bc4d78adb42cb5d685ecb83affa35eaa541aa9a87c2fc9cf3831ba9def6382c8a366f725a4ea7014c95c2e1cf3d357bc43a13c32c43f18e73b2539 languageName: node linkType: hard @@ -7163,7 +7056,7 @@ __metadata: languageName: node linkType: hard -"archiver@npm:^5.3.0": +"archiver@npm:^5.3.1": version: 5.3.1 resolution: "archiver@npm:5.3.1" dependencies: @@ -7300,15 +7193,6 @@ __metadata: languageName: node linkType: hard -"array-union@npm:^1.0.2": - version: 1.0.2 - resolution: "array-union@npm:1.0.2" - dependencies: - array-uniq: ^1.0.1 - checksum: 82cec6421b6e6766556c484835a6d476a873f1b71cace5ab2b4f1b15b1e3162dc4da0d16f7a2b04d4aec18146c6638fe8f661340b31ba8e469fd811a1b45dc8d - languageName: node - linkType: hard - "array-union@npm:^2.1.0": version: 2.1.0 resolution: "array-union@npm:2.1.0" @@ -7316,7 +7200,7 @@ __metadata: languageName: node linkType: hard -"array-uniq@npm:^1.0.1, array-uniq@npm:^1.0.2": +"array-uniq@npm:^1.0.2": version: 1.0.3 resolution: "array-uniq@npm:1.0.3" checksum: 1625f06b093d8bf279b81adfec6e72951c0857d65b5e3f65f053fffe9f9dd61c2fc52cff57e38a4700817e7e3f01a4faa433d505ea9e33cdae4514c334e0bf9e @@ -7374,13 +7258,6 @@ __metadata: languageName: node linkType: hard -"arrify@npm:^2.0.1": - version: 2.0.1 - resolution: "arrify@npm:2.0.1" - checksum: 067c4c1afd182806a82e4c1cb8acee16ab8b5284fbca1ce29408e6e91281c36bb5b612f6ddfbd40a0f7a7e0c75bf2696eb94c027f6e328d6e9c52465c98e4209 - languageName: node - linkType: hard - "asap@npm:^2.0.0, asap@npm:~2.0.3": version: 2.0.6 resolution: "asap@npm:2.0.6" @@ -7885,13 +7762,6 @@ __metadata: languageName: node linkType: hard -"base-x@npm:^1.1.0": - version: 1.1.0 - resolution: "base-x@npm:1.1.0" - checksum: 54e24c32919442627fe48aaa76338f8ff02187b42eae6e75e829eef3e44e37f43b128a271733322a7c798626ac7f4fb06184db2ef0ce1da7f37c448ef6f76e26 - languageName: node - linkType: hard - "base-x@npm:^3.0.2, base-x@npm:^3.0.6": version: 3.0.9 resolution: "base-x@npm:3.0.9" @@ -8325,15 +8195,6 @@ __metadata: languageName: node linkType: hard -"bs58@npm:^3.1.0": - version: 3.1.0 - resolution: "bs58@npm:3.1.0" - dependencies: - base-x: ^1.1.0 - checksum: 6d757a49958c43bc630f3b327511ce3ee065307c24240aa86189f72df0a4a8245c7ce7e7abad209b637f155006952c7b8f04bb4aa6ee4212a891882424bca82e - languageName: node - linkType: hard - "bs58@npm:^4.0.1": version: 4.0.1 resolution: "bs58@npm:4.0.1" @@ -8474,26 +8335,26 @@ __metadata: languageName: node linkType: hard -"builder-util-runtime@npm:9.0.2": - version: 9.0.2 - resolution: "builder-util-runtime@npm:9.0.2" +"builder-util-runtime@npm:9.0.3": + version: 9.0.3 + resolution: "builder-util-runtime@npm:9.0.3" dependencies: debug: ^4.3.4 sax: ^1.2.4 - checksum: 867768865f55ed0f326f542fd0cfb10bb3619a952ef691bf9896b5dc84271639d177cf8c1ec28aba36d467743ba77886573c6c8bb7af90740e7ea18f81ef5daf + checksum: 593a0aac49a8f3e66c759c0c74254f906a5239ec7e1c9ae82fd821a5969ede0816a25d4429d8dcc8ef5e7111ab9291b521c1bafc035b7aeab5eb040b45f6b286 languageName: node linkType: hard -"builder-util@npm:23.0.9": - version: 23.0.9 - resolution: "builder-util@npm:23.0.9" +"builder-util@npm:23.3.0": + version: 23.3.0 + resolution: "builder-util@npm:23.3.0" dependencies: 7zip-bin: ~5.1.1 "@types/debug": ^4.1.6 "@types/fs-extra": ^9.0.11 app-builder-bin: 4.0.0 bluebird-lst: ^1.0.9 - builder-util-runtime: 9.0.2 + builder-util-runtime: 9.0.3 chalk: ^4.1.1 cross-spawn: ^7.0.3 debug: ^4.3.4 @@ -8505,7 +8366,7 @@ __metadata: source-map-support: ^0.5.19 stat-mode: ^1.0.0 temp-file: ^3.4.0 - checksum: 14cee9beb7d32cc158bcd8d7107d5a6c527d6f01fe5d94091ddaef8604b6bf32ad63197fa6ef26802784c2a5773c655680ec6acaaf9c66f52129913156be8281 + checksum: 88cac3ff16c8fc1d70f8d9e00ed6377a8ca4695595cf55a19bb6b8ae4110f94f417863e134faae5004ff1b9a347fbf2e3dba3aec057c189c0271d6192b5abce7 languageName: node linkType: hard @@ -8516,6 +8377,15 @@ __metadata: languageName: node linkType: hard +"builtins@npm:^5.0.1": + version: 5.0.1 + resolution: "builtins@npm:5.0.1" + dependencies: + semver: ^7.0.0 + checksum: 66d204657fe36522822a95b288943ad11b58f5eaede235b11d8c4edaa28ce4800087d44a2681524c340494aadb120a0068011acabe99d30e8f11a7d826d83515 + languageName: node + linkType: hard + "bytes@npm:3.0.0": version: 3.0.0 resolution: "bytes@npm:3.0.0" @@ -8698,13 +8568,6 @@ __metadata: languageName: node linkType: hard -"camelcase@npm:^2.0.1": - version: 2.1.1 - resolution: "camelcase@npm:2.1.1" - checksum: 20a3ef08f348de832631d605362ffe447d883ada89617144a82649363ed5860923b021f8e09681624ef774afb93ff3597cfbcf8aaf0574f65af7648f1aea5e50 - languageName: node - linkType: hard - "camelcase@npm:^4.0.0": version: 4.1.0 resolution: "camelcase@npm:4.1.0" @@ -9126,17 +8989,6 @@ __metadata: languageName: node linkType: hard -"cliui@npm:^3.0.3": - version: 3.2.0 - resolution: "cliui@npm:3.2.0" - dependencies: - string-width: ^1.0.1 - strip-ansi: ^3.0.1 - wrap-ansi: ^2.0.0 - checksum: c68d1dbc3e347bfe79ed19cc7f48007d5edd6cd8438342e32073e0b4e311e3c44e1f4f19221462bc6590de56c2df520e427533a9dde95dee25710bec322746ad - languageName: node - linkType: hard - "cliui@npm:^6.0.0": version: 6.0.0 resolution: "cliui@npm:6.0.0" @@ -9562,6 +9414,13 @@ __metadata: languageName: node linkType: hard +"connect-history-api-fallback@npm:^2.0.0": + version: 2.0.0 + resolution: "connect-history-api-fallback@npm:2.0.0" + checksum: dc5368690f4a5c413889792f8df70d5941ca9da44523cde3f87af0745faee5ee16afb8195434550f0504726642734f2683d6c07f8b460f828a12c45fbd4c9a68 + languageName: node + linkType: hard + "consola@npm:^2.15.3": version: 2.15.3 resolution: "consola@npm:2.15.3" @@ -10013,35 +9872,6 @@ __metadata: languageName: node linkType: hard -"cp-file@npm:^7.0.0": - version: 7.0.0 - resolution: "cp-file@npm:7.0.0" - dependencies: - graceful-fs: ^4.1.2 - make-dir: ^3.0.0 - nested-error-stacks: ^2.0.0 - p-event: ^4.1.0 - checksum: dd60ed8d865d25a69548e15b21dd0d2fc66f10371e4970aa21b626a7578ebf419f44f386977ed3b3726c07401d4a64ee679cf1da566d8f66f01e9a359b85201f - languageName: node - linkType: hard - -"cpy@npm:^8.1.2": - version: 8.1.2 - resolution: "cpy@npm:8.1.2" - dependencies: - arrify: ^2.0.1 - cp-file: ^7.0.0 - globby: ^9.2.0 - has-glob: ^1.0.0 - junk: ^3.1.0 - nested-error-stacks: ^2.1.0 - p-all: ^2.1.0 - p-filter: ^2.1.0 - p-map: ^3.0.0 - checksum: e121f13f2b6af4a7c00de17984086a45b67eaaeeb0286a5cf67f2fdaf18d8ce6c2a9fe4ccfa37953e6982f55772f384f040f45f1961530655838c2b7486788a7 - languageName: node - linkType: hard - "crc-32@npm:^1.2.0": version: 1.2.2 resolution: "crc-32@npm:1.2.2" @@ -10545,16 +10375,6 @@ __metadata: languageName: node linkType: hard -"cwd@npm:^0.10.0": - version: 0.10.0 - resolution: "cwd@npm:0.10.0" - dependencies: - find-pkg: ^0.1.2 - fs-exists-sync: ^0.1.0 - checksum: 55ab180af86306fe7268c63dd87a737a12e1cb5146be6bcd7fe298df5f5c594cad85907a47fee02cee322d7dc98197a2b45e4d7ebfb0b2c93892bde7d787fe56 - languageName: node - linkType: hard - "cyclist@npm:^1.0.1": version: 1.0.1 resolution: "cyclist@npm:1.0.1" @@ -11074,7 +10894,7 @@ __metadata: languageName: node linkType: hard -"decamelize@npm:^1.1.0, decamelize@npm:^1.1.1, decamelize@npm:^1.2.0": +"decamelize@npm:^1.1.0, decamelize@npm:^1.2.0": version: 1.2.0 resolution: "decamelize@npm:1.2.0" checksum: ad8c51a7e7e0720c70ec2eeb1163b66da03e7616d7b98c9ef43cce2416395e84c1e9548dd94f5f6ffecfee9f8b94251fc57121a8b021f2ff2469b2bae247b8aa @@ -11256,7 +11076,7 @@ __metadata: languageName: node linkType: hard -"del@npm:^6.0.0": +"del@npm:^6.0.0, del@npm:^6.1.1": version: 6.1.1 resolution: "del@npm:6.1.1" dependencies: @@ -11473,15 +11293,6 @@ __metadata: languageName: node linkType: hard -"dir-glob@npm:^2.2.2": - version: 2.2.2 - resolution: "dir-glob@npm:2.2.2" - dependencies: - path-type: ^3.0.0 - checksum: 3aa48714a9f7845ffc30ab03a5c674fe760477cc55e67b0847333371549227d93953e6627ec160f75140c5bea5c5f88d13c01de79bd1997a588efbcf06980842 - languageName: node - linkType: hard - "dir-glob@npm:^3.0.1": version: 3.0.1 resolution: "dir-glob@npm:3.0.1" @@ -11491,13 +11302,13 @@ __metadata: languageName: node linkType: hard -"dmg-builder@npm:23.2.0": - version: 23.2.0 - resolution: "dmg-builder@npm:23.2.0" +"dmg-builder@npm:23.3.0": + version: 23.3.0 + resolution: "dmg-builder@npm:23.3.0" dependencies: - app-builder-lib: 23.2.0 - builder-util: 23.0.9 - builder-util-runtime: 9.0.2 + app-builder-lib: 23.3.0 + builder-util: 23.3.0 + builder-util-runtime: 9.0.3 dmg-license: ^1.0.11 fs-extra: ^10.0.0 iconv-lite: ^0.6.2 @@ -11505,7 +11316,7 @@ __metadata: dependenciesMeta: dmg-license: optional: true - checksum: 052c154733d96435281c5ca619b2be82097bff25cc7daa7f456259edb87636285523f224cf467873d00f70324777253165cfe3948deff8f31d1639c43dbf8aea + checksum: 942390af28f468775aca392665f4495bb78aebb0d0f5a7040165a2004d2b1ca294368830d1dd037024762387f2d2f2a94edcebe9cb087fdda9d0a8e926c709fd languageName: node linkType: hard @@ -11973,16 +11784,16 @@ __metadata: languageName: node linkType: hard -"electron-builder@npm:^23.2.0": - version: 23.2.0 - resolution: "electron-builder@npm:23.2.0" +"electron-builder@npm:^23.3.0": + version: 23.3.0 + resolution: "electron-builder@npm:23.3.0" dependencies: "@types/yargs": ^17.0.1 - app-builder-lib: 23.2.0 - builder-util: 23.0.9 - builder-util-runtime: 9.0.2 + app-builder-lib: 23.3.0 + builder-util: 23.3.0 + builder-util-runtime: 9.0.3 chalk: ^4.1.1 - dmg-builder: 23.2.0 + dmg-builder: 23.3.0 fs-extra: ^10.0.0 is-ci: ^3.0.0 lazy-val: ^1.0.5 @@ -11992,7 +11803,7 @@ __metadata: bin: electron-builder: cli.js install-app-deps: install-app-deps.js - checksum: 73be64f3aab3db8a8522cf7b43c3a49292614e37340c621d0e577b0854661471f61baa9771d6be1abccba30937cf1999aa05e90138bb41ed346e9feb34a0b6e8 + checksum: 77cfdf105fec315792228c379a9eb4bb4338982db860e76162a8f17c04d5592f63854bd918d0ca89692a5aeb3db94214317b317524396214effc5738c1757541 languageName: node linkType: hard @@ -12039,7 +11850,7 @@ __metadata: languageName: node linkType: hard -"electron-log@npm:^4.4.7": +"electron-log@npm:^4.4.8": version: 4.4.8 resolution: "electron-log@npm:4.4.8" checksum: 5a7958f8fd9a5c95987e885ffe1c5b88be15bb41d348b65c92e2d8664519faddf17ce0208cd4e7027b0908f92fecedfbef246a45061f53e4a53b5d2201aafbd9 @@ -12063,18 +11874,18 @@ __metadata: languageName: node linkType: hard -"electron-publish@npm:23.0.9": - version: 23.0.9 - resolution: "electron-publish@npm:23.0.9" +"electron-publish@npm:23.3.0": + version: 23.3.0 + resolution: "electron-publish@npm:23.3.0" dependencies: "@types/fs-extra": ^9.0.11 - builder-util: 23.0.9 - builder-util-runtime: 9.0.2 + builder-util: 23.3.0 + builder-util-runtime: 9.0.3 chalk: ^4.1.1 fs-extra: ^10.0.0 lazy-val: ^1.0.5 mime: ^2.5.2 - checksum: 866515f5eeabf1369e84c4d449184ae9292231f68d25e6cb1f170595d69ad23f83135fb63646601bd83e9a9092167e6b59773863a535e099cf8fe6a88db7e457 + checksum: c4f8ad9344e7b12d686a952cd66e3e8c06ab88b0be0cec352314f0bf8fc6da87fc8d2ac0ddbaa753f03fbda804039e11802a00a62d5a9f2690c4e6fc61a27a70 languageName: node linkType: hard @@ -12526,32 +12337,32 @@ __metadata: languageName: node linkType: hard -"eslint-config-standard-with-typescript@npm:^21.0.1": - version: 21.0.1 - resolution: "eslint-config-standard-with-typescript@npm:21.0.1" +"eslint-config-standard-with-typescript@npm:^22.0.0": + version: 22.0.0 + resolution: "eslint-config-standard-with-typescript@npm:22.0.0" dependencies: - "@typescript-eslint/parser": ^4.0.0 - eslint-config-standard: ^16.0.0 + "@typescript-eslint/parser": ^5.0.0 + eslint-config-standard: 17.0.0 peerDependencies: - "@typescript-eslint/eslint-plugin": ^4.0.1 - eslint: ^7.12.1 - eslint-plugin-import: ^2.22.1 - eslint-plugin-node: ^11.1.0 - eslint-plugin-promise: ^4.2.1 || ^5.0.0 - typescript: ^3.9 || ^4.0.0 - checksum: c3912558f26c4e4a38df4932d0f89fb95f7a9d5041386417995ce2bed35d64515c66a9a023265eb6c840a4e859bc7f434275ef773493a312e5941e1ceb0da508 + "@typescript-eslint/eslint-plugin": ^5.0.0 + eslint: ^8.0.1 + eslint-plugin-import: ^2.25.2 + eslint-plugin-n: ^15.0.0 + eslint-plugin-promise: ^6.0.0 + typescript: "*" + checksum: 4ed15e7383f8e4d657b5ba1fec52753024c3740decf98a737fe49467f913215f674d74515d40c0b692807f5b3ec519ec5b6a6e36ed39ea96f175e703c809dfa0 languageName: node linkType: hard -"eslint-config-standard@npm:^16.0.0": - version: 16.0.3 - resolution: "eslint-config-standard@npm:16.0.3" +"eslint-config-standard@npm:17.0.0": + version: 17.0.0 + resolution: "eslint-config-standard@npm:17.0.0" peerDependencies: - eslint: ^7.12.1 - eslint-plugin-import: ^2.22.1 - eslint-plugin-node: ^11.1.0 - eslint-plugin-promise: ^4.2.1 || ^5.0.0 - checksum: 6ae193634f289ae95dbbf2291dc1e7c5bedef2425c594db07ec58476c902e6eb51a2b1c9cd2bad3772e921f5515dc2f8fb5447f7a56c20c99801ef1296c3bfef + eslint: ^8.0.1 + eslint-plugin-import: ^2.25.2 + eslint-plugin-n: ^15.0.0 + eslint-plugin-promise: ^6.0.0 + checksum: dc0ed51e186fd963ff2c0819d33ef580afce11b11036cbcf5e74427e26e514c2b1be96b8ffe74fd2fd00263554a0d49cc873fcf76f17c3dfdba614b45d7fd7da languageName: node linkType: hard @@ -12587,6 +12398,18 @@ __metadata: languageName: node linkType: hard +"eslint-plugin-es@npm:^4.1.0": + version: 4.1.0 + resolution: "eslint-plugin-es@npm:4.1.0" + dependencies: + eslint-utils: ^2.0.0 + regexpp: ^3.0.0 + peerDependencies: + eslint: ">=4.19.1" + checksum: 26b87a216d3625612b1d3ca8653ac8a1d261046d2a973bb0eb2759070267d2bfb0509051facdeb5ae03dc8dfb51a434be23aff7309a752ca901d637da535677f + languageName: node + linkType: hard + "eslint-plugin-import@npm:^2.26.0": version: 2.26.0 resolution: "eslint-plugin-import@npm:2.26.0" @@ -12610,6 +12433,24 @@ __metadata: languageName: node linkType: hard +"eslint-plugin-n@npm:^15.2.4": + version: 15.2.4 + resolution: "eslint-plugin-n@npm:15.2.4" + dependencies: + builtins: ^5.0.1 + eslint-plugin-es: ^4.1.0 + eslint-utils: ^3.0.0 + ignore: ^5.1.1 + is-core-module: ^2.9.0 + minimatch: ^3.1.2 + resolve: ^1.10.1 + semver: ^7.3.7 + peerDependencies: + eslint: ">=7.0.0" + checksum: dd651651ab76120e45707ee968d846e3ffffb42d1035792fdef6d3b0dcfddf3673bc6a09cb2fac8c5f1d081f14f2a67fc52295d5ed1d2edfb5beead93284eaac + languageName: node + linkType: hard + "eslint-plugin-node@npm:^11.1.0": version: 11.1.0 resolution: "eslint-plugin-node@npm:11.1.0" @@ -12626,16 +12467,16 @@ __metadata: languageName: node linkType: hard -"eslint-plugin-promise@npm:^5.2.0": - version: 5.2.0 - resolution: "eslint-plugin-promise@npm:5.2.0" +"eslint-plugin-promise@npm:^6.0.0": + version: 6.0.0 + resolution: "eslint-plugin-promise@npm:6.0.0" peerDependencies: - eslint: ^7.0.0 - checksum: 5d6b2d28408c5afde6386942862427af3d83c9a130eb2555bb54b26a1761914e2c7326aca1be26dd3fee6405e65a2ee9432a4526147e5962545060ea0ef64058 + eslint: ^7.0.0 || ^8.0.0 + checksum: 7e761507c51267b77e4ad710e7c8938aa4f8f69b975886034e57497a1816e9527eda364e25aac03d1b4e0df2e738ba98e49ad075d028824fcfea533a1419751c languageName: node linkType: hard -"eslint-plugin-react@npm:^7.30.0": +"eslint-plugin-react@npm:^7.30.1": version: 7.30.1 resolution: "eslint-plugin-react@npm:7.30.1" dependencies: @@ -12720,9 +12561,9 @@ __metadata: languageName: node linkType: hard -"eslint@npm:^8.16.0": - version: 8.18.0 - resolution: "eslint@npm:8.18.0" +"eslint@npm:^8.19.0": + version: 8.19.0 + resolution: "eslint@npm:8.19.0" dependencies: "@eslint/eslintrc": ^1.3.0 "@humanwhocodes/config-array": ^0.9.2 @@ -12761,7 +12602,7 @@ __metadata: v8-compile-cache: ^2.0.3 bin: eslint: bin/eslint.js - checksum: d9b4b7488a9cee97608343cbb5ac652d3f316436f95ef0800cd9497c1c6f877b655a3275817989c02f1ff0d5dfd1959c5092af9251c7e3fcf60659da37752a10 + checksum: 0bc9df1a3a09dcd5a781ec728f280aa8af3ab19c2d1f14e2668b5ee5b8b1fb0e72dde5c3acf738e7f4281685fb24ec149b6154255470b06cf41de76350bca7a4 languageName: node linkType: hard @@ -13024,15 +12865,6 @@ __metadata: languageName: node linkType: hard -"expand-tilde@npm:^1.2.2": - version: 1.2.2 - resolution: "expand-tilde@npm:1.2.2" - dependencies: - os-homedir: ^1.0.1 - checksum: 18051cd104977bc06e2bb1347db9959b90504437beea0de6fd287a3c8c58b41e2330337bd189cfca2ee4be6bda9bf045f8c07daf23e622f85eb6ee1c420619a0 - languageName: node - linkType: hard - "expect@npm:^27.5.1": version: 27.5.1 resolution: "expect@npm:27.5.1" @@ -13223,20 +13055,6 @@ __metadata: languageName: node linkType: hard -"fast-glob@npm:^2.2.6": - version: 2.2.7 - resolution: "fast-glob@npm:2.2.7" - dependencies: - "@mrmlnc/readdir-enhanced": ^2.2.1 - "@nodelib/fs.stat": ^1.1.2 - glob-parent: ^3.1.0 - is-glob: ^4.0.0 - merge2: ^1.2.3 - micromatch: ^3.1.10 - checksum: 304ccff1d437fcc44ae0168b0c3899054b92e0fd6af6ad7c3ccc82ab4ddd210b99c7c739d60ee3686da2aa165cd1a31810b31fd91f7c2a575d297342a9fc0534 - languageName: node - linkType: hard - "fast-glob@npm:^3.1.1, fast-glob@npm:^3.2.11, fast-glob@npm:^3.2.7, fast-glob@npm:^3.2.9": version: 3.2.11 resolution: "fast-glob@npm:3.2.11" @@ -13482,20 +13300,21 @@ __metadata: languageName: node linkType: hard -"filemanager-webpack-plugin@npm:^6.1.7": - version: 6.1.7 - resolution: "filemanager-webpack-plugin@npm:6.1.7" +"filemanager-webpack-plugin@npm:^7.0.0": + version: 7.0.0 + resolution: "filemanager-webpack-plugin@npm:7.0.0" dependencies: - "@types/archiver": ^5.1.1 - archiver: ^5.3.0 - cpy: ^8.1.2 - del: ^6.0.0 - fs-extra: ^10.0.0 - is-glob: ^4.0.1 - schema-utils: ^3.1.1 + "@types/archiver": ^5.3.1 + archiver: ^5.3.1 + del: ^6.1.1 + fast-glob: ^3.2.11 + fs-extra: ^10.1.0 + is-glob: ^4.0.3 + normalize-path: ^3.0.0 + schema-utils: ^4.0.0 peerDependencies: webpack: ^5.0.0 - checksum: d24ef64db7907bf18d64c5c53ee9272ec7a1fa4cd7252416cb9a9d6b25ca2d34111dabe78f1b1efaf2318de0c3456bc2ed69d31998e057ed1491e12f9a710184 + checksum: 00fc9390609e31ea7b3f536ec5044c3cbdb688d9beddcc7b5d46ad7379242d174f829a1932d43923a3347001eea4258caa61c455733ecc7a6d541b7a1f62b24e languageName: node linkType: hard @@ -13578,16 +13397,6 @@ __metadata: languageName: node linkType: hard -"find-file-up@npm:^0.1.2": - version: 0.1.3 - resolution: "find-file-up@npm:0.1.3" - dependencies: - fs-exists-sync: ^0.1.0 - resolve-dir: ^0.1.0 - checksum: 95475fee7b727266ec65312527c580eb4f01884592620296cf7859e72cce7f4f6a667c964ad6feeec53fb72a7c3991805532ed7a53d8224e9a1ccd88479cabce - languageName: node - linkType: hard - "find-index@npm:^0.1.1": version: 0.1.1 resolution: "find-index@npm:0.1.1" @@ -13595,28 +13404,6 @@ __metadata: languageName: node linkType: hard -"find-pkg@npm:^0.1.2": - version: 0.1.2 - resolution: "find-pkg@npm:0.1.2" - dependencies: - find-file-up: ^0.1.2 - checksum: cd797bfa7dd419849559312cdd3aec767c39939e552daa92e53ff6b61108f331eb2c800d20a5973631eb894ea36c13dded01a868b10f457a685e0ae87a1746e1 - languageName: node - linkType: hard - -"find-process@npm:^1.4.7": - version: 1.4.7 - resolution: "find-process@npm:1.4.7" - dependencies: - chalk: ^4.0.0 - commander: ^5.1.0 - debug: ^4.1.1 - bin: - find-process: bin/find-process.js - checksum: 1953e6a16af86ec033d613ddfcac24f68b7ca6cc7d7aadc037ede4ccad4f03c5571d3c95165842475bfa9432120be5c995cc234c9c02726fc886ac6cd85ece3b - languageName: node - linkType: hard - "find-root@npm:^1.1.0": version: 1.1.0 resolution: "find-root@npm:1.1.0" @@ -13953,13 +13740,6 @@ __metadata: languageName: node linkType: hard -"fs-exists-sync@npm:^0.1.0": - version: 0.1.0 - resolution: "fs-exists-sync@npm:0.1.0" - checksum: 850a0d6e4c03a7bd2fd25043f77cd9d6be9c3b48bb99308bcfe9c94f3f92f65f2cd3fa036e13a1b0ba7a46d2e58792f53e578f01d75fbdcd56baeb9eed63b705 - languageName: node - linkType: hard - "fs-extra@npm:^10.0.0, fs-extra@npm:^10.1.0": version: 10.1.0 resolution: "fs-extra@npm:10.1.0" @@ -14494,13 +14274,6 @@ __metadata: languageName: node linkType: hard -"glob-to-regexp@npm:^0.3.0": - version: 0.3.0 - resolution: "glob-to-regexp@npm:0.3.0" - checksum: d34b3219d860042d508c4893b67617cd16e2668827e445ff39cff9f72ef70361d3dc24f429e003cdfb6607c75c9664b8eadc41d2eeb95690af0b0d3113c1b23b - languageName: node - linkType: hard - "glob-to-regexp@npm:^0.4.1": version: 0.4.1 resolution: "glob-to-regexp@npm:0.4.1" @@ -14602,16 +14375,6 @@ __metadata: languageName: node linkType: hard -"global-modules@npm:^0.2.3": - version: 0.2.3 - resolution: "global-modules@npm:0.2.3" - dependencies: - global-prefix: ^0.1.4 - is-windows: ^0.2.0 - checksum: 3801788df54897d994c9c8f3d09f253d1379cd879ae61fcddbcc3ecdfdf6fe23a1edb983e8d4dd24cebf7e49823752e1cd29a2d33bdb4de587de8b4a85b17e24 - languageName: node - linkType: hard - "global-modules@npm:^2.0.0": version: 2.0.0 resolution: "global-modules@npm:2.0.0" @@ -14621,18 +14384,6 @@ __metadata: languageName: node linkType: hard -"global-prefix@npm:^0.1.4": - version: 0.1.5 - resolution: "global-prefix@npm:0.1.5" - dependencies: - homedir-polyfill: ^1.0.0 - ini: ^1.3.4 - is-windows: ^0.2.0 - which: ^1.2.12 - checksum: ea1b818a1851655ebb2341cdd5446da81c25f31ca6f0ac358a234cbed5442edc1bfa5628771466988d67d9fcc6ad09ca0e68a8d3d7e3d92f7de3aec87020e183 - languageName: node - linkType: hard - "global-prefix@npm:^3.0.0": version: 3.0.0 resolution: "global-prefix@npm:3.0.0" @@ -14722,22 +14473,6 @@ __metadata: languageName: node linkType: hard -"globby@npm:^9.2.0": - version: 9.2.0 - resolution: "globby@npm:9.2.0" - dependencies: - "@types/glob": ^7.1.1 - array-union: ^1.0.2 - dir-glob: ^2.2.2 - fast-glob: ^2.2.6 - glob: ^7.1.3 - ignore: ^4.0.3 - pify: ^4.0.1 - slash: ^2.0.0 - checksum: 9b4cb70aa0b43bf89b18cf0e543695185e16d8dd99c17bdc6a1df0a9f88ff9dc8d2467aebace54c3842fc451a564882948c87a3b4fbdb1cacf3e05fd54b6ac5d - languageName: node - linkType: hard - "globule@npm:^1.0.0": version: 1.3.4 resolution: "globule@npm:1.3.4" @@ -14851,14 +14586,14 @@ __metadata: "@types/lodash": ^4.14.182 "@types/nconf": ^0.10.2 "@types/prettier": ^2.6.3 - "@types/react": ^17.0.45 + "@types/react": ^17.0.47 "@types/react-dom": ^17.0.17 "@types/unzipper": ^0.10.5 "@types/webpack-node-externals": ^2.5.3 "@types/ws": ^8.5.3 "@types/yargs": ^17.0.10 - "@typescript-eslint/eslint-plugin": ^5.27.0 - "@typescript-eslint/parser": ^5.27.0 + "@typescript-eslint/eslint-plugin": ^5.30.5 + "@typescript-eslint/parser": ^5.30.5 axios: ^0.24.0 bufferutil: ^4.0.6 canvas: ^2.9.1 @@ -14867,19 +14602,19 @@ __metadata: csv-stringify: ^6.0.5 debug: ^4.3.4 electron: 16.2.8 - electron-builder: ^23.2.0 + electron-builder: ^23.3.0 electron-debug: ^3.2.0 - electron-log: ^4.4.7 + electron-log: ^4.4.8 electron-reloader: ^1.2.3 electron-squirrel-startup: ^1.0.0 electron-store: ^8.0.2 electron-unhandled: ^4.0.1 - eslint: ^8.16.0 + eslint: ^8.19.0 eslint-config-prettier: ^8.5.0 - eslint-config-standard-with-typescript: ^21.0.1 + eslint-config-standard-with-typescript: ^22.0.0 eslint-plugin-import: ^2.26.0 eslint-plugin-node: ^11.1.0 - eslint-plugin-promise: ^5.2.0 + eslint-plugin-promise: ^6.0.0 extract-zip: ^2.0.1 fp-ts: ^2.11.9 io-ts: ^2.2.16 @@ -14894,7 +14629,7 @@ __metadata: node-loader: ^2.0.0 node-polyfill-webpack-plugin: ^1.1.4 pkg: ^5.7.0 - prettier: ^2.6.2 + prettier: ^2.7.1 puppeteer: ^13.7.0 puppeteer-core: ^13.7.0 puppeteer-extra: ^3.2.3 @@ -14902,17 +14637,17 @@ __metadata: puppeteer-in-electron: ^3.0.5 react: ^17.0.2 react-dom: ^17.0.2 - react-router: ^5.2.1 - react-router-dom: ^5.3.0 + react-router: ^5.3.3 + react-router-dom: ^5.3.3 react-swipeable-views: ^0.14.0 rpmbuild: ^0.0.23 string-replace-loader: ^3.1.0 style-loader: ^3.3.1 ts-jest: ^27.1.5 - ts-loader: ^9.3.0 - ts-node: ^10.8.0 + ts-loader: ^9.3.1 + ts-node: ^10.8.2 tsconfig-paths: ^3.14.1 - typescript: ^4.7.2 + typescript: ^4.7.4 unlazy-loader: ^0.1.3 unzipper: ^0.10.11 utf-8-validate: ^5.0.9 @@ -14921,6 +14656,7 @@ __metadata: webpack-cli: ^4.10.0 webpack-node-externals: ^3.0.0 ws: ^8.5.0 + xvfb: ^0.4.0 yaml: ^1.10.2 yargs: ^17.5.1 bin: @@ -15033,15 +14769,6 @@ __metadata: languageName: node linkType: hard -"has-glob@npm:^1.0.0": - version: 1.0.0 - resolution: "has-glob@npm:1.0.0" - dependencies: - is-glob: ^3.0.0 - checksum: cafad93e599f49f676a9ab444ec90210fcda35ac14ad6c9bb96c08057ad18a1318f1116b053aa6bdc744f19252537006872d3fc76785e842bbe8cc4312447fc8 - languageName: node - linkType: hard - "has-property-descriptors@npm:^1.0.0": version: 1.0.0 resolution: "has-property-descriptors@npm:1.0.0" @@ -15312,15 +15039,6 @@ __metadata: languageName: node linkType: hard -"homedir-polyfill@npm:^1.0.0": - version: 1.0.3 - resolution: "homedir-polyfill@npm:1.0.3" - dependencies: - parse-passwd: ^1.0.0 - checksum: 18dd4db87052c6a2179d1813adea0c4bfcfa4f9996f0e226fefb29eb3d548e564350fa28ec46b0bf1fbc0a1d2d6922ceceb80093115ea45ff8842a4990139250 - languageName: node - linkType: hard - "hosted-git-info@npm:^2.1.4": version: 2.8.9 resolution: "hosted-git-info@npm:2.8.9" @@ -15365,7 +15083,7 @@ __metadata: languageName: node linkType: hard -"html-escaper@npm:^2.0.0, html-escaper@npm:^2.0.2": +"html-escaper@npm:^2.0.0": version: 2.0.2 resolution: "html-escaper@npm:2.0.2" checksum: d2df2da3ad40ca9ee3a39c5cc6475ef67c8f83c234475f24d8e9ce0dc80a2c82df8e1d6fa78ddd1e9022a586ea1bd247a615e80a5cd9273d90111ddda7d9e974 @@ -15660,12 +15378,12 @@ __metadata: languageName: node linkType: hard -"i18next@npm:^21.6.10": - version: 21.8.11 - resolution: "i18next@npm:21.8.11" +"i18next@npm:^21.8.13": + version: 21.8.13 + resolution: "i18next@npm:21.8.13" dependencies: "@babel/runtime": ^7.17.2 - checksum: 94b9de3811456c2720b96d54febb89fe66cef3146044cdba9cc35543f93bf04843ccffb191ff4dbed614e7d4a64f2e29391dd514e054331738f2411c1936d85f + checksum: 0cc9a5213326d5c33a25b7fd753cd759a8357574d8e878372eeb84fa4e1b0839d3e840b6193ae6b37317d3af6c385c67799c830bb5f3ca388462b0bcb4464a97 languageName: node linkType: hard @@ -15736,13 +15454,6 @@ __metadata: languageName: node linkType: hard -"ignore@npm:^4.0.3": - version: 4.0.6 - resolution: "ignore@npm:4.0.6" - checksum: 248f82e50a430906f9ee7f35e1158e3ec4c3971451dd9f99c9bc1548261b4db2b99709f60ac6c6cac9333494384176cc4cc9b07acbe42d52ac6a09cad734d800 - languageName: node - linkType: hard - "ignore@npm:^5.1.1, ignore@npm:^5.1.4, ignore@npm:^5.2.0": version: 5.2.0 resolution: "ignore@npm:5.2.0" @@ -15888,7 +15599,7 @@ __metadata: languageName: node linkType: hard -"ini@npm:^1.3.0, ini@npm:^1.3.2, ini@npm:^1.3.4, ini@npm:^1.3.5, ini@npm:~1.3.0": +"ini@npm:^1.3.2, ini@npm:^1.3.4, ini@npm:^1.3.5, ini@npm:~1.3.0": version: 1.3.8 resolution: "ini@npm:1.3.8" checksum: dfd98b0ca3a4fc1e323e38a6c8eb8936e31a97a918d3b377649ea15bdb15d481207a0dda1021efbd86b464cae29a0d33c1d7dcaf6c5672bee17fa849bc50a1b3 @@ -15996,13 +15707,6 @@ __metadata: languageName: node linkType: hard -"invert-kv@npm:^1.0.0": - version: 1.0.0 - resolution: "invert-kv@npm:1.0.0" - checksum: aebeee31dda3b3d25ffd242e9a050926e7fe5df642d60953ab183aca1a7d1ffb39922eb2618affb0e850cf2923116f0da1345367759d88d097df5da1f1e1590e - languageName: node - linkType: hard - "io-ts-types@npm:^0.5.16": version: 0.5.16 resolution: "io-ts-types@npm:0.5.16" @@ -16352,7 +16056,7 @@ __metadata: languageName: node linkType: hard -"is-glob@npm:^3.0.0, is-glob@npm:^3.1.0": +"is-glob@npm:^3.1.0": version: 3.1.0 resolution: "is-glob@npm:3.1.0" dependencies: @@ -16740,13 +16444,6 @@ __metadata: languageName: node linkType: hard -"is-windows@npm:^0.2.0": - version: 0.2.0 - resolution: "is-windows@npm:0.2.0" - checksum: 3df25afda2fd9f3926b08cebacf1fc0a1fe7805a2cb73ef0f1b911c949e4e7648c4623979d74b4502bdd9af69471101eb6051b751595f7f88569148186cf7a7a - languageName: node - linkType: hard - "is-windows@npm:^1.0.2": version: 1.0.2 resolution: "is-windows@npm:1.0.2" @@ -17047,21 +16744,6 @@ __metadata: languageName: node linkType: hard -"jest-dev-server@npm:^6.0.3": - version: 6.0.3 - resolution: "jest-dev-server@npm:6.0.3" - dependencies: - chalk: ^4.1.2 - cwd: ^0.10.0 - find-process: ^1.4.7 - prompts: ^2.4.2 - spawnd: ^6.0.2 - tree-kill: ^1.2.2 - wait-on: ^6.0.0 - checksum: 58b41b3c395fc59bdc7356b74435c81ce1054bd9f0bfbcb4ddbfedd5a18dab40bea21e130ed5eafdf4bc8c5e066d3f653287f3a4438079d94d5a1417f96627d5 - languageName: node - linkType: hard - "jest-diff@npm:^27.5.1": version: 27.5.1 resolution: "jest-diff@npm:27.5.1" @@ -17897,13 +17579,6 @@ __metadata: languageName: node linkType: hard -"junk@npm:^3.1.0": - version: 3.1.0 - resolution: "junk@npm:3.1.0" - checksum: 6c4d68e8f8bc25b546baed802cd0e7be6a971e92f1e885c92cbfe98946d5690b961a32f8e7909e77765d3204c3e556d13c17f73e31697ffae1db07a58b9e68c0 - languageName: node - linkType: hard - "keyboardevent-from-electron-accelerator@npm:^2.0.0": version: 2.0.0 resolution: "keyboardevent-from-electron-accelerator@npm:2.0.0" @@ -18044,15 +17719,6 @@ __metadata: languageName: node linkType: hard -"lcid@npm:^1.0.0": - version: 1.0.0 - resolution: "lcid@npm:1.0.0" - dependencies: - invert-kv: ^1.0.0 - checksum: e8c7a4db07663068c5c44b650938a2bc41aa992037eebb69376214320f202c1250e70b50c32f939e28345fd30c2d35b8e8cd9a19d5932c398246a864ce54843d - languageName: node - linkType: hard - "level-concat-iterator@npm:~2.0.0": version: 2.0.1 resolution: "level-concat-iterator@npm:2.0.1" @@ -19004,7 +18670,7 @@ __metadata: languageName: node linkType: hard -"merge2@npm:^1.2.3, merge2@npm:^1.3.0, merge2@npm:^1.4.1": +"merge2@npm:^1.3.0, merge2@npm:^1.4.1": version: 1.4.1 resolution: "merge2@npm:1.4.1" checksum: 7268db63ed5169466540b6fb947aec313200bcf6d40c5ab722c22e242f651994619bcd85601602972d3c85bd2cc45a358a4c61937e9f11a061919a1da569b0c2 @@ -19152,7 +18818,7 @@ __metadata: languageName: node linkType: hard -"mime@npm:2.6.0, mime@npm:^2.5.0, mime@npm:^2.5.2": +"mime@npm:2.6.0, mime@npm:^2.5.2": version: 2.6.0 resolution: "mime@npm:2.6.0" bin: @@ -19716,18 +19382,6 @@ __metadata: languageName: node linkType: hard -"nconf@npm:^0.8.5": - version: 0.8.5 - resolution: "nconf@npm:0.8.5" - dependencies: - async: ^1.4.0 - ini: ^1.3.0 - secure-keys: ^1.0.0 - yargs: ^3.19.0 - checksum: 5f51381c7fab3ec5fe2edcb63dde0c4cc56186ea34643bbd06bd6a5f6fbb65e7e180dbb3c658a3e11c7d6181d9abcd7432a0a4ff005f3dd25e4f834f2b517d7a - languageName: node - linkType: hard - "ncp@npm:~0.4.2": version: 0.4.2 resolution: "ncp@npm:0.4.2" @@ -19764,13 +19418,6 @@ __metadata: languageName: node linkType: hard -"nested-error-stacks@npm:^2.0.0, nested-error-stacks@npm:^2.1.0": - version: 2.1.1 - resolution: "nested-error-stacks@npm:2.1.1" - checksum: 5f452fad75db8480b4db584e1602894ff5977f8bf3d2822f7ba5cb7be80e89adf1fffa34dada3347ef313a4288850b4486eb0635b315c32bdfb505577e8880e3 - languageName: node - linkType: hard - "netmask@npm:^2.0.2": version: 2.0.2 resolution: "netmask@npm:2.0.2" @@ -20578,22 +20225,6 @@ __metadata: languageName: node linkType: hard -"os-homedir@npm:^1.0.1": - version: 1.0.2 - resolution: "os-homedir@npm:1.0.2" - checksum: af609f5a7ab72de2f6ca9be6d6b91a599777afc122ac5cad47e126c1f67c176fe9b52516b9eeca1ff6ca0ab8587fe66208bc85e40a3940125f03cdb91408e9d2 - languageName: node - linkType: hard - -"os-locale@npm:^1.4.0": - version: 1.4.0 - resolution: "os-locale@npm:1.4.0" - dependencies: - lcid: ^1.0.0 - checksum: 0161a1b6b5a8492f99f4b47fe465df9fc521c55ba5414fce6444c45e2500487b8ed5b40a47a98a2363fe83ff04ab033785300ed8df717255ec4c3b625e55b1fb - languageName: node - linkType: hard - "os-name@npm:4.0.1": version: 4.0.1 resolution: "os-name@npm:4.0.1" @@ -20611,15 +20242,6 @@ __metadata: languageName: node linkType: hard -"p-all@npm:^2.1.0": - version: 2.1.0 - resolution: "p-all@npm:2.1.0" - dependencies: - p-map: ^2.0.0 - checksum: 6c20134eb3f16dca270d04a40cd14d2d05012b5a5762ca4f89962ae03a5fc13e13b09f64626a780f10bbe4e204b9370f708c6d8c079296bd2512d7e15462c76f - languageName: node - linkType: hard - "p-cancelable@npm:^1.0.0": version: 1.1.0 resolution: "p-cancelable@npm:1.1.0" @@ -20627,24 +20249,6 @@ __metadata: languageName: node linkType: hard -"p-event@npm:^4.1.0": - version: 4.2.0 - resolution: "p-event@npm:4.2.0" - dependencies: - p-timeout: ^3.1.0 - checksum: 8a3588f7a816a20726a3262dfeee70a631e3997e4773d23219176333eda55cce9a76219e3d2b441b331eb746e14fdb381eb2694ab9ff2fcf87c846462696fe89 - languageName: node - linkType: hard - -"p-filter@npm:^2.1.0": - version: 2.1.0 - resolution: "p-filter@npm:2.1.0" - dependencies: - p-map: ^2.0.0 - checksum: 76e552ca624ce2233448d68b19eec9de42b695208121998f7e011edce71d1079a83096ee6a2078fb2a59cfa8a5c999f046edf00ebf16a8e780022010b4693234 - languageName: node - linkType: hard - "p-finally@npm:^1.0.0": version: 1.0.0 resolution: "p-finally@npm:1.0.0" @@ -20722,22 +20326,6 @@ __metadata: languageName: node linkType: hard -"p-map@npm:^2.0.0": - version: 2.1.0 - resolution: "p-map@npm:2.1.0" - checksum: 9e3ad3c9f6d75a5b5661bcad78c91f3a63849189737cd75e4f1225bf9ac205194e5c44aac2ef6f09562b1facdb9bd1425584d7ac375bfaa17b3f1a142dab936d - languageName: node - linkType: hard - -"p-map@npm:^3.0.0": - version: 3.0.0 - resolution: "p-map@npm:3.0.0" - dependencies: - aggregate-error: ^3.0.0 - checksum: 49b0fcbc66b1ef9cd379de1b4da07fa7a9f84b41509ea3f461c31903623aaba8a529d22f835e0d77c7cb9fcc16e4fae71e308fd40179aea514ba68f27032b5d5 - languageName: node - linkType: hard - "p-map@npm:^4.0.0": version: 4.0.0 resolution: "p-map@npm:4.0.0" @@ -20757,15 +20345,6 @@ __metadata: languageName: node linkType: hard -"p-timeout@npm:^3.1.0": - version: 3.2.0 - resolution: "p-timeout@npm:3.2.0" - dependencies: - p-finally: ^1.0.0 - checksum: 3dd0eaa048780a6f23e5855df3dd45c7beacff1f820476c1d0d1bcd6648e3298752ba2c877aa1c92f6453c7dd23faaf13d9f5149fc14c0598a142e2c5e8d649c - languageName: node - linkType: hard - "p-try@npm:^1.0.0": version: 1.0.0 resolution: "p-try@npm:1.0.0" @@ -20932,13 +20511,6 @@ __metadata: languageName: node linkType: hard -"parse-passwd@npm:^1.0.0": - version: 1.0.0 - resolution: "parse-passwd@npm:1.0.0" - checksum: 4e55e0231d58f828a41d0f1da2bf2ff7bcef8f4cb6146e69d16ce499190de58b06199e6bd9b17fbf0d4d8aef9052099cdf8c4f13a6294b1a522e8e958073066e - languageName: node - linkType: hard - "parse-path@npm:^4.0.4": version: 4.0.4 resolution: "parse-path@npm:4.0.4" @@ -22076,7 +21648,7 @@ __metadata: languageName: node linkType: hard -"prettier@npm:^2.6.2": +"prettier@npm:^2.7.1": version: 2.7.1 resolution: "prettier@npm:2.7.1" bin: @@ -22218,7 +21790,7 @@ __metadata: languageName: node linkType: hard -"prompts@npm:^2.0.1, prompts@npm:^2.4.0, prompts@npm:^2.4.1, prompts@npm:^2.4.2": +"prompts@npm:^2.0.1, prompts@npm:^2.4.0, prompts@npm:^2.4.1": version: 2.4.2 resolution: "prompts@npm:2.4.2" dependencies: @@ -22873,12 +22445,11 @@ __metadata: languageName: node linkType: hard -"react-i18next@npm:^11.15.3": - version: 11.17.3 - resolution: "react-i18next@npm:11.17.3" +"react-i18next@npm:^11.18.0": + version: 11.18.0 + resolution: "react-i18next@npm:11.18.0" dependencies: "@babel/runtime": ^7.14.5 - html-escaper: ^2.0.2 html-parse-stringify: ^3.0.1 peerDependencies: i18next: ">= 19.0.0" @@ -22888,7 +22459,7 @@ __metadata: optional: true react-native: optional: true - checksum: 6b4959eb6257c7096f70e6dbc1e436ccc19258c0a8202a28fcc812c098c3574f4a0f4ec8f9dd8c7cbdbee1ac055125ec918f99ea1e1655574248f1c1e06c612c + checksum: 63bf023c4918f533086a4b9a3a257cc8a30074d8a3943f4a3c613cef7875c53c622a1006f89dc778a9ddfdbca45c39187c4d5315efe71963d3a8327ae66d1279 languageName: node linkType: hard @@ -23010,13 +22581,13 @@ __metadata: languageName: node linkType: hard -"react-refresh-typescript@npm:^2.0.5": - version: 2.0.5 - resolution: "react-refresh-typescript@npm:2.0.5" +"react-refresh-typescript@npm:^2.0.7": + version: 2.0.7 + resolution: "react-refresh-typescript@npm:2.0.7" peerDependencies: - react-refresh: 0.10.x || 0.11.x || 0.12.x || 0.13.x + react-refresh: 0.10.x || 0.11.x || 0.12.x || 0.13.x || 0.14.x typescript: ^4 - checksum: 04b1765280a828e9c989c024fff71ce8290d2374c28bd05a063eed741b660b2820b5223694eec2e2aa5a78b3f8200891fbb6bd6ec17bc8a3d4e1a231c7edae35 + checksum: a95a44f7e72b7b1de001d400c004b3d1adf22ca72cd335efeda13a0a7e154e476cd113105fbbd7413122147ce4f7fff0e4308b0c19ea13947a3665a4314fd629 languageName: node linkType: hard @@ -23039,7 +22610,7 @@ __metadata: languageName: node linkType: hard -"react-router-dom@npm:^5.2.0, react-router-dom@npm:^5.3.0": +"react-router-dom@npm:^5.2.0, react-router-dom@npm:^5.3.3": version: 5.3.3 resolution: "react-router-dom@npm:5.3.3" dependencies: @@ -23056,7 +22627,7 @@ __metadata: languageName: node linkType: hard -"react-router@npm:5.3.3, react-router@npm:^5.2.0, react-router@npm:^5.2.1": +"react-router@npm:5.3.3, react-router@npm:^5.2.0, react-router@npm:^5.3.3": version: 5.3.3 resolution: "react-router@npm:5.3.3" dependencies: @@ -23193,7 +22764,7 @@ __metadata: languageName: node linkType: hard -"react-use-clipboard@npm:^1.0.7": +"react-use-clipboard@npm:^1.0.8": version: 1.0.8 resolution: "react-use-clipboard@npm:1.0.8" dependencies: @@ -23995,16 +23566,6 @@ __metadata: languageName: node linkType: hard -"resolve-dir@npm:^0.1.0": - version: 0.1.1 - resolution: "resolve-dir@npm:0.1.1" - dependencies: - expand-tilde: ^1.2.2 - global-modules: ^0.2.3 - checksum: cc3e1885938f8fe9656a6faa651e21730d369260e907b8dd7c847a4aa18db348ac08ee0dbf2d6f87e2ba08715fb109432ec773bbb31698381bd2a48c0ea66072 - languageName: node - linkType: hard - "resolve-from@npm:5.0.0, resolve-from@npm:^5.0.0": version: 5.0.0 resolution: "resolve-from@npm:5.0.0" @@ -24256,17 +23817,17 @@ __metadata: "@release-it/conventional-changelog": ^4.3.0 "@types/node": ^16.11.36 "@types/prettier": ^2.6.3 - eslint: ^8.16.0 + eslint: ^8.19.0 + eslint-plugin-n: ^15.2.4 husky: ^7.0.4 jest: ^27.5.1 - jest-dev-server: ^6.0.3 lint-staged: ^12.3.3 pm2: ^5.2.0 - prettier: ^2.6.2 + prettier: ^2.7.1 release-it: ^14.14.3 release-it-yarn-workspaces: ^2.0.1 ts-jest: ^27.1.5 - typescript: ^4.7.2 + typescript: ^4.7.4 webpack: ^5.73.0 zx: ^6.2.5 languageName: unknown @@ -24666,7 +24227,7 @@ __metadata: languageName: node linkType: hard -"semver@npm:7.3.7, semver@npm:7.x, semver@npm:^7.1.3, semver@npm:^7.2, semver@npm:^7.3.2, semver@npm:^7.3.4, semver@npm:^7.3.5, semver@npm:^7.3.7": +"semver@npm:7.3.7, semver@npm:7.x, semver@npm:^7.0.0, semver@npm:^7.1.3, semver@npm:^7.2, semver@npm:^7.3.2, semver@npm:^7.3.4, semver@npm:^7.3.5, semver@npm:^7.3.7": version: 7.3.7 resolution: "semver@npm:7.3.7" dependencies: @@ -24961,7 +24522,7 @@ __metadata: languageName: node linkType: hard -"signal-exit@npm:^3.0.0, signal-exit@npm:^3.0.2, signal-exit@npm:^3.0.3, signal-exit@npm:^3.0.6, signal-exit@npm:^3.0.7": +"signal-exit@npm:^3.0.0, signal-exit@npm:^3.0.2, signal-exit@npm:^3.0.3, signal-exit@npm:^3.0.7": version: 3.0.7 resolution: "signal-exit@npm:3.0.7" checksum: a2f098f247adc367dffc27845853e9959b9e88b01cb301658cfe4194352d8d2bb32e18467c786a7fe15f1d44b233ea35633d076d5e737870b7139949d1ab6318 @@ -25018,13 +24579,6 @@ __metadata: languageName: node linkType: hard -"slash@npm:^2.0.0": - version: 2.0.0 - resolution: "slash@npm:2.0.0" - checksum: 512d4350735375bd11647233cb0e2f93beca6f53441015eea241fe784d8068281c3987fbaa93e7ef1c38df68d9c60013045c92837423c69115297d6169aa85e6 - languageName: node - linkType: hard - "slash@npm:^3.0.0": version: 3.0.0 resolution: "slash@npm:3.0.0" @@ -25039,6 +24593,16 @@ __metadata: languageName: node linkType: hard +"sleep@npm:6.1.0": + version: 6.1.0 + resolution: "sleep@npm:6.1.0" + dependencies: + nan: ^2.13.2 + node-gyp: latest + checksum: 607465cdcfd630a80effcd44c887b29af7de061c5341ef99af064d6c08154d61bbb865626eb84868939cbcd668e9eb301a0ba320f5c2f5bc3f4b459492c6259e + languageName: node + linkType: hard + "slice-ansi@npm:^3.0.0": version: 3.0.0 resolution: "slice-ansi@npm:3.0.0" @@ -25283,17 +24847,6 @@ __metadata: languageName: node linkType: hard -"spawnd@npm:^6.0.2": - version: 6.0.2 - resolution: "spawnd@npm:6.0.2" - dependencies: - exit: ^0.1.2 - signal-exit: ^3.0.6 - tree-kill: ^1.2.2 - checksum: 39060a101e908c07497e8bc1c8335cb93d2708c4c1065824653d2e8de788437c590bd32a6fc3b03ddf7ff937642f3870442b778814adef8526fbca3723092bdb - languageName: node - linkType: hard - "spdx-correct@npm:^3.0.0": version: 3.1.1 resolution: "spdx-correct@npm:3.1.1" @@ -25922,9 +25475,9 @@ __metadata: languageName: node linkType: hard -"superagent@npm:^7.1.3": - version: 7.1.5 - resolution: "superagent@npm:7.1.5" +"superagent@npm:^7.1.6": + version: 7.1.6 + resolution: "superagent@npm:7.1.6" dependencies: component-emitter: ^1.3.0 cookiejar: ^2.1.3 @@ -25933,17 +25486,17 @@ __metadata: form-data: ^4.0.0 formidable: ^2.0.1 methods: ^1.1.2 - mime: ^2.5.0 + mime: 2.6.0 qs: ^6.10.3 readable-stream: ^3.6.0 semver: ^7.3.7 - checksum: 4782ca099b18a0933705f93ac4678290bcfcd5e57ca1286dd4f733c7fbb554ca0c7cc1c97261254dc29623ac258aa64fc40aaa18da960cbe759085efa3e4b8f6 + checksum: b73316836003219f1a4886a6d77dd28551a6784c30e871009fb7bad699fae772b20370d39d2ccb5a543c9335ce12b43a76b959a3ca983f1d6365cb4b5682c08f languageName: node linkType: hard -"superagent@npm:^7.1.6": - version: 7.1.6 - resolution: "superagent@npm:7.1.6" +"superagent@npm:^8.0.0": + version: 8.0.0 + resolution: "superagent@npm:8.0.0" dependencies: component-emitter: ^1.3.0 cookiejar: ^2.1.3 @@ -25956,17 +25509,17 @@ __metadata: qs: ^6.10.3 readable-stream: ^3.6.0 semver: ^7.3.7 - checksum: b73316836003219f1a4886a6d77dd28551a6784c30e871009fb7bad699fae772b20370d39d2ccb5a543c9335ce12b43a76b959a3ca983f1d6365cb4b5682c08f + checksum: 14343e59327eafd85fa230acb876017079d5efcecc72a56566abc0f965220bb460af2e070dddecd9e2856410b2d2b318d81d9cc1d342aa5922da93c29a295dd7 languageName: node linkType: hard -"supertest@npm:^6.2.2": - version: 6.2.3 - resolution: "supertest@npm:6.2.3" +"supertest@npm:^6.2.4": + version: 6.2.4 + resolution: "supertest@npm:6.2.4" dependencies: methods: ^1.1.2 - superagent: ^7.1.3 - checksum: c1bed86c31723a4bc461153a58176fd80d675deb7d23ab7bd170213040673b35c38e3cbeab9a4eb8a325cf736176c08c6f6522e42f0293314f183e192a6681fa + superagent: ^8.0.0 + checksum: f2ddc4f3ba467a5c4036dd4aad41351e4b60eb13c39ecf5233ccd2ebb425504073b2b7036c973a70c7047f5c6bc1b9fef096b7bbff114d357cbe80654441db23 languageName: node linkType: hard @@ -26709,7 +26262,7 @@ __metadata: languageName: node linkType: hard -"ts-loader@npm:^9.3.0": +"ts-loader@npm:^9.3.1": version: 9.3.1 resolution: "ts-loader@npm:9.3.1" dependencies: @@ -26751,7 +26304,7 @@ __metadata: languageName: node linkType: hard -"ts-node@npm:^10.4.0, ts-node@npm:^10.8.0, ts-node@npm:^10.8.1": +"ts-node@npm:^10.4.0, ts-node@npm:^10.8.1": version: 10.8.1 resolution: "ts-node@npm:10.8.1" dependencies: @@ -26789,6 +26342,44 @@ __metadata: languageName: node linkType: hard +"ts-node@npm:^10.8.2": + version: 10.8.2 + resolution: "ts-node@npm:10.8.2" + dependencies: + "@cspotcode/source-map-support": ^0.8.0 + "@tsconfig/node10": ^1.0.7 + "@tsconfig/node12": ^1.0.7 + "@tsconfig/node14": ^1.0.0 + "@tsconfig/node16": ^1.0.2 + acorn: ^8.4.1 + acorn-walk: ^8.1.1 + arg: ^4.1.0 + create-require: ^1.1.0 + diff: ^4.0.1 + make-error: ^1.1.1 + v8-compile-cache-lib: ^3.0.1 + yn: 3.1.1 + peerDependencies: + "@swc/core": ">=1.2.50" + "@swc/wasm": ">=1.2.50" + "@types/node": "*" + typescript: ">=2.7" + peerDependenciesMeta: + "@swc/core": + optional: true + "@swc/wasm": + optional: true + bin: + ts-node: dist/bin.js + ts-node-cwd: dist/bin-cwd.js + ts-node-esm: dist/bin-esm.js + ts-node-script: dist/bin-script.js + ts-node-transpile-only: dist/bin-transpile.js + ts-script: dist/bin-script-deprecated.js + checksum: 1eede939beed9f4db35bcc88d78ef803815b99dcdbed1ecac728d861d74dc694918a7f0f437aa08d026193743a31e7e00e2ee34f875f909b5879981c1808e2a7 + languageName: node + linkType: hard + "tsconfig-paths-webpack-plugin@npm:^3.5.2": version: 3.5.2 resolution: "tsconfig-paths-webpack-plugin@npm:3.5.2" @@ -27037,7 +26628,7 @@ __metadata: languageName: node linkType: hard -"typescript@npm:^4.4.3, typescript@npm:^4.7.2": +"typescript@npm:^4.4.3, typescript@npm:^4.7.4": version: 4.7.4 resolution: "typescript@npm:4.7.4" bin: @@ -27047,7 +26638,7 @@ __metadata: languageName: node linkType: hard -"typescript@patch:typescript@^4.4.3#~builtin, typescript@patch:typescript@^4.7.2#~builtin": +"typescript@patch:typescript@^4.4.3#~builtin, typescript@patch:typescript@^4.7.4#~builtin": version: 4.7.4 resolution: "typescript@patch:typescript@npm%3A4.7.4#~builtin::version=4.7.4&hash=493e53" bin: @@ -28101,7 +27692,7 @@ __metadata: languageName: node linkType: hard -"webpack-dev-server@npm:^4.5.0, webpack-dev-server@npm:^4.9.2": +"webpack-dev-server@npm:^4.5.0": version: 4.9.2 resolution: "webpack-dev-server@npm:4.9.2" dependencies: @@ -28145,6 +27736,50 @@ __metadata: languageName: node linkType: hard +"webpack-dev-server@npm:^4.9.3": + version: 4.9.3 + resolution: "webpack-dev-server@npm:4.9.3" + dependencies: + "@types/bonjour": ^3.5.9 + "@types/connect-history-api-fallback": ^1.3.5 + "@types/express": ^4.17.13 + "@types/serve-index": ^1.9.1 + "@types/serve-static": ^1.13.10 + "@types/sockjs": ^0.3.33 + "@types/ws": ^8.5.1 + ansi-html-community: ^0.0.8 + bonjour-service: ^1.0.11 + chokidar: ^3.5.3 + colorette: ^2.0.10 + compression: ^1.7.4 + connect-history-api-fallback: ^2.0.0 + default-gateway: ^6.0.3 + express: ^4.17.3 + graceful-fs: ^4.2.6 + html-entities: ^2.3.2 + http-proxy-middleware: ^2.0.3 + ipaddr.js: ^2.0.1 + open: ^8.0.9 + p-retry: ^4.5.0 + rimraf: ^3.0.2 + schema-utils: ^4.0.0 + selfsigned: ^2.0.1 + serve-index: ^1.9.1 + sockjs: ^0.3.24 + spdy: ^4.0.2 + webpack-dev-middleware: ^5.3.1 + ws: ^8.4.2 + peerDependencies: + webpack: ^4.37.0 || ^5.0.0 + peerDependenciesMeta: + webpack-cli: + optional: true + bin: + webpack-dev-server: bin/webpack-dev-server.js + checksum: 845f2cc8e79a348ee7b17080eef9b332c675540888e0bc97ec6b62174882aca7995eaa7a3f49cfdd9af186da22f2f335fd03cb3c55cd49e387c8a3dc59700d66 + languageName: node + linkType: hard + "webpack-merge@npm:^5.7.3, webpack-merge@npm:^5.8.0": version: 5.8.0 resolution: "webpack-merge@npm:5.8.0" @@ -28356,7 +27991,7 @@ __metadata: languageName: node linkType: hard -"which@npm:^1.2.12, which@npm:^1.2.9, which@npm:^1.3.1": +"which@npm:^1.2.9, which@npm:^1.3.1": version: 1.3.1 resolution: "which@npm:1.3.1" dependencies: @@ -28419,15 +28054,6 @@ __metadata: languageName: node linkType: hard -"window-size@npm:^0.1.4": - version: 0.1.4 - resolution: "window-size@npm:0.1.4" - bin: - window-size: cli.js - checksum: 409accca0b1373c69897400e3cc6a56a2acc8a6ba9009f0cd8e4adda4ebf308e50425d3bd375c0c08efb803c8f0b09d84d7266faa05422b3fadfe6ee422d0aef - languageName: node - linkType: hard - "windows-release@npm:^4.0.0": version: 4.0.0 resolution: "windows-release@npm:4.0.0" @@ -28460,16 +28086,6 @@ __metadata: languageName: node linkType: hard -"wrap-ansi@npm:^2.0.0": - version: 2.1.0 - resolution: "wrap-ansi@npm:2.1.0" - dependencies: - string-width: ^1.0.1 - strip-ansi: ^3.0.1 - checksum: 2dacd4b3636f7a53ee13d4d0fe7fa2ed9ad81e9967e17231924ea88a286ec4619a78288de8d41881ee483f4449ab2c0287cde8154ba1bd0126c10271101b2ee3 - languageName: node - linkType: hard - "wrap-ansi@npm:^6.2.0": version: 6.2.0 resolution: "wrap-ansi@npm:6.2.0" @@ -28691,10 +28307,15 @@ __metadata: languageName: node linkType: hard -"y18n@npm:^3.2.0": - version: 3.2.2 - resolution: "y18n@npm:3.2.2" - checksum: 6154fd7544f8bbf5b18cdf77692ed88d389be49c87238ecb4e0d6a5276446cd2a5c29cc4bdbdddfc7e4e498b08df9d7e38df4a1453cf75eecfead392246ea74a +"xvfb@npm:^0.4.0": + version: 0.4.0 + resolution: "xvfb@npm:0.4.0" + dependencies: + sleep: 6.1.0 + dependenciesMeta: + sleep: + optional: true + checksum: d8c446dd6e31a19067692b7ccd562758c855483b8e77275f95d77da136b5f94a1606b1b3f4e97fe85218f6a3ff8a39af05641c050c5396f4269f5914bd41eb65 languageName: node linkType: hard @@ -28842,21 +28463,6 @@ __metadata: languageName: node linkType: hard -"yargs@npm:^3.19.0": - version: 3.32.0 - resolution: "yargs@npm:3.32.0" - dependencies: - camelcase: ^2.0.1 - cliui: ^3.0.3 - decamelize: ^1.1.1 - os-locale: ^1.4.0 - string-width: ^1.0.1 - window-size: ^0.1.4 - y18n: ^3.2.0 - checksum: 3e0f7fc1bc2052bcaaa7354cbd33d05a86fc0f236432d107ecd088989fbd175174c562d17e762727acbf25d04e8520d43625f7581b2a6ce55ce10034e80675fc - languageName: node - linkType: hard - "yauzl@npm:2.9.2 - 2.10.0, yauzl@npm:^2.10.0": version: 2.10.0 resolution: "yauzl@npm:2.10.0"