diff --git a/package.json b/package.json index 851c90a..e71d237 100755 --- a/package.json +++ b/package.json @@ -24,18 +24,13 @@ ], "homepage": "https://github.com/EtherealEngine/Universal-Volumetric#readme", "dependencies": { - "@types/three": "0.153.0", - "commander": "^10.0.0", - "draco3d": "^1.5.6", - "glob": "7.2.0", - "ngrok": "^5.0.0-beta.2", - "three": "0.153.0", - "xmlhttprequest": "^1.8.0" + "many-keys-map": "^2.0.1" }, "devDependencies": { "@trivago/prettier-plugin-sort-imports": "^4.1.1", - "@types/draco3d": "^1.4.2", + "@types/three": "^0.155.0", "prettier": "^2.8.7", + "three": "^0.155.0", "ts-loader": "^9.4.2", "ts-node": "^10.9.1", "typescript": "^5.0.3", diff --git a/src/Interfaces.ts b/src/Interfaces.ts index 51b0d07..b969fc4 100755 --- a/src/Interfaces.ts +++ b/src/Interfaces.ts @@ -1,3 +1,20 @@ +import { + RGBA_ASTC_4x4_Format, + RGBA_ASTC_5x4_Format, + RGBA_ASTC_5x5_Format, + RGBA_ASTC_6x5_Format, + RGBA_ASTC_6x6_Format, + RGBA_ASTC_8x5_Format, + RGBA_ASTC_8x6_Format, + RGBA_ASTC_8x8_Format, + RGBA_ASTC_10x5_Format, + RGBA_ASTC_10x6_Format, + RGBA_ASTC_10x8_Format, + RGBA_ASTC_10x10_Format, + RGBA_ASTC_12x10_Format, + RGBA_ASTC_12x12_Format +} from 'three' + export interface V1FrameData { frameNumber: number keyframeNumber: number @@ -14,121 +31,292 @@ export interface V1Schema { frameRate: number } -export type AudioFileFormat = 'mp3' -export type GeometryFileFormat = 'obj' | 'draco' -export type TextureFileFormat = 'mp4' | 'ktx2' | 'etc2' // | 'astc.ktx' | 'etc1.ktx' +export interface AudioInput { + /** + * Path to audio an audio file. + */ + path: string +} -export interface GeometryTarget { +export interface GeometryInput { /** - * The frame rate to encode the geometry data at. + * Path to geometry data. This can be a plain file path, or a file path with an index substitution pattern. + * + * Supported formats: + * Alembic - should be specified as a plain file path, eg: input/geometry.abc + * OBJ - should be specified with an index pattern, eg: input/frame_[0001-1000].obj + * + * When referencing indexed files, the index should be specified as a range, eg: frame_[00001-10000].obj + * If the first frame is 0, the index should be specified with all zeros, eg: frame_[00000-10000].obj + * Indexed file names should be 0-padded to the same number of digits, eg: frame_00001.obj, frame_00002.obj, etc. + */ + path: string + /** + * Frame rate of the geometry data. This is only required for OBJ files. */ - "frameRate": number, + frameRate: number +} +export interface TextureInput { /** - * Total Geometry frames + * Path to texture data. This can be a plain file path, or a file path with an index substitution pattern. + * + * Supported formats: + * PNG - should be specified as with an index pattern, eg: input/baseColor/frame_[00001-10000].png + * JPEG - should be specified as with an index pattern, eg: input/baseColor/frame_[00001-10000].jpg + * MP4 - should be specified as a single file, eg: input/baseColor.mp4 + * + * When referencing indexed files, the index should be specified as a range, eg: frame_[00001-10000].png + * If the first frame is 0, the index should be specified with all zeros, eg: frame_[00000-10000].png + * Indexed file names should be 0-padded to the same number of digits, eg: frame_00001.png, frame_00002.png, etc. + * + * If the path is a single file, the frame number should be omitted, eg: baseColor.mp4 */ - "frameCount": number, + path: string + /** + * Frame rate of the texture data. When using indexed files, each file is assumed to be a single frame. + */ + frameRate: number + /** + * A tag to identify this texture input. + * + * Default: "default" + */ + tag?: string +} + +export type AudioFileFormat = 'mp3' | 'wav' +export type GeometryFormat = 'draco' | 'glb' +export type TextureFormat = 'ktx2' | 'astc/ktx' +export type OptionalTextureType = 'normal' | 'metallicRoughness' | 'emissive' | 'occlusion' +export type TextureType = 'baseColor' | OptionalTextureType +export interface DracoEncodeOptions { + /** + * Draco compression level. [0-10], most=10, least=0, default=0. + */ + compressionLevel?: number + /** + * The number of bits to quantize the position attribute. Default=11. + */ + positionQuantizationBits?: number + /** + * The number of bits to quantize the texture coordinate attribute. Default=10. + */ + textureQuantizationBits?: number + /** + * The number of bits to quantize the normal vector attribute. Default=8. + */ + normalQuantizationBits?: number + /** + * The number of bits to quantize any generic attribute. Default=8. + */ + genericQuantizationBits?: number +} + +export interface GLBEncodeOptions { + /** + * simplify meshes targeting triangle count ratio R (default: 1; R should be between 0 and 1) + * @default 1 + */ + simplificationRatio?: number + /** + * aggressively simplify to the target ratio disregarding quality + * @default false + */ + simplifyAggressively?: boolean +} +export interface GeometryTarget { /** * Geometry encoding format. - * */ - "format": GeometryFileFormat + format: GeometryFormat + /** + * The frame rate to encode the geometry data at. + */ + frameRate: number + /** + * Total frame count. This information is supplied by the encoder. + */ + frameCount: number + /** + * Scale of the model. + * This is read by the player, actual geometry data is not scaled. + * @default 1 + */ + scale: number } -export type TextureType = "baseColor" | "normal" | "metallicRoughness" | "emissive" | "occlusion" +export interface DRACOTarget extends GeometryTarget { + format: 'draco' + /** + * Draco encoding options for the geometry data. + */ + settings: DracoEncodeOptions +} -export interface TextureTarget { +export interface GLBTarget extends GeometryTarget { + format: 'glb' /** - * Texture encoding format. + * GLB encoding options for the geometry data. + * @default {simplificationRatio: 1, simplifyAggressively: false} */ - "format": TextureFileFormat + settings: GLBEncodeOptions +} + +export interface TextureTarget { /** - * Resolution to encode the texture data at. + * Texture encoding format. */ - "resolution": [number, number], + format: TextureFormat /** - * Type + * The frame rate to encode the geometry data at. */ - "type": TextureType|TextureType[] + frameRate: number /** - * + * Total frame count. This information is supplied by the encoder. */ - "tag"?: string + frameCount: number } -export interface KTX2TextureTarget extends TextureTarget { +export interface KTX2EncodeOptions { + /** + * The compression_level parameter controls the encoder perf vs. file size tradeoff for ETC1S files + * It does not directly control file size vs. quality - see qualityLevel + * Range is [0, 5] + * @default 1 + */ + compressionLevel?: number /** - * The number of frames to encode in each KTX2 file + * Sets the ETC1S encoder's quality level, which controls the file size vs. quality tradeoff + * Range is [1, 255] + * @default 128 */ - "sequenceSize": number, + qualityLevel?: number /** - * Total number of sequences + * Resize images to @e width X @e height. + * If not specified, uses the image as is. */ - "sequenceCount": number, + resolution: { + width: number + height: number + } + /** - * The frame rate to encode the texture data at. + * Vertically flip images */ - "frameRate": number, + lower_left_maps_to_s0t0?: boolean +} +export interface KTX2TextureTarget extends TextureTarget { + format: 'ktx2' + settings: KTX2EncodeOptions +} + +export interface ASTCEncodeOptions { + blocksize: + | '4x4' + | '5x4' + | '5x5' + | '6x5' + | '6x6' + | '8x5' + | '8x6' + | '10x5' + | '10x6' + | '8x8' + | '10x8' + | '10x10' + | '12x10' + | '12x12' + quality: '-fastest' | '-fast' | '-medium' | '-thorough' | '-verythorough' | '-exhaustive' | number + yflip?: boolean + resolution: { + width: number + height: number + } +} + +export interface ASTCTextureTarget extends TextureTarget { + format: 'astc/ktx' + settings: ASTCEncodeOptions } export interface V2Schema { - "version": "v2", - "audio": { + version: 'v2' + input: { + audio?: AudioInput + geometry: GeometryInput | GeometryInput[] + texture: { + baseColor: TextureInput | TextureInput[] + normal?: TextureInput | TextureInput[] + metallicRoughness?: TextureInput | TextureInput[] + emissive?: TextureInput | TextureInput[] + occlusion?: TextureInput | TextureInput[] + } + } + audio?: { /** * Path template to the output audio data. - * + * * The following template substitutions are supported: - * + * * [ext] - the file extension of the texture, (e.g., ".mp3", ".wav", etc.) - * + * * E.g. "output/audio[ext]" */ - "path": string, + path: string /** * The audio encoding format. - * + * * The following options are supported: - * "mp3" - MP3 audio + * "mp3", "wav" - MP3 audio */ - "format": AudioFileFormat|AudioFileFormat[] - }, - "geometry": { + formats: AudioFileFormat[] + /** + * PlayBack rate. + * This is read by the player, actual playbackRate data is not changed. + * @default 1 + */ + playbackRate: number + } + geometry: { /** * Encoding targets for the geometry data. */ - "targets": Record + targets: Record /** * Path template to the output geometry data. - * + * * The following template substitutions are supported: * [target] - one of the geometry targets, defined in the "targets" section - * [######] - 0-padded index for each file with the same extension, e.g., ("000001.drc", "000002.drc", etc.) + * [index] - the index of the frame * [ext] - the file extension of the data - * - * E.g. "output/geometry_[target]/[######][ext]" - */ - "path": string, - }, - "texture": { - /** - * Encoding targets for the texture data. + * + * E.g. "output/geometry_[target]/[index][ext]" */ - "targets": Record, + path: string + } + texture: { + baseColor: { + targets: Record + } /** * Path template to the output texture data. - * + * * The following template substitutions are supported: * [target] - one of the texture targets, defined in the "targets" section - * [type] - the type of texture, eg: "baseColor", "metallicRoughness", "normal" * [tag] - a custom tag for describing texture variants, eg: "default", "red_dress", "blue_dress", etc. - * [######] - 0-padded index for each file with the same extension, e.g., ("000001", "000002", etc.) + * [index] - 0-padded index for each file with the same extension, e.g., ("000001", "000002", etc.) * [ext] - the file extension of the texture, (e.g., ".mp4", ".ktx2", ".astc.ktx", etc.) - * - * E.g. "output/texture_[target]_[type]_[tag]/[######][ext]"" + * + * E.g. "output/texture_[target]_[type]_[tag]/[index][ext]"" */ - "path": string, - } + path: string + } & Partial<{ + [key in OptionalTextureType]: { + targets: Record + } + }> } export type UVOLManifestSchema = V1Schema | V2Schema @@ -154,16 +342,38 @@ export enum PlayMode { } export const FORMATS_TO_EXT = { - 'mp3': '.mp3', - 'draco': '.drc', - 'ktx2': '.ktx2', - 'etc2': '.etc2' + mp3: '.mp3', + wav: '.wav', + draco: '.drc', + glb: '.glb', + ktx2: '.ktx2', + 'astc/ktx': '.ktx' } +export const GEOMETRY_FORMAT_PRIORITY = { + draco: 0, + glb: 1 +} // more value => more priority export const TEXTURE_FORMAT_PRIORITY = { - 'ktx2': 0, - 'etc2': 1, // if etc2 is supported - 'etc1': 2, // if etc1 is supported -} \ No newline at end of file + ktx2: 0, + 'astc/ktx': 1 // if astc is supported +} + +export const ASTC_BLOCK_SIZE_TO_FORMAT = { + '4x4': RGBA_ASTC_4x4_Format, + '5x4': RGBA_ASTC_5x4_Format, + '5x5': RGBA_ASTC_5x5_Format, + '6x5': RGBA_ASTC_6x5_Format, + '6x6': RGBA_ASTC_6x6_Format, + '8x5': RGBA_ASTC_8x5_Format, + '8x6': RGBA_ASTC_8x6_Format, + '10x5': RGBA_ASTC_10x5_Format, + '10x6': RGBA_ASTC_10x6_Format, + '8x8': RGBA_ASTC_8x8_Format, + '10x8': RGBA_ASTC_10x8_Format, + '10x10': RGBA_ASTC_10x10_Format, + '12x10': RGBA_ASTC_12x10_Format, + '12x12': RGBA_ASTC_12x12_Format +} diff --git a/src/V2/player.ts b/src/V2/player.ts index 16d072b..e4472ee 100644 --- a/src/V2/player.ts +++ b/src/V2/player.ts @@ -1,30 +1,48 @@ +import ManyKeysMap from 'many-keys-map' import { + Box3, BufferGeometry, Color, - CompressedArrayTexture, - CompressedTexture, - GLSL3, + CompressedPixelFormat, Material, Mesh, MeshBasicMaterial, - RGB_ETC2_Format, - ShaderChunk, - ShaderMaterial, - UnsignedByteType, + Sphere, + Texture, + Vector3, WebGLRenderer } from 'three' +import { MeshoptDecoder } from 'three/examples/jsm/libs/meshopt_decoder.module.js' +import { DRACOLoader } from 'three/examples/jsm/loaders/DRACOLoader' +import { GLTFLoader } from 'three/examples/jsm/loaders/GLTFLoader' +import { KTX2Loader } from 'three/examples/jsm/loaders/KTX2Loader' +import { KTXLoader } from 'three/examples/jsm/loaders/KTXLoader' -import { onFrameShowCallback, onMeshBufferingCallback, onTrackEndCallback, V2Schema } from '../Interfaces' import { - FORMATS_TO_EXT, - GeometryTarget, - KTX2TextureTarget, - TEXTURE_FORMAT_PRIORITY, - TextureFileFormat + ASTC_BLOCK_SIZE_TO_FORMAT, + GEOMETRY_FORMAT_PRIORITY, + GLBEncodeOptions, + onFrameShowCallback, + onMeshBufferingCallback, + onTrackEndCallback, + TextureType, + V2Schema } from '../Interfaces' -import { DRACOLoader } from '../lib/DRACOLoader' -import { KTX2Loader } from '../lib/KTX2Loader' -import { countHashChar, getAbsoluteURL, isTextureFormatSupported, pad } from '../utils' +import { FORMATS_TO_EXT, TEXTURE_FORMAT_PRIORITY } from '../Interfaces' +import { + GeometryFrameCount as _GeometryFrameCount, + TextureFrameCount as _TextureFrameCount, + calculateGeometryFrame, + calculateTextureFrame, + decodeGeometry, + decodeTexture, + getAbsoluteURL, + getGeometryURL, + getTextureURL, + isTextureFormatSupported, + updateGeometry, + updateTexture +} from './utils' export interface fetchBuffersCallback { (): void @@ -39,208 +57,311 @@ export type PlayerConstructorArgs = { audio?: HTMLAudioElement | HTMLVideoElement // both