diff --git a/.gitignore b/.gitignore index c3c12e51f..bd6d8db76 100644 --- a/.gitignore +++ b/.gitignore @@ -4,10 +4,16 @@ *.swp *__pycache__ *.egg-info -dist +*dist* build tags +# Agents +*.npy +*.parquet +*.db +wandb + # test-related .coverage .cache @@ -26,3 +32,7 @@ examples/training_logs # virtual environment venv + +# Credentials +keys.txt* + diff --git a/.travis.yml b/.travis.yml index 4d61d2b96..1dc5b1f05 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,3 +33,4 @@ notifications: on_success: change on_failure: always on_start: never + diff --git a/Dockerfile b/Dockerfile index bc71c6b9e..934419b4a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM tensorflow/tensorflow:2.7.0-gpu +FROM tensorflow/tensorflow:2.8.0-gpu ENV NVIDIA_VISIBLE_DEVICES \ ${NVIDIA_VISIBLE_DEVICES:-all} @@ -16,8 +16,10 @@ COPY . ./ SHELL ["/bin/bash", "-c"] RUN apt-get update && \ - apt-get install -yq --assume-yes --no-install-recommends git \ + apt-get install -yq --assume-yes --no-install-recommends cmake \ + git \ libgl1-mesa-glx \ + libopenmpi-dev \ python3-pip \ python3-dev \ python3-setuptools \ @@ -32,10 +34,8 @@ RUN apt-get update && \ make install && \ cd .. && \ pip3 install --no-cache-dir --upgrade pip && \ - pip3 install --no-cache-dir -r "./examples/requirements.txt" && \ + pip3 install --no-cache-dir -e . && \ + pip3 install --no-cache-dir -e ".[full]" && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* ta-lib-0.4.0-src.tar.gz -# Faster compilation for tests -RUN pip3 install --no-cache-dir -e ".[docs,tests]" - diff --git a/MANIFEST.in b/MANIFEST.in index 93ec6bca6..0e6a84598 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,4 +3,4 @@ include README.md include CONTRIBUTING.md graft docs graft examples -graft tests \ No newline at end of file +graft tests diff --git a/Makefile b/Makefile index 31ebefc36..0fa7b4196 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ -GPU_IMAGE?="tensortrade:latest-gpu" -CPU_IMAGE?="tensortrade:latest" +GPU_IMAGE?="tensortrade:2.8.0-gpu" +CPU_IMAGE?="tensortrade:2.8.0" SHM_SIZE?="3.0gb" # TODO: Automate me! clean: @@ -39,7 +39,7 @@ build-gpu-if-not-built: if [ ! $$(docker images -q ${GPU_IMAGE}) ]; then $(MAKE) build-gpu; fi; run-notebook: build-cpu-if-not-built - docker run -it --rm -p=8888:8888 -p=6006:6006 -v ${PWD}/examples:/examples --shm-size=${SHM_SIZE} ${CPU_IMAGE} jupyter notebook --ip='*' --port=8888 --no-browser --allow-root ./examples/ + docker run -it --rm -p=8888:8888 -p=6006:6006 -v ${PWD}:/app --shm-size=${SHM_SIZE} ${CPU_IMAGE} jupyter notebook --ip='*' --port=8888 --no-browser --allow-root /app run-docs: build-cpu-if-not-built if [ $$(docker ps -aq --filter name=tensortrade_docs) ]; then docker rm $$(docker ps -aq --filter name=tensortrade_docs); fi; @@ -50,7 +50,7 @@ run-tests: build-cpu-if-not-built docker run -it --rm --shm-size=${SHM_SIZE} ${CPU_IMAGE} make test run-notebook-gpu: build-gpu-if-not-built - docker run -it --rm -p=8888:8888 -p=6006:6006 -v ${PWD}/examples:/examples --shm-size=${SHM_SIZE} ${GPU_IMAGE} jupyter notebook --ip='*' --port=8888 --no-browser --allow-root /examples/ + docker run -it --rm -p=8888:8888 -p=6006:6006 -v ${PWD}:/app --shm-size=${SHM_SIZE} ${GPU_IMAGE} jupyter notebook --ip='*' --port=8888 --no-browser --allow-root /app run-docs-gpu: build-gpu-if-not-built if [ $$(docker ps -aq --filter name=tensortrade_docs) ]; then docker rm $$(docker ps -aq --filter name=tensortrade_docs); fi; diff --git a/examples/.gitignore b/examples/.gitignore index 29a7166e8..f53ae5a0c 100644 --- a/examples/.gitignore +++ b/examples/.gitignore @@ -1,2 +1,3 @@ *.csv - +*.zip +logs/ diff --git a/examples/requirements.txt b/examples/requirements.txt index 2d8684301..b79de1e57 100644 --- a/examples/requirements.txt +++ b/examples/requirements.txt @@ -1,13 +1,9 @@ -r ../requirements.txt -ray[default,tune,rllib,serve]==1.9.2 -ccxt>=1.72.37 -jupyterlab>=1.1.4 +.[notebook] + +stochastic>=0.6.0 +ccxt==1.72.37 feature_engine scikit-learn optuna -quantstats -ta>=0.4.7 -TA-Lib -git+https://github.com/twopirllc/pandas-ta@development - diff --git a/requirements.txt b/requirements.txt index 74bddf9a7..c98aa16d4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,10 +2,6 @@ numpy>=1.17.0 pandas>=0.25.0 gym>=0.15.7 pyyaml>=5.1.2 -stochastic>=0.6.0 -tensorflow>=2.7.0 -ipython>=7.12.0 matplotlib>=3.1.1 plotly>=4.5.0 -ipywidgets>=7.0.0 # Required for rendering deprecated>=1.2.13 diff --git a/requirements_extras.txt b/requirements_extras.txt new file mode 100644 index 000000000..06146e113 --- /dev/null +++ b/requirements_extras.txt @@ -0,0 +1,51 @@ +-r ./requirements.txt + +# [ray] +ray[default,tune,rllib,serve]==1.10.0 + +# [stable_baselines] +stable-baselines3[extra] +sb3-contrib + +# [tensorflow] +tensorflow>=2.8.0 + +# [pytorch] +torch>=1.10.2 + +# [indicators] +quantstats +TA-Lib +pandas-ta @ git+https://github.com/twopirllc/pandas-ta.git@development +ta>=0.4.7 + +# [agents] +pyarrow +wandb +tensorflow-probability>=0.15.0 +opencv-python + +# [binance] +python-binance +binance-futures @ git+https://github.com/Binance-docs/Binance_Futures_python.git#egg=binance-futures + +# [notebook] +jupyterlab>=1.1.4 +ipython>=7.12.0 +ipywidgets>=7.0.0 + +# [tests] +pytest>=5.1.1 +tensorflow>=2.8.0 +ta>=0.4.7 +ipython>=7.12.0 + +# [docs] +sphinx +sphinx_rtd_theme +sphinxcontrib.apidoc +nbsphinx +nbsphinx_link +recommonmark +sphinx_markdown_tables +ipykernel diff --git a/setup.py b/setup.py index afec24d9a..16bc83240 100644 --- a/setup.py +++ b/setup.py @@ -19,17 +19,78 @@ from setuptools import find_packages, setup +def split(sequence, sep): + chunk = [] + for val in sequence: + if val == sep: + yield chunk + chunk = [] + else: + chunk.append(val) + yield chunk + + if sys.version_info.major != 3: raise NotImplementedError("TensorTrade is only compatible with Python 3.") tensortrade_directory = os.path.abspath(os.path.dirname(__file__)) -with open(os.path.join(tensortrade_directory, 'tensortrade', 'version.py'), 'r') as filehandle: +with open(os.path.join(tensortrade_directory, 'tensortrade', 'version.py'), mode='r', encoding='utf-8') as filehandle: for line in filehandle: if line.startswith('__version__'): version = line[15:-2] + +with open(os.path.join(tensortrade_directory, 'requirements.txt'), mode='r', encoding='utf-8') as f: + install_requires = f.read().splitlines() + +extras_require = {} + +with open(os.path.join(tensortrade_directory, 'examples', 'requirements.txt'), mode='r', encoding='utf-8') as f: + extras_require['examples'] = f.read().splitlines()[4:] + +with open(os.path.join(tensortrade_directory, 'requirements_extras.txt'), mode='r', encoding='utf-8') as f: + for extra in list(split(f.read().splitlines(), ''))[1:]: + extras_require[extra[0][3:-1]] = extra[1:] + +extras_require['examples'] += extras_require['notebook'] + +extras_require['full'] = \ + extras_require['ray'] + \ + extras_require['stable_baselines'] + \ + extras_require['tensorflow'] + \ + extras_require['pytorch'] + \ + extras_require['indicators'] + \ + extras_require['agents'] + \ + extras_require['binance'] + \ + extras_require['examples'] + \ + extras_require['tests'] + \ + extras_require['docs'] + + +classifiers = [ + 'Development Status :: 5 - Production/Stable', + 'Natural Language :: English', + 'Intended Audience :: Developers', + 'Intended Audience :: Education', + 'Intended Audience :: Science/Research', + 'Intended Audience :: Financial and Insurance Industry', + 'Intended Audience :: Information Technology', + 'License :: OSI Approved :: Apache Software License', + 'Operating System :: OS Independent', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Topic :: Office/Business :: Financial :: Investment', + 'Topic :: Office/Business :: Financial', + 'Topic :: Scientific/Engineering :: Information Analysis', + 'Topic :: System :: Distributed Computing', + 'Topic :: Scientific/Engineering :: Artificial Intelligence', + 'Topic :: Software Development :: Libraries', + 'Topic :: Software Development :: Libraries :: Python Modules', +] + setup( name='tensortrade', version=version, @@ -45,54 +106,15 @@ ], license='Apache 2.0', python_requires='>=3.7', - install_requires=[ - 'numpy>=1.17.0', - 'pandas>=0.25.0', - 'gym>=0.15.7', - 'pyyaml>=5.1.2', - 'stochastic>=0.6.0', - 'tensorflow>=2.7.0', - 'ipython>=7.12.0', - 'matplotlib>=3.1.1', - 'plotly>=4.5.0', - 'deprecated>=1.2.13' - ], - extras_require={ - 'tests': [ - 'pytest>=5.1.1', - 'ta>=0.4.7' - ], - 'docs': [ - 'sphinx', - 'sphinx_rtd_theme', - 'sphinxcontrib.apidoc', - 'nbsphinx', - 'nbsphinx_link', - 'recommonmark', - 'sphinx_markdown_tables', - 'ipykernel' + install_package_data=True, + install_requires=install_requires, + extras_require=extras_require, + entry_points={ + 'console_scripts': [ + 'tensortrade=tensortrade.agents.cli:execute', ], }, - classifiers=[ - 'Development Status :: 5 - Production/Stable', - 'Natural Language :: English', - 'Intended Audience :: Developers', - 'Intended Audience :: Education', - 'Intended Audience :: Science/Research', - 'Intended Audience :: Financial and Insurance Industry', - 'Intended Audience :: Information Technology', - 'License :: OSI Approved :: Apache Software License', - 'Operating System :: OS Independent', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Topic :: Office/Business :: Financial :: Investment', - 'Topic :: Office/Business :: Financial', - 'Topic :: Scientific/Engineering :: Information Analysis', - 'Topic :: System :: Distributed Computing', - 'Topic :: Scientific/Engineering :: Artificial Intelligence', - 'Topic :: Software Development :: Libraries', - 'Topic :: Software Development :: Libraries :: Python Modules', - ], + classifiers=classifiers, zip_safe=False ) + diff --git a/tensortrade/contrib/__init__.py b/tensortrade/contrib/__init__.py deleted file mode 100644 index e69de29bb..000000000