diff --git a/Makefile b/Makefile index 4be08de4..c040c158 100644 --- a/Makefile +++ b/Makefile @@ -66,7 +66,7 @@ double-env: # run all e2e tests e2e-test: - ./scripts/wait_for_server_ready.sh 12 # 12 services should have log, 2 gateways has no log + ./scripts/wait_for_server_ready.sh 6 # 12 services should have log, 2 gateways has no log cat double.vars source double.vars && PIPENV_PIPFILE=backend/Pipfile pipenv run pytest \ backend/tests/e2e_tests -k "$(T)" -W ignore::DeprecationWarning diff --git a/backend/Pipfile b/backend/Pipfile index a05d72e2..c420fed7 100644 --- a/backend/Pipfile +++ b/backend/Pipfile @@ -7,7 +7,6 @@ url = "https://pypi.org/simple" verify_ssl = true [dev-packages] -fakeredis = "*" black = "*" pyre-check = "*" pyre-extensions = "*" @@ -17,9 +16,6 @@ ipython = "*" [packages] diem-sample-wallet = {editable = true,path = "."} -dramatiq = {extras = ["redis", "watch"],version = "*"} -flask-redis = "*" -redis = "*" Flask = ">=1.1.1" Flask-WTF = "*" WTForms = "*" diff --git a/backend/Pipfile.lock b/backend/Pipfile.lock index c29d9346..8526581c 100644 --- a/backend/Pipfile.lock +++ b/backend/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "3a58f24adcc77b417de050a74786b9f63f4a5fe34c1e9e213dd747727a582f4d" + "sha256": "c2a820ce11198560dbb2aa72dc1158341cc4ed1e6c4cfb821a19f450625058fd" }, "pipfile-spec": 6, "requires": { @@ -74,13 +74,6 @@ ], "version": "==1.4.4" }, - "argh": { - "hashes": [ - "sha256:a9b3aaa1904eeb78e32394cd46c6f37ac0fb4af6dc488daa58971bdc7d7fcaf3", - "sha256:e9535b8c84dc9571a48999094fda7f33e63c3f1b74f3e5f3ac0105a58405bb65" - ], - "version": "==0.26.2" - }, "astroid": { "hashes": [ "sha256:38b95085e9d92e2ca06cf8b35c12a74fa81da395a6f9e65803742e6509c05892", @@ -243,18 +236,6 @@ ], "version": "==0.3.2" }, - "dramatiq": { - "extras": [ - "redis", - "watch" - ], - "hashes": [ - "sha256:bb209d0d54e36e7c3f31862ca3f088752d9a3799a5a46d7979d6891326de5a14", - "sha256:dea14f45d3fba339d5e6aadfe67f6b403b0c9464e2d2210a79fcf54a6ae763a8" - ], - "index": "pypi", - "version": "==1.11.0" - }, "entrypoints": { "hashes": [ "sha256:589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19", @@ -294,14 +275,6 @@ "index": "pypi", "version": "==2.0.1" }, - "flask-redis": { - "hashes": [ - "sha256:8d79eef4eb1217095edab603acc52f935b983ae4b7655ee7c82c0dfd87315d17", - "sha256:e1fccc11e7ea35c2a4d68c0b9aa58226a098e45e834d615c7b6c4928b01ddd6c" - ], - "index": "pypi", - "version": "==0.4.0" - }, "flask-wtf": { "hashes": [ "sha256:6ff7af73458f182180906a37a783e290bdc8a3817fe4ad17227563137ca285bf", @@ -310,40 +283,6 @@ "index": "pypi", "version": "==0.15.1" }, - "gevent": { - "hashes": [ - "sha256:16574e4aa902ebc7bad564e25aa9740a82620fdeb61e0bbf5cbc32e84c13cb6a", - "sha256:188c3c6da67e17ffa28f960fc80f8b7e4ba0f4efdc7519822c9d3a1784ca78ea", - "sha256:1e5af63e452cc1758924528a2ba6d3e472f5338e1534b7233cd01d3429fc1082", - "sha256:242e32cc011ad7127525ca9181aef3379ce4ad9c733aefe311ecf90248ad9a6f", - "sha256:2a9ae0a0fd956cbbc9c326b8f290dcad2b58acfb2e2732855fe1155fb110a04d", - "sha256:33741e3cd51b90483b14f73b6a3b32b779acf965aeb91d22770c0c8e0c937b73", - "sha256:3694f393ab08372bd337b9bc8eebef3ccab3c1623ef94536762a1eee68821449", - "sha256:464ec84001ba5108a9022aded4c5e69ea4d13ef11a2386d3ec37c1d08f3074c9", - "sha256:520cc2a029a9eef436e4e56b007af7859315cafa21937d43c1d5269f12f2c981", - "sha256:77b65a68c83e1c680f52dc39d5e5406763dd10a18ce08420665504b6f047962e", - "sha256:7bdfee07be5eee4f687bf90c54c2a65c909bcf2b6c4878faee51218ffa5d5d3e", - "sha256:969743debf89d6409423aaeae978437cc042247f91f5801e946a07a0a3b59148", - "sha256:96f704561a9dd9a817c67f2e279e23bfad6166cf95d63d35c501317e17f68bcf", - "sha256:9f99c3ec61daed54dc074fbcf1a86bcf795b9dfac2f6d4cdae6dfdb8a9125692", - "sha256:a130a1885603eabd8cea11b3e1c3c7333d4341b537eca7f0c4794cb5c7120db1", - "sha256:a54b9c7516c211045d7897a73a4ccdc116b3720c9ad3c591ef9592b735202a3b", - "sha256:ac98570649d9c276e39501a1d1cbf6c652b78f57a0eb1445c5ff25ff80336b63", - "sha256:afaeda9a7e8e93d0d86bf1d65affe912366294913fe43f0d107145dc32cd9545", - "sha256:b6ffc1131e017aafa70d7ec19cc24010b19daa2f11d5dc2dc191a79c3c9ea147", - "sha256:ba0c6ad94614e9af4240affbe1b4839c54da5a0a7e60806c6f7f69c1a7f5426e", - "sha256:bdb3677e77ab4ebf20c4752ac49f3b1e47445678dd69f82f9905362c68196456", - "sha256:c2c4326bb507754ef354635c05f560a217c171d80f26ca65bea81aa59b1ac179", - "sha256:cfb2878c2ecf27baea436bb9c4d8ab8c2fa7763c3916386d5602992b6a056ff3", - "sha256:e370e0a861db6f63c75e74b6ee56a40f5cdac90212ec404621445afa12bfc94b", - "sha256:e8a5d9fcf5d031f2e4c499f5f4b53262face416e22e8769078354f641255a663", - "sha256:ecff28416c99e0f73137f35849c3027cc3edde9dc13b7707825ebbf728623928", - "sha256:f0498df97a303da77e180a9368c9228b0fc94d10dd2ce79fc5ebb63fec0d2fc9", - "sha256:f91fd07b9cf642f24e58ed381e19ec33e28b8eee8726c19b026ea24fcc9ff897" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", - "version": "==21.1.2" - }, "greenlet": { "hashes": [ "sha256:03f28a5ea20201e70ab70518d151116ce939b412961c33827519ce620957d44c", @@ -396,7 +335,7 @@ "sha256:f42ad188466d946f1b3afc0a9e1a266ac8926461ee0786c06baac6bd71f8a6f3", "sha256:f92731609d6625e1cc26ff5757db4d32b6b810d2a3363b0ff94ff573e5901f6f" ], - "markers": "platform_python_implementation == 'CPython' and python_version >= '3'", + "markers": "python_version >= '3'", "version": "==1.1.0" }, "idna": { @@ -667,12 +606,6 @@ "markers": "python_version >= '3.6'", "version": "==21.0" }, - "pathtools": { - "hashes": [ - "sha256:7c35c5421a39bb82e58018febd90e3b6e5db34c5443aaaf742b3f33d4655f1c0" - ], - "version": "==0.1.2" - }, "pluggy": { "hashes": [ "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0", @@ -681,14 +614,6 @@ "index": "pypi", "version": "==0.13.1" }, - "prometheus-client": { - "hashes": [ - "sha256:3a8baade6cb80bcfe43297e33e7623f3118d660d41387593758e2fb1ea173a86", - "sha256:b014bc76815eb1399da8ce5fc84b7717a3e63652b0c0f8804092c9363acab1b2" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==0.11.0" - }, "protobuf": { "hashes": [ "sha256:13ee7be3c2d9a5d2b42a1030976f760f28755fcf5863c55b1460fd205e6cd637", @@ -914,14 +839,6 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", "version": "==5.4.1" }, - "redis": { - "hashes": [ - "sha256:0e7e0cfca8660dea8b7d5cd8c4f6c5e29e11f31158c0b0ae91a397f00e5a05a2", - "sha256:432b788c4530cfe16d8d943a09d40ca6c16149727e4afe8c2c9d5580c59d9f24" - ], - "index": "pypi", - "version": "==3.5.3" - }, "requests": { "hashes": [ "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804", @@ -993,7 +910,7 @@ "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.10.2" }, "tox": { @@ -1046,7 +963,7 @@ "sha256:50b6f157849174217d0656f99dc82fe932884fb250826c18350e159ec6cdf342", "sha256:779383f6086d90c99ae41cf0ff39aac8a7937a9283ce0a414e5dd782f4c94a84" ], - "markers": "python_version < '3.8' and python_version < '3.8'", + "markers": "python_version < '3.8'", "version": "==3.10.0.0" }, "typing-inspect": { @@ -1073,19 +990,6 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==20.4.7" }, - "watchdog": { - "hashes": [ - "sha256:7e65882adb7746039b6f3876ee174952f8eaaa34491ba34333ddf1fe35de4162" - ], - "version": "==0.8.3" - }, - "watchdog-gevent": { - "hashes": [ - "sha256:a9ef201bbbbaa1f87a6e4c164d55cd077cc9b4406b2f7444854d2bbde7a599de", - "sha256:d19f1276a728dfb3ae3f1c8ada6e791f1726fd4336dff18b4ea35ed3ade35c6a" - ], - "version": "==0.1" - }, "wcwidth": { "hashes": [ "sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784", @@ -1167,70 +1071,6 @@ ], "index": "pypi", "version": "==3.5.0" - }, - "zope.event": { - "hashes": [ - "sha256:2666401939cdaa5f4e0c08cf7f20c9b21423b95e88f4675b1443973bdb080c42", - "sha256:5e76517f5b9b119acf37ca8819781db6c16ea433f7e2062c4afc2b6fbedb1330" - ], - "version": "==4.5.0" - }, - "zope.interface": { - "hashes": [ - "sha256:08f9636e99a9d5410181ba0729e0408d3d8748026ea938f3b970a0249daa8192", - "sha256:0b465ae0962d49c68aa9733ba92a001b2a0933c317780435f00be7ecb959c702", - "sha256:0cba8477e300d64a11a9789ed40ee8932b59f9ee05f85276dbb4b59acee5dd09", - "sha256:0cee5187b60ed26d56eb2960136288ce91bcf61e2a9405660d271d1f122a69a4", - "sha256:0ea1d73b7c9dcbc5080bb8aaffb776f1c68e807767069b9ccdd06f27a161914a", - "sha256:0f91b5b948686659a8e28b728ff5e74b1be6bf40cb04704453617e5f1e945ef3", - "sha256:15e7d1f7a6ee16572e21e3576d2012b2778cbacf75eb4b7400be37455f5ca8bf", - "sha256:17776ecd3a1fdd2b2cd5373e5ef8b307162f581c693575ec62e7c5399d80794c", - "sha256:194d0bcb1374ac3e1e023961610dc8f2c78a0f5f634d0c737691e215569e640d", - "sha256:1c0e316c9add0db48a5b703833881351444398b04111188069a26a61cfb4df78", - "sha256:205e40ccde0f37496904572035deea747390a8b7dc65146d30b96e2dd1359a83", - "sha256:273f158fabc5ea33cbc936da0ab3d4ba80ede5351babc4f577d768e057651531", - "sha256:2876246527c91e101184f63ccd1d716ec9c46519cc5f3d5375a3351c46467c46", - "sha256:2c98384b254b37ce50eddd55db8d381a5c53b4c10ee66e1e7fe749824f894021", - "sha256:2e5a26f16503be6c826abca904e45f1a44ff275fdb7e9d1b75c10671c26f8b94", - "sha256:334701327f37c47fa628fc8b8d28c7d7730ce7daaf4bda1efb741679c2b087fc", - "sha256:3748fac0d0f6a304e674955ab1365d515993b3a0a865e16a11ec9d86fb307f63", - "sha256:3c02411a3b62668200910090a0dff17c0b25aaa36145082a5a6adf08fa281e54", - "sha256:3dd4952748521205697bc2802e4afac5ed4b02909bb799ba1fe239f77fd4e117", - "sha256:3f24df7124c323fceb53ff6168da70dbfbae1442b4f3da439cd441681f54fe25", - "sha256:469e2407e0fe9880ac690a3666f03eb4c3c444411a5a5fddfdabc5d184a79f05", - "sha256:4de4bc9b6d35c5af65b454d3e9bc98c50eb3960d5a3762c9438df57427134b8e", - "sha256:5208ebd5152e040640518a77827bdfcc73773a15a33d6644015b763b9c9febc1", - "sha256:52de7fc6c21b419078008f697fd4103dbc763288b1406b4562554bd47514c004", - "sha256:5bb3489b4558e49ad2c5118137cfeaf59434f9737fa9c5deefc72d22c23822e2", - "sha256:5dba5f530fec3f0988d83b78cc591b58c0b6eb8431a85edd1569a0539a8a5a0e", - "sha256:5dd9ca406499444f4c8299f803d4a14edf7890ecc595c8b1c7115c2342cadc5f", - "sha256:5f931a1c21dfa7a9c573ec1f50a31135ccce84e32507c54e1ea404894c5eb96f", - "sha256:63b82bb63de7c821428d513607e84c6d97d58afd1fe2eb645030bdc185440120", - "sha256:66c0061c91b3b9cf542131148ef7ecbecb2690d48d1612ec386de9d36766058f", - "sha256:6f0c02cbb9691b7c91d5009108f975f8ffeab5dff8f26d62e21c493060eff2a1", - "sha256:71aace0c42d53abe6fc7f726c5d3b60d90f3c5c055a447950ad6ea9cec2e37d9", - "sha256:7d97a4306898b05404a0dcdc32d9709b7d8832c0c542b861d9a826301719794e", - "sha256:7df1e1c05304f26faa49fa752a8c690126cf98b40b91d54e6e9cc3b7d6ffe8b7", - "sha256:8270252effc60b9642b423189a2fe90eb6b59e87cbee54549db3f5562ff8d1b8", - "sha256:867a5ad16892bf20e6c4ea2aab1971f45645ff3102ad29bd84c86027fa99997b", - "sha256:877473e675fdcc113c138813a5dd440da0769a2d81f4d86614e5d62b69497155", - "sha256:8892f89999ffd992208754851e5a052f6b5db70a1e3f7d54b17c5211e37a98c7", - "sha256:9a9845c4c6bb56e508651f005c4aeb0404e518c6f000d5a1123ab077ab769f5c", - "sha256:a1e6e96217a0f72e2b8629e271e1b280c6fa3fe6e59fa8f6701bec14e3354325", - "sha256:a8156e6a7f5e2a0ff0c5b21d6bcb45145efece1909efcbbbf48c56f8da68221d", - "sha256:a9506a7e80bcf6eacfff7f804c0ad5350c8c95b9010e4356a4b36f5322f09abb", - "sha256:af310ec8335016b5e52cae60cda4a4f2a60a788cbb949a4fbea13d441aa5a09e", - "sha256:b0297b1e05fd128d26cc2460c810d42e205d16d76799526dfa8c8ccd50e74959", - "sha256:bf68f4b2b6683e52bec69273562df15af352e5ed25d1b6641e7efddc5951d1a7", - "sha256:d0c1bc2fa9a7285719e5678584f6b92572a5b639d0e471bb8d4b650a1a910920", - "sha256:d4d9d6c1a455d4babd320203b918ccc7fcbefe308615c521062bc2ba1aa4d26e", - "sha256:db1fa631737dab9fa0b37f3979d8d2631e348c3b4e8325d6873c2541d0ae5a48", - "sha256:dd93ea5c0c7f3e25335ab7d22a507b1dc43976e1345508f845efc573d3d779d8", - "sha256:f44e517131a98f7a76696a7b21b164bcb85291cee106a23beccce454e1f433a4", - "sha256:f7ee479e96f7ee350db1cf24afa5685a5899e2b34992fb99e1f7c1b0b758d263" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", - "version": "==5.4.0" } }, "develop": { @@ -1310,14 +1150,6 @@ ], "version": "==0.6.2" }, - "fakeredis": { - "hashes": [ - "sha256:18fc1808d2ce72169d3f11acdb524a00ef96bd29970c6d34cfeb2edb3fc0c020", - "sha256:f1ffdb134538e6d7c909ddfb4fc5edeb4a73d0ea07245bc69b8135fbc4144b04" - ], - "index": "pypi", - "version": "==1.5.2" - }, "importlib-metadata": { "hashes": [ "sha256:079ada16b7fc30dfbb5d13399a5113110dab1aa7c2bc62f66af75f0b717c8cac", @@ -1586,14 +1418,6 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", "version": "==5.4.1" }, - "redis": { - "hashes": [ - "sha256:0e7e0cfca8660dea8b7d5cd8c4f6c5e29e11f31158c0b0ae91a397f00e5a05a2", - "sha256:432b788c4530cfe16d8d943a09d40ca6c16149727e4afe8c2c9d5580c59d9f24" - ], - "index": "pypi", - "version": "==3.5.3" - }, "regex": { "hashes": [ "sha256:0eb2c6e0fcec5e0f1d3bcc1133556563222a2ffd2211945d7b1480c1b1a42a6f", @@ -1640,21 +1464,6 @@ ], "version": "==2021.7.6" }, - "six": { - "hashes": [ - "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", - "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" - ], - "index": "pypi", - "version": "==1.16.0" - }, - "sortedcontainers": { - "hashes": [ - "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", - "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0" - ], - "version": "==2.4.0" - }, "stringcase": { "hashes": [ "sha256:48a06980661908efe8d9d34eab2b6c13aefa2163b3ced26972902e3bdfd87008" @@ -1673,7 +1482,7 @@ "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.10.2" }, "traitlets": { @@ -1726,7 +1535,7 @@ "sha256:50b6f157849174217d0656f99dc82fe932884fb250826c18350e159ec6cdf342", "sha256:779383f6086d90c99ae41cf0ff39aac8a7937a9283ce0a414e5dd782f4c94a84" ], - "markers": "python_version < '3.8' and python_version < '3.8'", + "markers": "python_version < '3.8'", "version": "==3.10.0.0" }, "typing-inspect": { @@ -1739,9 +1548,30 @@ }, "watchdog": { "hashes": [ - "sha256:7e65882adb7746039b6f3876ee174952f8eaaa34491ba34333ddf1fe35de4162" + "sha256:0bcdf7b99b56a3ae069866c33d247c9994ffde91b620eaf0306b27e099bd1ae0", + "sha256:0bcfe904c7d404eb6905f7106c54873503b442e8e918cc226e1828f498bdc0ca", + "sha256:201cadf0b8c11922f54ec97482f95b2aafca429c4c3a4bb869a14f3c20c32686", + "sha256:3a7d242a7963174684206093846537220ee37ba9986b824a326a8bb4ef329a33", + "sha256:3e305ea2757f81d8ebd8559d1a944ed83e3ab1bdf68bcf16ec851b97c08dc035", + "sha256:431a3ea70b20962e6dee65f0eeecd768cd3085ea613ccb9b53c8969de9f6ebd2", + "sha256:44acad6f642996a2b50bb9ce4fb3730dde08f23e79e20cd3d8e2a2076b730381", + "sha256:54e057727dd18bd01a3060dbf5104eb5a495ca26316487e0f32a394fd5fe725a", + "sha256:6fe9c8533e955c6589cfea6f3f0a1a95fb16867a211125236c82e1815932b5d7", + "sha256:85b851237cf3533fabbc034ffcd84d0fa52014b3121454e5f8b86974b531560c", + "sha256:8805a5f468862daf1e4f4447b0ccf3acaff626eaa57fbb46d7960d1cf09f2e6d", + "sha256:9628f3f85375a17614a2ab5eac7665f7f7be8b6b0a2a228e6f6a2e91dd4bfe26", + "sha256:a12539ecf2478a94e4ba4d13476bb2c7a2e0a2080af2bb37df84d88b1b01358a", + "sha256:acc4e2d5be6f140f02ee8590e51c002829e2c33ee199036fcd61311d558d89f4", + "sha256:b5fc5c127bad6983eecf1ad117ab3418949f18af9c8758bd10158be3647298a9", + "sha256:b8ddb2c9f92e0c686ea77341dcb58216fa5ff7d5f992c7278ee8a392a06e86bb", + "sha256:bf84bd94cbaad8f6b9cbaeef43080920f4cb0e61ad90af7106b3de402f5fe127", + "sha256:d9456f0433845e7153b102fffeb767bde2406b76042f2216838af3b21707894e", + "sha256:e4929ac2aaa2e4f1a30a36751160be391911da463a8799460340901517298b13", + "sha256:e5236a8e8602ab6db4b873664c2d356c365ab3cac96fbdec4970ad616415dd45", + "sha256:fd8c595d5a93abd441ee7c5bb3ff0d7170e79031520d113d6f401d0cf49d7c8f" ], - "version": "==0.8.3" + "markers": "python_version >= '3.6'", + "version": "==2.1.3" }, "wcwidth": { "hashes": [ diff --git a/backend/README.md b/backend/README.md index c75a753c..b33e0a53 100644 --- a/backend/README.md +++ b/backend/README.md @@ -28,7 +28,7 @@ Then run the following to build the images from source and stand up a local depl The default gateway port is 8080, so you can visit http://localhost:8080. Develop mode also exposes the other ports from the backend services onto the host. The main backend webserver is on 5000. Swagger API docs can be found at http://localhost:5000/apidocs. -Getting started in the code, we've provided detailed logs through each of the [workflows](wallet/background_tasks), which can be found at http://localhost:5000/execution_logs. +Getting started in the code, we've provided detailed logs through each of the [workflows](wallet), which can be found at http://localhost:5000/execution_logs. ### Docker Compose @@ -55,15 +55,6 @@ If you want to go the extra mile, you can also use helm to install the Diem Refe The Python backend relies on pipenv. To set up, run `pipenv install --dev` in /backend directory. -The workflows use [dramatiq](https://dramatiq.io/) with the Redis broker. Install Redis and run it as a daemon: - - $ redis-server --daemonize yes - -After ensuring you have Redis running: - - $ ./run_web.sh # runs main webserver - $ ./run_worker.sh # runs dramatiq workers - To test: $ ./format.sh # runs black diff --git a/backend/pubsub/__main__.py b/backend/pubsub/__main__.py index 32220aed..e69de29b 100644 --- a/backend/pubsub/__main__.py +++ b/backend/pubsub/__main__.py @@ -1,34 +0,0 @@ -# Copyright (c) The Diem Core Contributors -# SPDX-License-Identifier: Apache-2.0 - -import argparse -import json -import sys - -from pubsub import DEFL_CONFIG, VASP_ADDR -from pubsub.client import LRWPubSubClient - - -parser = argparse.ArgumentParser( - description="Pubsub CLI tool. Takes in pubsub config file or VASP_ADDR environment variable" -) -parser.add_argument("-f", "--file", type=str, help="LRW pubsub config file path") -args = parser.parse_args() - -if args.file: - try: - with open(args.file) as f: - conf = json.load(f) - except: - print("Missing or invalid config file, exiting...") - sys.exit(1) -else: # load in by env var - if not VASP_ADDR: - print("Missing VASP_ADDR environment variable, exiting...") - sys.exit(1) - conf = DEFL_CONFIG - -print(conf) - -client = LRWPubSubClient(conf) -client.start() diff --git a/backend/pubsub/client.py b/backend/pubsub/client.py index a25c9b36..7ca42db0 100644 --- a/backend/pubsub/client.py +++ b/backend/pubsub/client.py @@ -9,9 +9,11 @@ import logging import json -from wallet.background_tasks.background import process_incoming_txn from .types import LRWPubSubEvent +from wallet.services.transaction import process_incoming_transaction +from diem_utils.types.currencies import DiemCurrency from diem import jsonrpc +from . import DEFL_CONFIG class FileProgressStorage: @@ -31,14 +33,14 @@ def save_state(self, state: Dict[str, int]) -> None: class LRWPubSubClient: - def __init__(self, config: Dict[str, Any]) -> None: + def __init__(self, config: Dict[str, Any] = DEFL_CONFIG) -> None: self.sync_interval_ms = config["sync_interval_ms"] self.accounts = config["accounts"] self.diem_node_uri = config["diem_node_uri"] self.progress_file_path = config["progress_file_path"] self.fetch_batch_size = 10 - self.processor = config.get("processor", process_incoming_txn) + self.processor = config.get("processor", LRWPubSubClient.process_incoming_txn) logging.info(f"Loaded LRWPubSubClient with config: {config}") @@ -63,7 +65,7 @@ def sync( ) for event in events: lrw_event = LRWPubSubEvent.from_jsonrpc_event(event) - self.processor.send(lrw_event) + self.processor(lrw_event) logging.info(f"SUCCESS: sent to wallet onchain {lrw_event}") after_sync_state[key] = sequence_num + len(events) @@ -87,3 +89,23 @@ def init_progress_state(self) -> Dict[str, int]: if account.received_events_key not in state: state[account.received_events_key] = 0 return state + + @staticmethod + def process_incoming_txn(txn: LRWPubSubEvent) -> None: + metadata = txn.metadata + blockchain_version = txn.version + sender_address = txn.sender + receiver_address = txn.receiver + sequence = txn.sequence + amount = txn.amount + currency = DiemCurrency[txn.currency] + process_incoming_transaction( + blockchain_version=blockchain_version, + sender_address=sender_address, + receiver_address=receiver_address, + sequence=sequence, + amount=amount, + currency=currency, + metadata=metadata, + ) + diff --git a/backend/run_pubsub.sh b/backend/run_pubsub.sh deleted file mode 100755 index 766b3aa8..00000000 --- a/backend/run_pubsub.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -# Copyright (c) The Diem Core Contributors -# SPDX-License-Identifier: Apache-2.0 - -python3 -m pubsub diff --git a/backend/run_worker.sh b/backend/run_worker.sh deleted file mode 100755 index 2736b93f..00000000 --- a/backend/run_worker.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -# Copyright (c) The Diem Core Contributors -# SPDX-License-Identifier: Apache-2.0 - -PROCS=${PROCS:-2} -THREADS=${THREADS:-2} - -if [ -z "${PROCS##*[!0-9]*}" ] || [ -z "${THREADS##*[!0-9]*}" ]; then - echo "PROCS and THREADS env vars should be integers" - exit 1 -fi - -dramatiq wallet -p $PROCS -t $THREADS --verbose --watch . "$@" diff --git a/backend/test.sh b/backend/test.sh index 331bc6f9..a2cb63b9 100755 --- a/backend/test.sh +++ b/backend/test.sh @@ -11,7 +11,7 @@ set -euo pipefail # NOTE: # each of these test modules need to be run independently due to how they -# instantiate dramatiq and other test dependencies in conftest.py +# instantiate test dependencies in conftest.py pipenv run python3 $DIR/setup.py pytest --addopts="$DIR/tests/wallet_tests $@" pipenv run python3 $DIR/setup.py pytest --addopts="$DIR/tests/webapp_tests $@" pipenv run python3 $DIR/setup.py pytest --addopts="$DIR/tests/pubsub_tests $@" diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index a4217657..5246b400 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -89,14 +89,6 @@ def wait_for_transaction(*args): yield network -@pytest.fixture(autouse=True) -def no_background_tasks(monkeypatch) -> None: - def mocked() -> bool: - return False - - monkeypatch.setattr(services, "run_bg_tasks", mocked) - - class LpClientMock: QUOTES: Dict[QuoteId, QuoteData] = {} TRADES: Dict[TradeId, TradeData] = {} diff --git a/backend/tests/wallet_tests/services/test_transaction.py b/backend/tests/wallet_tests/services/test_transaction.py index 7174de6b..73dfffab 100644 --- a/backend/tests/wallet_tests/services/test_transaction.py +++ b/backend/tests/wallet_tests/services/test_transaction.py @@ -53,7 +53,7 @@ def test_send_transaction() -> None: assert tx.destination_subaddress == "receiver_subaddress" -def test_transaction_direction(no_background_tasks) -> None: +def test_transaction_direction() -> None: account_id, send_tx = send_fake_tx() assert get_transaction_direction(account_id, send_tx) == TransactionDirection.SENT diff --git a/backend/wallet/__init__.py b/backend/wallet/__init__.py index 3ac58d76..2fffdb2b 100644 --- a/backend/wallet/__init__.py +++ b/backend/wallet/__init__.py @@ -1,9 +1,4 @@ # Copyright (c) The Diem Core Contributors # SPDX-License-Identifier: Apache-2.0 -# keep config import first, because we MUST configure dramatiq broker prior to background_tasks import -# once @actor is being used (e.g. in background_tasks) and no global dramatiq broker configured - -# dramatiq will try to use default RabbitMQ one (this is hardcoded and can't be changed) - which will fail - from . import config -from . import background_tasks diff --git a/backend/wallet/background_tasks/__init__.py b/backend/wallet/background_tasks/__init__.py deleted file mode 100644 index 7f07ecbd..00000000 --- a/backend/wallet/background_tasks/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from .background import * - -# Copyright (c) The Diem Core Contributors -# SPDX-License-Identifier: Apache-2.0 diff --git a/backend/wallet/background_tasks/background.py b/backend/wallet/background_tasks/background.py deleted file mode 100644 index fb1dd03b..00000000 --- a/backend/wallet/background_tasks/background.py +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright (c) The Diem Core Contributors -# SPDX-License-Identifier: Apache-2.0 - -import sys -from time import sleep - -import dramatiq - -from pubsub.types import LRWPubSubEvent -from wallet.services.order import ( - execute_order, - cover_order, -) -from .utils import retry -from ..logging import debug_log, log_execution -from ..services.kyc import verify_kyc -from ..services.transaction import ( - submit_onchain, - process_incoming_transaction, -) -from diem_utils.types.currencies import DiemCurrency - -TIME_BEFORE_KYC_APPROVAL = 5 - - -@dramatiq.actor(store_results=True) -@debug_log(None) -def async_start_kyc(user_id: int) -> None: - sys.stdout.write("hhhhhhhhh") - log_execution("Enter async_start_kyc") - sleep(TIME_BEFORE_KYC_APPROVAL) - verify_kyc(user_id) - - -@dramatiq.actor(store_results=True) -@debug_log(None) -def async_execute_order(order_id, payment_method) -> None: - log_execution("Enter async_execute_order") - execute_order(order_id, payment_method) - - -@dramatiq.actor(store_results=True) -@debug_log(None) -def async_cover_order(order_id) -> None: - log_execution("Enter async_cover") - cover_order(order_id) - - -@dramatiq.actor(store_results=True) -@debug_log(None) -def async_external_transaction(transaction_id: int) -> None: - log_execution("Enter async_external_transaction") - submit_onchain(transaction_id=transaction_id) - - -@dramatiq.actor(store_results=True) -@retry(Exception, delay=1) -def process_incoming_txn(txn: LRWPubSubEvent) -> None: - metadata = txn.metadata - blockchain_version = txn.version - sender_address = txn.sender - receiver_address = txn.receiver - sequence = txn.sequence - amount = txn.amount - currency = DiemCurrency[txn.currency] - process_incoming_transaction( - blockchain_version=blockchain_version, - sender_address=sender_address, - receiver_address=receiver_address, - sequence=sequence, - amount=amount, - currency=currency, - metadata=metadata, - ) diff --git a/backend/wallet/background_tasks/utils.py b/backend/wallet/background_tasks/utils.py deleted file mode 100644 index 1c50c793..00000000 --- a/backend/wallet/background_tasks/utils.py +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright (c) The Diem Core Contributors -# SPDX-License-Identifier: Apache-2.0 - -import typing -import time -from functools import wraps - -RT = typing.TypeVar("RT") -TFun = typing.Callable[..., typing.Optional[RT]] - - -def retry( - exceptions: typing.Type[Exception], - tries: int = 4, - delay: int = 3, - backoff: int = 2, - logger: typing.Optional[typing.Callable[[str], None]] = None, -) -> typing.Callable[[TFun], TFun]: - """ - Retry calling the decorated function using an exponential backoff. - Args: - exceptions: The exception to check. may be a tuple of - exceptions to check. - tries: Number of times to try (not retry) before giving up. - delay: Initial delay between retries in seconds. - backoff: Backoff multiplier (e.g. value of 2 will double the delay - each retry). - logger: Logger to use. If None, print. - """ - - def deco_retry(f: TFun) -> TFun: - @wraps(f) - def f_retry(*args, **kwargs): # pyre-ignore - mtries, mdelay = tries, delay - while mtries > 1: - try: - return f(*args, **kwargs) - except exceptions as e: - msg = "{}, Retrying in {} seconds...".format(e, mdelay) - if logger: - logger(msg) - else: - print(msg) - time.sleep(mdelay) - mtries -= 1 - mdelay *= backoff - return f(*args, **kwargs) - - return typing.cast(TFun, f_retry) # true decorator - - return deco_retry diff --git a/backend/wallet/config.py b/backend/wallet/config.py index fd8bd840..f1b925c3 100644 --- a/backend/wallet/config.py +++ b/backend/wallet/config.py @@ -8,13 +8,6 @@ import sys from typing import Optional -import dramatiq -import redis -from dramatiq.brokers.redis import RedisBroker, Broker -from dramatiq.encoder import PickleEncoder -from dramatiq.results import Results -from dramatiq.results.backends.redis import RedisBackend - import logging logging.basicConfig( @@ -26,10 +19,6 @@ # logging.getLogger("sqlalchemy.engine").setLevel(logging.DEBUG) -REDIS_HOST: str = os.getenv("REDIS_HOST", "localhost") -REDIS_PORT: int = int(os.getenv("REDIS_PORT", 6379)) -REDIS_DB: int = int(os.getenv("REDIS_DB", 0)) -REDIS_PASSWORD: str = os.getenv("REDIS_PASSWORD", "") DB_URL: str = os.getenv("DB_URL", "sqlite:////tmp/test.db") ADMIN_USERNAME: str = os.getenv("ADMIN_USERNAME", "admin@lrw") ADMIN_LOGIN_ENABLED: bool = ( @@ -37,28 +26,6 @@ ) SECRET_KEY: str = os.getenv("SECRET_KEY", "you-will-never-guess") -SESSION_TYPE: str = "redis" - - -# init redis and dramatiq broker -def setup_redis_broker() -> None: - _connection_pool: redis.BlockingConnectionPool = redis.BlockingConnectionPool( - host=REDIS_HOST, port=REDIS_PORT, db=REDIS_DB, password=REDIS_PASSWORD - ) - _redis_db: redis.StrictRedis = redis.StrictRedis(connection_pool=_connection_pool) - _result_backend = RedisBackend(encoder=PickleEncoder(), client=_redis_db) - _result_middleware = Results(backend=_result_backend) - broker: Broker = RedisBroker( - connection_pool=_connection_pool, - middleware=[_result_middleware], - namespace="lrw", - ) - dramatiq.set_broker(broker) - dramatiq.set_encoder(dramatiq.PickleEncoder()) - - -if dramatiq.broker.global_broker is None: - if "VASP_ADDR" in os.environ: - context.set(context.from_env()) - setup_redis_broker() +if "VASP_ADDR" in os.environ: + context.set(context.from_env()) diff --git a/backend/wallet/services/__init__.py b/backend/wallet/services/__init__.py index f5abb47b..4356b816 100644 --- a/backend/wallet/services/__init__.py +++ b/backend/wallet/services/__init__.py @@ -3,11 +3,4 @@ import os -_RUN_BACKGROUND_TASKS = bool(os.getenv("RUN_BACKGROUND_TASKS", True)) - - -def run_bg_tasks() -> bool: - return _RUN_BACKGROUND_TASKS - - INVENTORY_ACCOUNT_NAME = "Inventory" diff --git a/backend/wallet/services/offchain/fund_pull_pre_approval.py b/backend/wallet/services/offchain/fund_pull_pre_approval.py index 86159b93..1f8163e7 100644 --- a/backend/wallet/services/offchain/fund_pull_pre_approval.py +++ b/backend/wallet/services/offchain/fund_pull_pre_approval.py @@ -63,7 +63,7 @@ def create_and_approve( max_transaction_amount_currency: str = None, description: str = None, ) -> None: - """ Create and approve fund pull pre approval by payer """ + """Create and approve fund pull pre approval by payer""" validate_expiration_timestamp(expiration_timestamp) command = get_command_by_id(funds_pull_pre_approval_id) diff --git a/backend/wallet/services/offchain/payment_command.py b/backend/wallet/services/offchain/payment_command.py index a12ab0e9..ee98a22f 100644 --- a/backend/wallet/services/offchain/payment_command.py +++ b/backend/wallet/services/offchain/payment_command.py @@ -218,7 +218,7 @@ def payment_command_to_model( def get_command_account_id(command: offchain.PaymentCommand) -> int: - """ Find the account id for the command """ + """Find the account id for the command""" sender_address_bech32 = command.payment.sender.address sender_address, sender_sub_address = identifier.decode_account( sender_address_bech32, context.get().config.diem_address_hrp() diff --git a/backend/wallet/services/order.py b/backend/wallet/services/order.py index 6ddd87fb..e12da2ef 100644 --- a/backend/wallet/services/order.py +++ b/backend/wallet/services/order.py @@ -124,12 +124,7 @@ def create_order( def process_order(order_id: OrderId, payment_method: str): - if services.run_bg_tasks(): - from ..background_tasks.background import async_execute_order - - async_execute_order.send(order_id, payment_method) - else: - execute_order(order_id=order_id, payment_method=payment_method) + execute_order(order_id=order_id, payment_method=payment_method) def execute_order(order_id: OrderId, payment_method: Optional[str] = None): @@ -146,12 +141,7 @@ def execute_order(order_id: OrderId, payment_method: Optional[str] = None): if order.order_type == OrderType.Trade: if execute_trade(order): - if services.run_bg_tasks(): - from ..background_tasks.background import async_cover_order - - async_cover_order.send(order_id) - else: - cover_order(order_id=order_id) + cover_order(order_id=order_id) else: execute_convert(order) diff --git a/backend/wallet/services/transaction.py b/backend/wallet/services/transaction.py index 45079d3f..46c19b08 100644 --- a/backend/wallet/services/transaction.py +++ b/backend/wallet/services/transaction.py @@ -567,12 +567,7 @@ def external_transaction( original_txn_id=original_txn_id, ) - if services.run_bg_tasks(): - from ..background_tasks.background import async_external_transaction - - async_external_transaction.send(transaction.id) - else: - submit_onchain(transaction_id=transaction.id) + submit_onchain(transaction_id=transaction.id) return transaction diff --git a/backend/wallet/types.py b/backend/wallet/types.py index 7949e9cf..0e19d9fd 100644 --- a/backend/wallet/types.py +++ b/backend/wallet/types.py @@ -173,7 +173,7 @@ class ConvertResult(str, Enum): class BalanceError(Exception): - """ Indicates a insufficient funds """ + """Indicates a insufficient funds""" pass diff --git a/backend/webapp/__init__.py b/backend/webapp/__init__.py index d0b5e433..d86daaf0 100644 --- a/backend/webapp/__init__.py +++ b/backend/webapp/__init__.py @@ -10,6 +10,9 @@ from threading import Thread from flasgger import Swagger from flask import Flask + +from pubsub import DEFL_CONFIG +from pubsub.client import LRWPubSubClient from wallet.services.system import sync_db from werkzeug.middleware.proxy_fix import ProxyFix @@ -86,6 +89,11 @@ def run(): Thread(target=run, daemon=True).start() +def _onchain_pubsub() -> None: + client = LRWPubSubClient() + Thread(target=client.start, daemon=True).start() + + def _create_app() -> Flask: app = Flask(__name__) # register api endpoints @@ -126,6 +134,7 @@ def init(): _init_with_log("update_rates_thread", _schedule_update_rates) _init_with_log("sync-db", _sync_db) _init_with_log("offchain-tasks", _offchain_tasks) + _init_with_log("onchain-pubsub", _onchain_pubsub) return app diff --git a/docker/dev.docker-compose.yaml b/docker/dev.docker-compose.yaml index 978823c1..bc82fc02 100644 --- a/docker/dev.docker-compose.yaml +++ b/docker/dev.docker-compose.yaml @@ -17,21 +17,6 @@ services: - /wallet/src - /tmp:/tmp - backend-worker: - environment: - COMPOSE_ENV: development - volumes: - - ../backend:/wallet - - /wallet/src - - /tmp:/tmp - - backend-pubsub-worker: - environment: - COMPOSE_ENV: development - volumes: - - ../backend:/wallet - - /wallet/src - liquidity: environment: COMPOSE_ENV: development diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index ce0ba674..86d85442 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -19,7 +19,6 @@ services: env_file: - ../backend/.env environment: - - REDIS_HOST=redis - COMPOSE_ENV=production - DB_URL=postgresql://backenduser:backendpassword@db/backend_db - LP_DB_URL=postgresql://backenduser:backendpassword@db/liquidity_provider_db @@ -27,32 +26,6 @@ services: command: sh -c "/wait.sh liquidity 5000 && ./run_web.sh" restart: always - backend-worker: - image: lrw_backend - env_file: - - ../backend/.env - environment: - - REDIS_HOST=redis - - COMPOSE_ENV=production - - DB_URL=postgresql://backenduser:backendpassword@db/backend_db - - LP_DB_URL=postgresql://backenduser:backendpassword@db/liquidity_provider_db - - ADMIN_USERNAME=admin@lrw - - PROCS=${PROCS:-2} - - THREADS=${THREADS:-2} - command: sh -c "/wait.sh backend-web-server 5000 && ./run_worker.sh" - restart: always - - backend-pubsub-worker: - image: lrw_backend - env_file: - - ../backend/.env - environment: - - REDIS_HOST=redis - - COMPOSE_ENV=production - - DB_URL=postgresql://backenduser:backendpassword@db/backend_db - command: sh -c "/wait.sh backend-web-server 5000 && ./run_pubsub.sh" - restart: always - liquidity: image: lrw_liquidity build: @@ -69,10 +42,6 @@ services: command: sh -c "/liquidity/wait.sh db 5432 && /liquidity/run.sh" restart: always - redis: - image: redis:5 - restart: always - db: image: lrw_db build: diff --git a/docker/e2e.docker-compose.yaml b/docker/e2e.docker-compose.yaml index 332ac6fd..f9c828fe 100644 --- a/docker/e2e.docker-compose.yaml +++ b/docker/e2e.docker-compose.yaml @@ -10,14 +10,6 @@ services: ports: - "5000" - backend-worker: - environment: - - COMPOSE_ENV="development" - - backend-pubsub-worker: - environment: - - COMPOSE_ENV="development" - liquidity: environment: - COMPOSE_ENV="development" diff --git a/helm/reference-wallet/templates/_helpers.tpl b/helm/reference-wallet/templates/_helpers.tpl index 12df816a..19253bc6 100644 --- a/helm/reference-wallet/templates/_helpers.tpl +++ b/helm/reference-wallet/templates/_helpers.tpl @@ -62,14 +62,6 @@ Create the name of the service account to use {{- end }} {{- end }} -{{- define "reference-wallet.peripherals.redis.url" -}} -{{- if .Values.peripherals.redis.create }} -{{- (include "reference-wallet.fullname" .) }}-redis -{{- else }} -{{ .Values.peripherals.redis.host }} -{{- end }} -{{- end }} - {{- define "reference-wallet.peripherals.database.url" -}} {{- if .Values.peripherals.database.create }} {{- $host := printf "%s-db" (include "reference-wallet.fullname" .) }} diff --git a/helm/reference-wallet/templates/backend-deployment.yaml b/helm/reference-wallet/templates/backend-deployment.yaml index 0ee20ae8..210cfbb9 100644 --- a/helm/reference-wallet/templates/backend-deployment.yaml +++ b/helm/reference-wallet/templates/backend-deployment.yaml @@ -43,8 +43,6 @@ spec: value: "production" - name: WALLET_PORT value: {{.Values.service.targetPort | quote}} - - name: REDIS_HOST - value: {{ include "reference-wallet.peripherals.redis.url" . }} - name: DB_URL value: {{ include "reference-wallet.peripherals.database.url" . }} - name: ADMIN_USERNAME @@ -78,159 +76,3 @@ spec: - name: http containerPort: {{.Values.service.targetPort}} protocol: TCP ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "reference-wallet.fullname" . }}-worker - labels: - app: {{ include "reference-wallet.fullname" . }}-worker - {{- include "reference-wallet.labels" . | nindent 4 }} -spec: -{{- if not .Values.autoscaling.enabled }} - replicas: {{ .Values.replicaCount }} -{{- end }} - selector: - matchLabels: - app: {{ include "reference-wallet.fullname" . }}-worker - {{- include "reference-wallet.selectorLabels" . | nindent 6 }} - template: - metadata: - {{- with .Values.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - app: {{ include "reference-wallet.fullname" . }}-worker - {{- include "reference-wallet.selectorLabels" . | nindent 8 }} - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "reference-wallet.serviceAccountName" . }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - initContainers: - - name: init-wait - image: "{{ .Values.images.backend }}" - command: ["sh", "-c", '/wait.sh {{ include "reference-wallet.fullname" . }}-web {{ .Values.service.port }} || exit 1'] - imagePullPolicy: {{ .Values.images.pullPolicy }} - containers: - - command: - - /wallet/run_worker.sh - env: - - name: REDIS_HOST - value: {{ include "reference-wallet.peripherals.redis.url" . }} - - name: DB_URL - value: {{ include "reference-wallet.peripherals.database.url" . }} - - name: ADMIN_USERNAME - value: {{ .Values.walletAdminUsername }} - - name: CUSTODY_PRIVATE_KEYS - value: {{ include "reference-wallet.custodyPrivateKeys" . }} - - name: LIQUIDITY_SERVICE_PORT - value: {{ .Values.liquidityProviderPort | quote }} - - name: LIQUIDITY_SERVICE_HOST - value: {{ .Values.liquidityProviderHost | quote }} - - name: PROCS - value: {{ .Values.workers.numProcs | quote }} - - name: THREADS - value: {{ .Values.workers.numThreads | quote }} - - name: FAUCET_URL - value: {{ .Values.sdk.faucet }} - - name: VASP_ADDR - value: {{ .Values.vaspAddr }} - - name: WALLET_CUSTODY_ACCOUNT_NAME - value: {{ .Values.custodyWalletAccountName | quote }} - - name: VASP_COMPLIANCE_KEY - value: {{ required "Wallet VASP compliance key must be set in .Values.vaspComplianceKey" .Values.vaspComplianceKey | quote }} - - name: JSON_RPC_URL - value: {{ .Values.sdk.jsonRpc }} - - name: VASP_BASE_URL - value: http://{{ .Values.ingress.host }}/api/offchain - - name: CHAIN_ID - value: {{ .Values.chainId | quote }} - - name: GAS_CURRENCY_CODE - value: {{ .Values.gasCurrencyCode }} - image: "{{ .Values.images.backend }}" - imagePullPolicy: {{ .Values.images.pullPolicy }} - name: lrw-worker - ports: - - name: http - containerPort: 8080 - protocol: TCP ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "reference-wallet.fullname" . }}-pubsub - labels: - app: {{ include "reference-wallet.fullname" . }}-pubsub - {{- include "reference-wallet.labels" . | nindent 4 }} -spec: -{{- if not .Values.autoscaling.enabled }} - replicas: {{ .Values.replicaCount }} -{{- end }} - selector: - matchLabels: - app: {{ include "reference-wallet.fullname" . }}-pubsub - {{- include "reference-wallet.selectorLabels" . | nindent 6 }} - template: - metadata: - {{- with .Values.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - app: {{ include "reference-wallet.fullname" . }}-pubsub - {{- include "reference-wallet.selectorLabels" . | nindent 8 }} - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "reference-wallet.serviceAccountName" . }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - initContainers: - - name: init-wait - image: "{{ .Values.images.backend }}" - command: ["sh", "-c", '/wait.sh {{ include "reference-wallet.fullname" . }}-web {{ .Values.service.port }} || exit 1'] - imagePullPolicy: {{ .Values.images.pullPolicy }} - containers: - - command: - - /wallet/run_pubsub.sh - env: - - name: REDIS_HOST - value: {{ include "reference-wallet.peripherals.redis.url" . }} - - name: DB_URL - value: {{ include "reference-wallet.peripherals.database.url" . }} - - name: CUSTODY_PRIVATE_KEYS - value: {{ include "reference-wallet.custodyPrivateKeys" . }} - - name: LIQUIDITY_SERVICE_PORT - value: {{ .Values.liquidityProviderPort | quote }} - - name: LIQUIDITY_SERVICE_HOST - value: {{ .Values.liquidityProviderHost | quote }} - - name: FAUCET_URL - value: {{ .Values.sdk.faucet }} - - name: VASP_ADDR - value: {{ .Values.vaspAddr }} - - name: WALLET_CUSTODY_ACCOUNT_NAME - value: {{ .Values.custodyWalletAccountName | quote }} - - name: VASP_COMPLIANCE_KEY - value: {{ required "Wallet VASP compliance key must be set in .Values.vaspComplianceKey" .Values.vaspComplianceKey | quote }} - - name: JSON_RPC_URL - value: {{ .Values.sdk.jsonRpc }} - - name: VASP_BASE_URL - value: http://{{ .Values.ingress.host }}/api/offchain - - name: CHAIN_ID - value: {{ .Values.chainId | quote }} - - name: GAS_CURRENCY_CODE - value: {{ .Values.gasCurrencyCode }} - image: "{{ .Values.images.backend }}" - imagePullPolicy: {{ .Values.images.pullPolicy }} - name: lrw-pubsub - ports: - - name: http - containerPort: 8080 - protocol: TCP diff --git a/helm/reference-wallet/templates/backend-service.yaml b/helm/reference-wallet/templates/backend-service.yaml index 486f926b..9d9a4fee 100644 --- a/helm/reference-wallet/templates/backend-service.yaml +++ b/helm/reference-wallet/templates/backend-service.yaml @@ -15,39 +15,3 @@ spec: selector: {{- include "reference-wallet.selectorLabels" . | nindent 4 }} app: {{ include "reference-wallet.fullname" . }}-web ---- -apiVersion: v1 -kind: Service -metadata: - name: {{ include "reference-wallet.fullname" . }}-worker - labels: - {{- include "reference-wallet.labels" . | nindent 4 }} - app: {{ include "reference-wallet.fullname" . }}-worker -spec: - type: {{ .Values.service.type }} - ports: - - port: {{ .Values.service.port }} - targetPort: {{ .Values.service.targetPort }} - protocol: TCP - name: http - selector: - {{- include "reference-wallet.selectorLabels" . | nindent 4 }} - app: {{ include "reference-wallet.fullname" . }}-worker ---- -apiVersion: v1 -kind: Service -metadata: - name: {{ include "reference-wallet.fullname" . }}-pubsub - labels: - {{- include "reference-wallet.labels" . | nindent 4 }} - app: {{ include "reference-wallet.fullname" . }}-pubsub -spec: - type: {{ .Values.service.type }} - ports: - - port: {{ .Values.service.port }} - targetPort: {{ .Values.service.targetPort }} - protocol: TCP - name: http - selector: - {{- include "reference-wallet.selectorLabels" . | nindent 4 }} - app: {{ include "reference-wallet.fullname" . }}-pubsub diff --git a/helm/reference-wallet/templates/peripherals-deployment.yaml b/helm/reference-wallet/templates/peripherals-deployment.yaml index a8b3778a..9b000931 100644 --- a/helm/reference-wallet/templates/peripherals-deployment.yaml +++ b/helm/reference-wallet/templates/peripherals-deployment.yaml @@ -1,53 +1,3 @@ -{{- if .Values.peripherals.redis.create -}} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "reference-wallet.fullname" . }}-redis - labels: - app: {{ include "reference-wallet.fullname" . }}-redis - {{- include "reference-wallet.labels" . | nindent 4 }} -spec: - selector: - matchLabels: - app: {{ include "reference-wallet.fullname" . }}-redis - {{- include "reference-wallet.selectorLabels" . | nindent 6 }} - replicas: 1 - template: - metadata: - {{- with .Values.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - app: {{ include "reference-wallet.fullname" . }}-redis - {{- include "reference-wallet.selectorLabels" . | nindent 8 }} - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "reference-wallet.serviceAccountName" . }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - containers: - - image: "{{ .Values.peripherals.redis.image }}" - name: redis - ports: - - containerPort: {{ .Values.peripherals.redis.port }} - {{- if .Values.peripherals.redis.persistence }} - volumeMounts: - - mountPath: /data - name: {{ include "reference-wallet.fullname" . }}-redis-pv - {{- end }} - restartPolicy: Always - {{- if .Values.peripherals.redis.persistence }} - volumes: - - name: {{ include "reference-wallet.fullname" . }}-redis-pv - persistentVolumeClaim: - claimName: {{ include "reference-wallet.fullname" . }}-redis-pv-claim - {{- end }} ---- -{{- end }} {{- if .Values.peripherals.database.create -}} apiVersion: apps/v1 kind: Deployment diff --git a/helm/reference-wallet/templates/peripherals-service.yaml b/helm/reference-wallet/templates/peripherals-service.yaml index 2abbb2ad..cd59a0cf 100644 --- a/helm/reference-wallet/templates/peripherals-service.yaml +++ b/helm/reference-wallet/templates/peripherals-service.yaml @@ -1,22 +1,3 @@ -{{- if .Values.peripherals.redis.create -}} -apiVersion: v1 -kind: Service -metadata: - name: {{ include "reference-wallet.fullname" . }}-redis - labels: - app: {{ include "reference-wallet.fullname" . }}-redis - {{- include "reference-wallet.labels" . | nindent 4 }} -spec: - selector: - app: {{ include "reference-wallet.fullname" . }}-redis - {{- include "reference-wallet.selectorLabels" . | nindent 4 }} - ports: - - protocol: "TCP" - port: {{ .Values.peripherals.redis.port }} - targetPort: {{ .Values.peripherals.redis.port }} - name: redis ---- -{{- end }} {{- if .Values.peripherals.database.create -}} apiVersion: v1 kind: Service diff --git a/helm/reference-wallet/templates/volumes.yaml b/helm/reference-wallet/templates/volumes.yaml index 7e6928b7..c7793ab0 100644 --- a/helm/reference-wallet/templates/volumes.yaml +++ b/helm/reference-wallet/templates/volumes.yaml @@ -1,34 +1,3 @@ -{{- if .Values.peripherals.redis.persistence -}} -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: {{ include "reference-wallet.fullname" . }}-redis-pv-claim - labels: - app: {{ include "reference-wallet.fullname" . }}-redis -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: {{ .Values.peripherals.redis.persistence.claimSize }} ---- -kind: PersistentVolume -apiVersion: v1 -metadata: - name: {{ include "reference-wallet.fullname" . }}-redis-pv - labels: - type: local - app: {{ include "reference-wallet.fullname" . }}-redis -spec: - storageClassName: manual - capacity: - storage: {{ .Values.peripherals.redis.persistence.pvSize }} - accessModes: - - ReadWriteOnce - hostPath: - path: "/mnt/data" ---- -{{- end }} {{- if .Values.peripherals.database.persistence -}} apiVersion: v1 kind: PersistentVolumeClaim diff --git a/helm/reference-wallet/values.yaml b/helm/reference-wallet/values.yaml index 72a3f59d..472a4751 100644 --- a/helm/reference-wallet/values.yaml +++ b/helm/reference-wallet/values.yaml @@ -39,16 +39,6 @@ workers: numThreads: 2 peripherals: - redis: - # Should a database service be created? - # If false, host:port should point to an existing DB instance - create: false - image: "redis:5" - # if redis is created by this helm chart, there is no need to fill the following - host: "" - port: 6379 - persistence: {} - database: # Should a database service be created? # If false, host:port should point to an existing DB instance diff --git a/scripts/lrw.sh b/scripts/lrw.sh index 122a15c5..2d6c9d7b 100755 --- a/scripts/lrw.sh +++ b/scripts/lrw.sh @@ -181,9 +181,6 @@ develop() { local follow=${2:-true} echo "debug mode with gw port ${port}" - # build the entire docker services using compose - docker-compose -f ${COMPOSE_YAML} -f ${COMPOSE_DEV_YAML} pull redis - GW_PORT=$port docker-compose -f ${COMPOSE_YAML} -f ${COMPOSE_DEV_YAML} up --detach --no-build if [ "$follow" == true ]; then @@ -314,7 +311,6 @@ debug() { docker-compose -f ${COMPOSE_YAML} -f ${COMPOSE_DEV_YAML} -f ${COMPOSE_DEBUG_YAML} build $gateway docker-compose -f ${COMPOSE_YAML} -f ${COMPOSE_DEV_YAML} -f ${COMPOSE_DEBUG_YAML} up -d --scale $backend=0 ./build.sh - REDIS_HOST=127.0.0.1 ./run_web.sh docker-compose -f ${COMPOSE_YAML} -f ${COMPOSE_DEV_YAML} -f ${COMPOSE_DEBUG_YAML} rm -fsv $gateway docker-compose -f ${COMPOSE_YAML} -f ${COMPOSE_DEV_YAML} build $gateway @@ -339,7 +335,7 @@ deploy_minikube() { build 8080 true helm upgrade --install lrw helm/reference-wallet \ - --set peripherals.redis.create=true --set peripherals.database.create=true + --set peripherals.database.create=true } watch_test() {