From 28ca852d8c81afc79107fc0441851e3ea5c092a4 Mon Sep 17 00:00:00 2001 From: Morty Space Date: Thu, 11 Nov 2021 00:02:33 +0000 Subject: [PATCH 01/15] [JOB] Updated API pairs and coins --- src/cryptocom/exchange/coins.py | 1 + src/cryptocom/exchange/pairs.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/src/cryptocom/exchange/coins.py b/src/cryptocom/exchange/coins.py index 13d0487..9820a40 100644 --- a/src/cryptocom/exchange/coins.py +++ b/src/cryptocom/exchange/coins.py @@ -143,6 +143,7 @@ USDP = Coin("USDP") USDT = Coin("USDT") VET = Coin("VET") VTHO = Coin("VTHO") +VVS = Coin("VVS") WAVE = Coin("WAVE") WAVES = Coin("WAVES") WAXP = Coin("WAXP") diff --git a/src/cryptocom/exchange/pairs.py b/src/cryptocom/exchange/pairs.py index ba9c3be..6341485 100644 --- a/src/cryptocom/exchange/pairs.py +++ b/src/cryptocom/exchange/pairs.py @@ -15,6 +15,7 @@ ALGO_USDT = Pair("ALGO_USDT", price_precision=4, quantity_precision=2) ALICE_USDT = Pair("ALICE_USDT", price_precision=3, quantity_precision=3) AMP_USDT = Pair("AMP_USDT", price_precision=5, quantity_precision=1) ANKR_USDT = Pair("ANKR_USDT", price_precision=6, quantity_precision=1) +ARPA_USDC = Pair("ARPA_USDC", price_precision=5, quantity_precision=1) AR_USDC = Pair("AR_USDC", price_precision=3, quantity_precision=3) ATOM_BTC = Pair("ATOM_BTC", price_precision=7, quantity_precision=2) ATOM_CRO = Pair("ATOM_CRO", price_precision=2, quantity_precision=2) @@ -197,12 +198,14 @@ SNX_USDT = Pair("SNX_USDT", price_precision=3, quantity_precision=3) SOL_BTC = Pair("SOL_BTC", price_precision=7, quantity_precision=3) SOL_USDC = Pair("SOL_USDC", price_precision=3, quantity_precision=3) SOL_USDT = Pair("SOL_USDT", price_precision=3, quantity_precision=3) +SRM_USDC = Pair("SRM_USDC", price_precision=4, quantity_precision=2) STORJ_USDT = Pair("STORJ_USDT", price_precision=4, quantity_precision=2) STX_USDT = Pair("STX_USDT", price_precision=4, quantity_precision=2) SUSHI_USDT = Pair("SUSHI_USDT", price_precision=3, quantity_precision=3) TFUEL_USDT = Pair("TFUEL_USDT", price_precision=5, quantity_precision=1) THETA_USDT = Pair("THETA_USDT", price_precision=4, quantity_precision=2) TRB_USDT = Pair("TRB_USDT", price_precision=3, quantity_precision=3) +TRU_USDT = Pair("TRU_USDT", price_precision=5, quantity_precision=1) UMA_USDT = Pair("UMA_USDT", price_precision=3, quantity_precision=4) UNI_CRO = Pair("UNI_CRO", price_precision=3, quantity_precision=2) UNI_USDC = Pair("UNI_USDC", price_precision=3, quantity_precision=3) @@ -227,6 +230,7 @@ XRP_USDT = Pair("XRP_USDT", price_precision=5, quantity_precision=1) XTZ_BTC = Pair("XTZ_BTC", price_precision=8, quantity_precision=2) XTZ_CRO = Pair("XTZ_CRO", price_precision=3, quantity_precision=2) XTZ_USDT = Pair("XTZ_USDT", price_precision=4, quantity_precision=2) +XYO_USDT = Pair("XYO_USDT", price_precision=6, quantity_precision=0) YFI_BTC = Pair("YFI_BTC", price_precision=4, quantity_precision=6) YFI_CRO = Pair("YFI_CRO", price_precision=2, quantity_precision=6) YFI_USDT = Pair("YFI_USDT", price_precision=2, quantity_precision=6) From 09fbbaaea3fac9926b4e030d85e36f22c5dfc3b5 Mon Sep 17 00:00:00 2001 From: Morty Space <43354956+mortyspace@users.noreply.github.com> Date: Thu, 11 Nov 2021 16:59:19 +0200 Subject: [PATCH 02/15] Update commit of new pairs --- azure-pipelines.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index bef7044..bd91dd8 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -58,6 +58,7 @@ steps: git commit -m "[JOB] Updated API pairs and coins" git push origin HEAD:master displayName: 'Update API structs' + condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master')) - script: | cd docs && make doctest && cd .. From 0ec95652457880532ea522f43ca1b212b8b64bd4 Mon Sep 17 00:00:00 2001 From: Morty Space Date: Thu, 11 Nov 2021 17:11:47 +0200 Subject: [PATCH 03/15] Added python3.10 check --- azure-pipelines.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index bd91dd8..d5995be 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -29,7 +29,8 @@ strategy: python.version: '3.8' Python39: python.version: '3.9' - + Python310: + python.version: '3.10' steps: - checkout: self persistCredentials: true From 97997bbbfbd3892d4325d4f13dfac84b3a310e97 Mon Sep 17 00:00:00 2001 From: Morty Space Date: Sat, 13 Nov 2021 00:02:49 +0000 Subject: [PATCH 04/15] [JOB] Updated API pairs and coins --- src/cryptocom/exchange/coins.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/cryptocom/exchange/coins.py b/src/cryptocom/exchange/coins.py index 9820a40..cb96926 100644 --- a/src/cryptocom/exchange/coins.py +++ b/src/cryptocom/exchange/coins.py @@ -46,6 +46,7 @@ EFI = Coin("EFI") EGLD = Coin("EGLD") ELON = Coin("ELON") ENJ = Coin("ENJ") +ENS = Coin("ENS") EOS = Coin("EOS") EPS = Coin("EPS") ETC = Coin("ETC") @@ -62,6 +63,7 @@ GHST = Coin("GHST") GRT = Coin("GRT") GTC = Coin("GTC") GUSD = Coin("GUSD") +HBAR = Coin("HBAR") HNT = Coin("HNT") HOT = Coin("HOT") HUSD = Coin("HUSD") @@ -114,6 +116,7 @@ REN = Coin("REN") RGT = Coin("RGT") RLC = Coin("RLC") RLY = Coin("RLY") +RNDR = Coin("RNDR") RSR = Coin("RSR") RUNE = Coin("RUNE") RVN = Coin("RVN") From d5d175198f32f199111346038a9ca3d006a7d48b Mon Sep 17 00:00:00 2001 From: Morty Space Date: Tue, 16 Nov 2021 00:02:17 +0000 Subject: [PATCH 05/15] [JOB] Updated API pairs and coins --- src/cryptocom/exchange/pairs.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/cryptocom/exchange/pairs.py b/src/cryptocom/exchange/pairs.py index 6341485..ff344b8 100644 --- a/src/cryptocom/exchange/pairs.py +++ b/src/cryptocom/exchange/pairs.py @@ -61,6 +61,7 @@ CTSI_USDT = Pair("CTSI_USDT", price_precision=5, quantity_precision=1) DAI_CRO = Pair("DAI_CRO", price_precision=3, quantity_precision=3) DAI_USDC = Pair("DAI_USDC", price_precision=4, quantity_precision=2) DAI_USDT = Pair("DAI_USDT", price_precision=4, quantity_precision=3) +DAR_USDT = Pair("DAR_USDT", price_precision=4, quantity_precision=2) DERC_USDT = Pair("DERC_USDT", price_precision=4, quantity_precision=2) DOGE_BTC = Pair("DOGE_BTC", price_precision=10, quantity_precision=1) DOGE_CRO = Pair("DOGE_CRO", price_precision=4, quantity_precision=2) @@ -81,6 +82,7 @@ ENJ_BTC = Pair("ENJ_BTC", price_precision=9, quantity_precision=2) ENJ_CRO = Pair("ENJ_CRO", price_precision=3, quantity_precision=2) ENJ_USDC = Pair("ENJ_USDC", price_precision=4, quantity_precision=2) ENJ_USDT = Pair("ENJ_USDT", price_precision=5, quantity_precision=1) +ENS_USDT = Pair("ENS_USDT", price_precision=3, quantity_precision=3) EOS_BTC = Pair("EOS_BTC", price_precision=8, quantity_precision=2) EOS_USDT = Pair("EOS_USDT", price_precision=4, quantity_precision=2) EPS_USDT = Pair("EPS_USDT", price_precision=5, quantity_precision=1) @@ -183,12 +185,15 @@ RARI_USDC = Pair("RARI_USDC", price_precision=3, quantity_precision=3) RARI_USDT = Pair("RARI_USDT", price_precision=3, quantity_precision=3) REN_CRO = Pair("REN_CRO", price_precision=4, quantity_precision=2) REN_USDT = Pair("REN_USDT", price_precision=5, quantity_precision=2) +RGT_USDC = Pair("RGT_USDC", price_precision=3, quantity_precision=3) RLC_USDT = Pair("RLC_USDT", price_precision=3, quantity_precision=3) RLY_USDT = Pair("RLY_USDT", price_precision=5, quantity_precision=2) +RNDR_USDT = Pair("RNDR_USDT", price_precision=4, quantity_precision=2) RSR_USDT = Pair("RSR_USDT", price_precision=5, quantity_precision=1) RUNE_USDT = Pair("RUNE_USDT", price_precision=3, quantity_precision=3) SAND_USDT = Pair("SAND_USDT", price_precision=5, quantity_precision=2) SC_USDT = Pair("SC_USDT", price_precision=6, quantity_precision=0) +SDN_USDT = Pair("SDN_USDT", price_precision=4, quantity_precision=2) SHIB_USDC = Pair("SHIB_USDC", price_precision=9, quantity_precision=0) SHIB_USDT = Pair("SHIB_USDT", price_precision=9, quantity_precision=0) SKL_USDT = Pair("SKL_USDT", price_precision=5, quantity_precision=2) @@ -205,6 +210,7 @@ SUSHI_USDT = Pair("SUSHI_USDT", price_precision=3, quantity_precision=3) TFUEL_USDT = Pair("TFUEL_USDT", price_precision=5, quantity_precision=1) THETA_USDT = Pair("THETA_USDT", price_precision=4, quantity_precision=2) TRB_USDT = Pair("TRB_USDT", price_precision=3, quantity_precision=3) +TRIBE_USDT = Pair("TRIBE_USDT", price_precision=4, quantity_precision=2) TRU_USDT = Pair("TRU_USDT", price_precision=5, quantity_precision=1) UMA_USDT = Pair("UMA_USDT", price_precision=3, quantity_precision=4) UNI_CRO = Pair("UNI_CRO", price_precision=3, quantity_precision=2) @@ -216,6 +222,7 @@ VET_CRO = Pair("VET_CRO", price_precision=4, quantity_precision=0) VET_USDC = Pair("VET_USDC", price_precision=6, quantity_precision=0) VET_USDT = Pair("VET_USDT", price_precision=6, quantity_precision=0) VTHO_USDT = Pair("VTHO_USDT", price_precision=6, quantity_precision=0) +VVS_USDC = Pair("VVS_USDC", price_precision=8, quantity_precision=0) WAVES_USDT = Pair("WAVES_USDT", price_precision=3, quantity_precision=3) WAXP_USDT = Pair("WAXP_USDT", price_precision=5, quantity_precision=1) WBTC_BTC = Pair("WBTC_BTC", price_precision=4, quantity_precision=6) From b4db756df5a8c928aa81421cd9a7016d1626fc40 Mon Sep 17 00:00:00 2001 From: Morty Space Date: Thu, 18 Nov 2021 00:02:13 +0000 Subject: [PATCH 06/15] [JOB] Updated API pairs and coins --- src/cryptocom/exchange/coins.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/cryptocom/exchange/coins.py b/src/cryptocom/exchange/coins.py index cb96926..ab3fae8 100644 --- a/src/cryptocom/exchange/coins.py +++ b/src/cryptocom/exchange/coins.py @@ -19,6 +19,7 @@ BAND = Coin("BAND") BAT = Coin("BAT") BCH = Coin("BCH") BNT = Coin("BNT") +BOBA = Coin("BOBA") BOSON = Coin("BOSON") BRZ = Coin("BRZ") BTC = Coin("BTC") @@ -73,6 +74,7 @@ ILV = Coin("ILV") INJ = Coin("INJ") IOTX = Coin("IOTX") IQ = Coin("IQ") +JASMY = Coin("JASMY") KAVA = Coin("KAVA") KEEP = Coin("KEEP") KLAY = Coin("KLAY") @@ -113,6 +115,7 @@ QUICK = Coin("QUICK") RAD = Coin("RAD") RARI = Coin("RARI") REN = Coin("REN") +REQ = Coin("REQ") RGT = Coin("RGT") RLC = Coin("RLC") RLY = Coin("RLY") From 2d53f68852f48efaea23ceef76ee9cd926672d91 Mon Sep 17 00:00:00 2001 From: Morty Space Date: Sat, 20 Nov 2021 00:02:18 +0000 Subject: [PATCH 07/15] [JOB] Updated API pairs and coins --- src/cryptocom/exchange/pairs.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/cryptocom/exchange/pairs.py b/src/cryptocom/exchange/pairs.py index ff344b8..5c2cfdf 100644 --- a/src/cryptocom/exchange/pairs.py +++ b/src/cryptocom/exchange/pairs.py @@ -110,6 +110,7 @@ GTC_USDT = Pair("GTC_USDT", price_precision=3, quantity_precision=3) HNT_USDT = Pair("HNT_USDT", price_precision=3, quantity_precision=3) HOT_CRO = Pair("HOT_CRO", price_precision=5, quantity_precision=0) HOT_USDT = Pair("HOT_USDT", price_precision=6, quantity_precision=0) +ICP_BTC = Pair("ICP_BTC", price_precision=8, quantity_precision=0) ICP_USDT = Pair("ICP_USDT", price_precision=3, quantity_precision=3) ICX_BTC = Pair("ICX_BTC", price_precision=8, quantity_precision=1) ICX_CRO = Pair("ICX_CRO", price_precision=3, quantity_precision=0) @@ -117,6 +118,7 @@ ICX_USDT = Pair("ICX_USDT", price_precision=4, quantity_precision=2) ILV_USDT = Pair("ILV_USDT", price_precision=2, quantity_precision=4) INJ_USDT = Pair("INJ_USDT", price_precision=3, quantity_precision=3) IOTX_USDC = Pair("IOTX_USDC", price_precision=6, quantity_precision=0) +IOTX_USDT = Pair("IOTX_USDT", price_precision=5, quantity_precision=1) IQ_USDT = Pair("IQ_USDT", price_precision=6, quantity_precision=0) KAVA_USDT = Pair("KAVA_USDT", price_precision=4, quantity_precision=2) KEEP_USDT = Pair("KEEP_USDT", price_precision=5, quantity_precision=1) @@ -185,6 +187,7 @@ RARI_USDC = Pair("RARI_USDC", price_precision=3, quantity_precision=3) RARI_USDT = Pair("RARI_USDT", price_precision=3, quantity_precision=3) REN_CRO = Pair("REN_CRO", price_precision=4, quantity_precision=2) REN_USDT = Pair("REN_USDT", price_precision=5, quantity_precision=2) +REQ_USDT = Pair("REQ_USDT", price_precision=5, quantity_precision=1) RGT_USDC = Pair("RGT_USDC", price_precision=3, quantity_precision=3) RLC_USDT = Pair("RLC_USDT", price_precision=3, quantity_precision=3) RLY_USDT = Pair("RLY_USDT", price_precision=5, quantity_precision=2) @@ -226,6 +229,7 @@ VVS_USDC = Pair("VVS_USDC", price_precision=8, quantity_precision=0) WAVES_USDT = Pair("WAVES_USDT", price_precision=3, quantity_precision=3) WAXP_USDT = Pair("WAXP_USDT", price_precision=5, quantity_precision=1) WBTC_BTC = Pair("WBTC_BTC", price_precision=4, quantity_precision=6) +WBTC_USDC = Pair("WBTC_USDC", price_precision=2, quantity_precision=6) WBTC_USDT = Pair("WBTC_USDT", price_precision=2, quantity_precision=6) XLM_BTC = Pair("XLM_BTC", price_precision=9, quantity_precision=0) XLM_CRO = Pair("XLM_CRO", price_precision=3, quantity_precision=2) From 19b3e4401bd1c7cc9ade1d251a235e73be0cc1b8 Mon Sep 17 00:00:00 2001 From: Morty Space Date: Wed, 24 Nov 2021 00:02:50 +0000 Subject: [PATCH 08/15] [JOB] Updated API pairs and coins --- src/cryptocom/exchange/coins.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/cryptocom/exchange/coins.py b/src/cryptocom/exchange/coins.py index ab3fae8..4d2aabe 100644 --- a/src/cryptocom/exchange/coins.py +++ b/src/cryptocom/exchange/coins.py @@ -61,6 +61,7 @@ FTM = Coin("FTM") FXS = Coin("FXS") GALA = Coin("GALA") GHST = Coin("GHST") +GLM = Coin("GLM") GRT = Coin("GRT") GTC = Coin("GTC") GUSD = Coin("GUSD") @@ -109,6 +110,7 @@ PENDLE = Coin("PENDLE") PERP = Coin("PERP") PLA = Coin("PLA") POLY = Coin("POLY") +QI = Coin("QI") QNT = Coin("QNT") QTUM = Coin("QTUM") QUICK = Coin("QUICK") From a1592fdc3d7b1683cc9089ee4548f02365e28cbd Mon Sep 17 00:00:00 2001 From: Morty Space Date: Fri, 26 Nov 2021 00:03:41 +0000 Subject: [PATCH 09/15] [JOB] Updated API pairs and coins --- src/cryptocom/exchange/pairs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cryptocom/exchange/pairs.py b/src/cryptocom/exchange/pairs.py index 5c2cfdf..fba8ad4 100644 --- a/src/cryptocom/exchange/pairs.py +++ b/src/cryptocom/exchange/pairs.py @@ -104,6 +104,7 @@ FTM_USDT = Pair("FTM_USDT", price_precision=4, quantity_precision=2) FXS_USDC = Pair("FXS_USDC", price_precision=4, quantity_precision=2) GALA_USDT = Pair("GALA_USDT", price_precision=6, quantity_precision=0) GHST_USDC = Pair("GHST_USDC", price_precision=4, quantity_precision=2) +GLM_USDT = Pair("GLM_USDT", price_precision=5, quantity_precision=1) GRT_CRO = Pair("GRT_CRO", price_precision=3, quantity_precision=2) GRT_USDT = Pair("GRT_USDT", price_precision=5, quantity_precision=2) GTC_USDT = Pair("GTC_USDT", price_precision=3, quantity_precision=3) From 6734410470615298fe3b3de1923d31e71d204d5a Mon Sep 17 00:00:00 2001 From: Irishery Date: Fri, 26 Nov 2021 23:34:41 +0200 Subject: [PATCH 10/15] update RateLimiter logic --- src/cryptocom/exchange/__init__.py | 1 + src/cryptocom/exchange/api.py | 19 +++----- src/cryptocom/exchange/rate_limiter.py | 60 +++++++++++++++++++++----- tests/test_api.py | 13 ++++-- 4 files changed, 65 insertions(+), 28 deletions(-) diff --git a/src/cryptocom/exchange/__init__.py b/src/cryptocom/exchange/__init__.py index 30a3ba4..947080a 100644 --- a/src/cryptocom/exchange/__init__.py +++ b/src/cryptocom/exchange/__init__.py @@ -9,6 +9,7 @@ from .structs import ( from .market import Exchange from .private import Account from .api import ApiError, ApiProvider +from .rate_limiter import RateLimiterError, RateLimiter from . import pairs, coins if platform.system() == 'Windows': diff --git a/src/cryptocom/exchange/api.py b/src/cryptocom/exchange/api.py index 8bb339d..4c395ab 100644 --- a/src/cryptocom/exchange/api.py +++ b/src/cryptocom/exchange/api.py @@ -50,6 +50,8 @@ class ApiProvider: 'private/margin/get-order-history': (1, 1) } + self.rate_limiter = RateLimiter(self.limits, 1) + # NOTE: do not change this, due to crypto.com rate-limits # TODO: add more strict settings, req/per second or milliseconds @@ -97,27 +99,16 @@ class ApiProvider: async def request(self, method, path, params=None, data=None, sign=False): original_data = data timeout = aiohttp.ClientTimeout(total=self.timeout) - request_type = path.split('/')[0] - - if not (path in self.limits.keys()) and request_type == 'public': - rate_limit, period = 100, 1 - elif not (path in self.limits.keys()) and request_type == 'private': - rate_limit, period = 3, 0.1 - elif not (path in self.limits.keys()): - raise ApiError(f'Wrong path: {path}') - else: - rate_limit, period = self.limits[path] - rate_limiter = RateLimiter(rate_limit=rate_limit, period=period, - concurrency_limit=1) + self.rate_limiter.set_config(path) for count in range(self.retries + 1): if sign: data = self._sign(path, original_data) try: - async with rate_limiter: + async with self.rate_limiter: async with aiohttp.ClientSession(timeout=timeout) as session: - async with rate_limiter.throttle(): + async with self.rate_limiter.throttle(): resp = await session.request( method, urljoin(self.root_url, path), params=params, json=data, diff --git a/src/cryptocom/exchange/rate_limiter.py b/src/cryptocom/exchange/rate_limiter.py index dc58df6..38b3cb1 100644 --- a/src/cryptocom/exchange/rate_limiter.py +++ b/src/cryptocom/exchange/rate_limiter.py @@ -1,26 +1,64 @@ import asyncio import math import time -import traceback from contextlib import asynccontextmanager +class RateLimiterError(Exception): + pass + + class RateLimiter: def __init__(self, - rate_limit: int, - period: float or int, # takes seconds - concurrency_limit: int) -> None: - if not rate_limit or rate_limit < 1: - raise ValueError('rate limit must be non zero positive number') + limits, + concurrency_limit) -> None: + if not concurrency_limit or concurrency_limit < 1: raise ValueError('concurrent limit must be non zero positive number') + + self.rate_limit = int + self.period = float or int # takes seconds + self.tokens_queue = object # asyncio.Queue expecting + self.tokens_consumer_task = object # asyncio.create_task expecting + self.semaphore = object # asyncio.Semaphore expecting + + self.config_setted = False + self.concurrency_limit = concurrency_limit + self.limits = limits + + def get_url_config_data(self, url): + request_type = url.split('/')[0] + + if not(url in self.limits.keys()): + + if request_type == 'public': + rate_limit, period = 100, 1 + + elif request_type == 'private': + rate_limit, period = 3, 0.1 + + elif not (url in self.limits.keys()): + raise RateLimiterError(f'Wrong path: {url}') + + else: + rate_limit, period = self.limits[url] + + return rate_limit, period + + def set_config(self, url): + rate_limit, period = self.get_url_config_data(url) + + if not rate_limit or rate_limit < 1: + raise ValueError('rate limit must be non zero positive number') self.rate_limit = rate_limit self.period = period self.tokens_queue = asyncio.Queue(rate_limit) self.tokens_consumer_task = asyncio.create_task(self.consume_tokens()) - self.semaphore = asyncio.Semaphore(concurrency_limit) + self.semaphore = asyncio.Semaphore(self.concurrency_limit) + + self.config_setted = True async def add_token(self) -> None: await self.tokens_queue.put(1) @@ -67,6 +105,9 @@ class RateLimiter: @asynccontextmanager async def throttle(self): + if not self.config_setted: + raise RateLimiterError('Config is not setted. You need to set it via set_config() before throttling') + await self.semaphore.acquire() await self.add_token() try: @@ -80,7 +121,6 @@ class RateLimiter: async def __aexit__(self, exc_type, exc_val, exc_tb): if exc_type: pass - # print(traceback.format_exc()) await self.close() @@ -90,9 +130,7 @@ class RateLimiter: self.tokens_consumer_task.cancel() await self.tokens_consumer_task except asyncio.CancelledError: - # print(traceback.format_exc()) pass - except Exception as e: - # print(traceback.format_exc()) + except Exception: raise diff --git a/tests/test_api.py b/tests/test_api.py index aff2ce8..8b7d4fb 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,8 +1,11 @@ import os import time import pytest +import aiohttp +import asyncio import cryptocom.exchange as cro +from cryptocom.exchange import rate_limiter def test_timeframe(): @@ -49,11 +52,11 @@ def test_api_args(monkeypatch): async def test_wrong_api_response(): api = cro.ApiProvider(from_env=True) - with pytest.raises(cro.ApiError): + with pytest.raises(cro.RateLimiterError): await api.get('somepath') api = cro.ApiProvider(auth_required=False) - with pytest.raises(cro.ApiError): + with pytest.raises(cro.RateLimiterError): await api.post('account') @@ -62,11 +65,15 @@ async def test_wrong_api_response(): # api = cro.ApiProvider(from_env=True) # account = cro.Account(from_env=True) +# rate_limiter = cro.RateLimiter(cro.api.limits) + # for _ in range(0, 100): -# await account.get_balance() +# print(await account.get_balance()) # for _ in range(0, 100): # await account.get_orders_history(cro.pairs.CRO_USDT, page_size=50) # for _ in range(0, 100): # await api.get('public/get-ticker') + +# async with From c01c75d588da0e2cfb9afe300eec7235a73ec370 Mon Sep 17 00:00:00 2001 From: Morty Space Date: Sat, 27 Nov 2021 00:03:00 +0000 Subject: [PATCH 12/15] [JOB] Updated API pairs and coins --- src/cryptocom/exchange/pairs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cryptocom/exchange/pairs.py b/src/cryptocom/exchange/pairs.py index fba8ad4..1eb7b59 100644 --- a/src/cryptocom/exchange/pairs.py +++ b/src/cryptocom/exchange/pairs.py @@ -178,6 +178,7 @@ PENDLE_USDT = Pair("PENDLE_USDT", price_precision=5, quantity_precision=1) PERP_USDT = Pair("PERP_USDT", price_precision=3, quantity_precision=3) PLA_USDT = Pair("PLA_USDT", price_precision=5, quantity_precision=1) POLY_USDC = Pair("POLY_USDC", price_precision=5, quantity_precision=1) +QI_USDT = Pair("QI_USDT", price_precision=5, quantity_precision=1) QNT_USDC = Pair("QNT_USDC", price_precision=2, quantity_precision=4) QNT_USDT = Pair("QNT_USDT", price_precision=2, quantity_precision=4) QTUM_CRO = Pair("QTUM_CRO", price_precision=3, quantity_precision=2) From 03dec18b962283539d1e70c317fb866cd126dc85 Mon Sep 17 00:00:00 2001 From: Morty Space Date: Tue, 30 Nov 2021 00:02:47 +0000 Subject: [PATCH 13/15] [JOB] Updated API pairs and coins --- src/cryptocom/exchange/coins.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/cryptocom/exchange/coins.py b/src/cryptocom/exchange/coins.py index 4d2aabe..2f80cf1 100644 --- a/src/cryptocom/exchange/coins.py +++ b/src/cryptocom/exchange/coins.py @@ -20,6 +20,7 @@ BAT = Coin("BAT") BCH = Coin("BCH") BNT = Coin("BNT") BOBA = Coin("BOBA") +BOND = Coin("BOND") BOSON = Coin("BOSON") BRZ = Coin("BRZ") BTC = Coin("BTC") @@ -67,11 +68,13 @@ GTC = Coin("GTC") GUSD = Coin("GUSD") HBAR = Coin("HBAR") HNT = Coin("HNT") +HOD = Coin("HOD") HOT = Coin("HOT") HUSD = Coin("HUSD") ICP = Coin("ICP") ICX = Coin("ICX") ILV = Coin("ILV") +IMX = Coin("IMX") INJ = Coin("INJ") IOTX = Coin("IOTX") IQ = Coin("IQ") @@ -98,6 +101,7 @@ NANO = Coin("NANO") NEAR = Coin("NEAR") NEO = Coin("NEO") NKN = Coin("NKN") +NMR = Coin("NMR") NU = Coin("NU") OCEAN = Coin("OCEAN") OGN = Coin("OGN") From 5b779ece03b7f0acdd26318b0ca987cf26fe2598 Mon Sep 17 00:00:00 2001 From: Morty Space Date: Wed, 1 Dec 2021 00:03:11 +0000 Subject: [PATCH 14/15] [JOB] Updated API pairs and coins --- src/cryptocom/exchange/pairs.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/cryptocom/exchange/pairs.py b/src/cryptocom/exchange/pairs.py index 1eb7b59..98447d8 100644 --- a/src/cryptocom/exchange/pairs.py +++ b/src/cryptocom/exchange/pairs.py @@ -102,6 +102,7 @@ FORTH_USDT = Pair("FORTH_USDT", price_precision=3, quantity_precision=3) FTM_BTC = Pair("FTM_BTC", price_precision=9, quantity_precision=2) FTM_USDT = Pair("FTM_USDT", price_precision=4, quantity_precision=2) FXS_USDC = Pair("FXS_USDC", price_precision=4, quantity_precision=2) +GALA_BTC = Pair("GALA_BTC", price_precision=9, quantity_precision=1) GALA_USDT = Pair("GALA_USDT", price_precision=6, quantity_precision=0) GHST_USDC = Pair("GHST_USDC", price_precision=4, quantity_precision=2) GLM_USDT = Pair("GLM_USDT", price_precision=5, quantity_precision=1) @@ -116,6 +117,7 @@ ICP_USDT = Pair("ICP_USDT", price_precision=3, quantity_precision=3) ICX_BTC = Pair("ICX_BTC", price_precision=8, quantity_precision=1) ICX_CRO = Pair("ICX_CRO", price_precision=3, quantity_precision=0) ICX_USDT = Pair("ICX_USDT", price_precision=4, quantity_precision=2) +ILV_BTC = Pair("ILV_BTC", price_precision=6, quantity_precision=4) ILV_USDT = Pair("ILV_USDT", price_precision=2, quantity_precision=4) INJ_USDT = Pair("INJ_USDT", price_precision=3, quantity_precision=3) IOTX_USDC = Pair("IOTX_USDC", price_precision=6, quantity_precision=0) From 195c47d7320d3b939cf95664eb3b5e6f55e25f4c Mon Sep 17 00:00:00 2001 From: Irishery Date: Thu, 2 Dec 2021 18:56:40 +0200 Subject: [PATCH 15/15] refactored rate_limiting - switched to limiter library --- src/cryptocom/exchange/__init__.py | 1 - src/cryptocom/exchange/api.py | 58 +++++++---- src/cryptocom/exchange/rate_limiter.py | 136 ------------------------- tests/test_api.py | 63 ++++++++---- 4 files changed, 82 insertions(+), 176 deletions(-) delete mode 100644 src/cryptocom/exchange/rate_limiter.py diff --git a/src/cryptocom/exchange/__init__.py b/src/cryptocom/exchange/__init__.py index 947080a..30a3ba4 100644 --- a/src/cryptocom/exchange/__init__.py +++ b/src/cryptocom/exchange/__init__.py @@ -9,7 +9,6 @@ from .structs import ( from .market import Exchange from .private import Account from .api import ApiError, ApiProvider -from .rate_limiter import RateLimiterError, RateLimiter from . import pairs, coins if platform.system() == 'Windows': diff --git a/src/cryptocom/exchange/api.py b/src/cryptocom/exchange/api.py index 4c395ab..17aa007 100644 --- a/src/cryptocom/exchange/api.py +++ b/src/cryptocom/exchange/api.py @@ -1,5 +1,6 @@ import os import json +from re import S import time import hmac import random @@ -7,7 +8,8 @@ import asyncio import hashlib from urllib.parse import urljoin -from .rate_limiter import RateLimiter +from aiolimiter import AsyncLimiter + import aiohttp @@ -32,6 +34,8 @@ class ApiProvider: self.ws_root_url = ws_root_url self.timeout = timeout self.retries = retries + self.limiter = AsyncLimiter(1, 1) + self.last_request = '' self.limits = { # method: (req_limit, period) 'private/create-order': (15, 0.1), @@ -50,11 +54,6 @@ class ApiProvider: 'private/margin/get-order-history': (1, 1) } - self.rate_limiter = RateLimiter(self.limits, 1) - - # NOTE: do not change this, due to crypto.com rate-limits - # TODO: add more strict settings, req/per second or milliseconds - if not auth_required: return @@ -96,29 +95,48 @@ class ApiProvider: ).hexdigest() return data + def set_limit(self, url): + + if not(url in self.limits.keys()): + if url.startswith('private'): + rate_limit, period = 3, 0.1 + + elif url.startswith('public'): + rate_limit, period = 100, 1 + + else: + raise ApiError(f'Wrong path: {url}') + + else: + rate_limit, period = self.limits[url] + + self.limiter.max_rate = rate_limit + self.limiter.time_period = period + async def request(self, method, path, params=None, data=None, sign=False): original_data = data timeout = aiohttp.ClientTimeout(total=self.timeout) - self.rate_limiter.set_config(path) + if not (path == self.last_request): + self.set_limit(path) + self.last_request = path for count in range(self.retries + 1): if sign: data = self._sign(path, original_data) try: - async with self.rate_limiter: - async with aiohttp.ClientSession(timeout=timeout) as session: - async with self.rate_limiter.throttle(): - resp = await session.request( - method, urljoin(self.root_url, path), - params=params, json=data, - headers={'content-type': 'application/json'} - ) - resp_json = await resp.json() - if resp.status != 200: - raise ApiError( - f"Error: {resp_json}. " - f"Status: {resp.status}. Json params: {data}") + async with aiohttp.ClientSession(timeout=timeout) as session: + async with self.limiter: + resp = await session.request( + method, urljoin(self.root_url, path), + params=params, json=data, + headers={'content-type': 'application/json'} + ) + resp_json = await resp.json() + if resp.status != 200: + raise ApiError( + f"Error: {resp_json}. " + f"Status: {resp.status}. Json params: {data}") except aiohttp.ClientConnectorError: raise ApiError(f"Cannot connect to host {self.root_url}") except asyncio.TimeoutError: diff --git a/src/cryptocom/exchange/rate_limiter.py b/src/cryptocom/exchange/rate_limiter.py deleted file mode 100644 index 38b3cb1..0000000 --- a/src/cryptocom/exchange/rate_limiter.py +++ /dev/null @@ -1,136 +0,0 @@ -import asyncio -import math -import time - -from contextlib import asynccontextmanager - - -class RateLimiterError(Exception): - pass - - -class RateLimiter: - def __init__(self, - limits, - concurrency_limit) -> None: - - if not concurrency_limit or concurrency_limit < 1: - raise ValueError('concurrent limit must be non zero positive number') - - self.rate_limit = int - self.period = float or int # takes seconds - self.tokens_queue = object # asyncio.Queue expecting - self.tokens_consumer_task = object # asyncio.create_task expecting - self.semaphore = object # asyncio.Semaphore expecting - - self.config_setted = False - self.concurrency_limit = concurrency_limit - self.limits = limits - - def get_url_config_data(self, url): - request_type = url.split('/')[0] - - if not(url in self.limits.keys()): - - if request_type == 'public': - rate_limit, period = 100, 1 - - elif request_type == 'private': - rate_limit, period = 3, 0.1 - - elif not (url in self.limits.keys()): - raise RateLimiterError(f'Wrong path: {url}') - - else: - rate_limit, period = self.limits[url] - - return rate_limit, period - - def set_config(self, url): - rate_limit, period = self.get_url_config_data(url) - - if not rate_limit or rate_limit < 1: - raise ValueError('rate limit must be non zero positive number') - - self.rate_limit = rate_limit - self.period = period - self.tokens_queue = asyncio.Queue(rate_limit) - self.tokens_consumer_task = asyncio.create_task(self.consume_tokens()) - self.semaphore = asyncio.Semaphore(self.concurrency_limit) - - self.config_setted = True - - async def add_token(self) -> None: - await self.tokens_queue.put(1) - return None - - async def consume_tokens(self): - try: - consumption_rate = self.period / self.rate_limit - last_consumption_time = 0 - - while True: - if self.tokens_queue.empty(): - await asyncio.sleep(consumption_rate) - continue - - current_consumption_time = time.monotonic() - total_tokens = self.tokens_queue.qsize() - tokens_to_consume = self.get_tokens_amount_to_consume( - consumption_rate, - current_consumption_time, - last_consumption_time, - total_tokens - ) - - for _ in range(0, tokens_to_consume): - self.tokens_queue.get_nowait() - - last_consumption_time = time.monotonic() - - await asyncio.sleep(consumption_rate) - except asyncio.CancelledError: - raise - except Exception as e: - raise - - @staticmethod - def get_tokens_amount_to_consume(consumption_rate, current_consumption_time, - last_consumption_time, total_tokens): - time_from_last_consumption = current_consumption_time - last_consumption_time - calculated_tokens_to_consume = math.floor(time_from_last_consumption / consumption_rate) - tokens_to_consume = min(total_tokens, calculated_tokens_to_consume) - - return tokens_to_consume - - @asynccontextmanager - async def throttle(self): - if not self.config_setted: - raise RateLimiterError('Config is not setted. You need to set it via set_config() before throttling') - - await self.semaphore.acquire() - await self.add_token() - try: - yield - finally: - self.semaphore.release() - - async def __aenter__(self): - return self - - async def __aexit__(self, exc_type, exc_val, exc_tb): - if exc_type: - pass - - await self.close() - - async def close(self) -> None: - if self.tokens_consumer_task and not self.tokens_consumer_task.cancelled(): - try: - self.tokens_consumer_task.cancel() - await self.tokens_consumer_task - except asyncio.CancelledError: - pass - - except Exception: - raise diff --git a/tests/test_api.py b/tests/test_api.py index 8b7d4fb..aad16da 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,11 +1,8 @@ import os import time import pytest -import aiohttp -import asyncio import cryptocom.exchange as cro -from cryptocom.exchange import rate_limiter def test_timeframe(): @@ -52,28 +49,56 @@ def test_api_args(monkeypatch): async def test_wrong_api_response(): api = cro.ApiProvider(from_env=True) - with pytest.raises(cro.RateLimiterError): + with pytest.raises(cro.ApiError): await api.get('somepath') api = cro.ApiProvider(auth_required=False) - with pytest.raises(cro.RateLimiterError): + with pytest.raises(cro.ApiError): await api.post('account') -# @pytest.mark.asyncio -# async def test_api_rate_limits(): -# api = cro.ApiProvider(from_env=True) -# account = cro.Account(from_env=True) +@pytest.mark.asyncio +async def test_api_rate_limits(): + api = cro.ApiProvider(from_env=True) + pair = cro.pairs.CRO_USDT + + page = 0 + page_size = 50 + + params = {'page_size': page_size, 'page': page} + + if pair: + params['instrument_name'] = pair.name + + start_time = time.time() + await api.post('private/get-order-history', {'params': params}) + await api.post('private/get-order-history', {'params': params}) + finish_time = (time.time() - start_time) + + assert finish_time > 1 + + start_time = time.time() + await api.post('private/get-order-history', {'params': params}) + await api.post('private/get-order-history', {'params': params}) + await api.post('private/get-order-history', {'params': params}) + await api.post('private/get-order-history', {'params': params}) + + finish_time = time.time() - start_time + assert finish_time > 4 + + start_time = time.time() + await api.get('public/get-instruments') + await api.get('public/get-instruments') + await api.get('public/get-instruments') + await api.get('public/get-instruments') -# rate_limiter = cro.RateLimiter(cro.api.limits) + finish_time = time.time() - start_time + assert finish_time < 4 -# for _ in range(0, 100): -# print(await account.get_balance()) + start_time = time.time() + await api.post('private/get-order-history', {'params': params}) + await api.post('private/get-order-history', {'params': params}) + await api.post('private/get-order-history', {'params': params}) -# for _ in range(0, 100): -# await account.get_orders_history(cro.pairs.CRO_USDT, page_size=50) - -# for _ in range(0, 100): -# await api.get('public/get-ticker') - -# async with + finish_time = time.time() - start_time + assert finish_time > 3