diff --git a/azure-pipelines.yml b/azure-pipelines.yml index bef7044..d5995be 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -29,7 +29,8 @@ strategy: python.version: '3.8' Python39: python.version: '3.9' - + Python310: + python.version: '3.10' steps: - checkout: self persistCredentials: true @@ -58,6 +59,7 @@ steps: git commit -m "[JOB] Updated API pairs and coins" git push origin HEAD:master displayName: 'Update API structs' + condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master')) - script: | cd docs && make doctest && cd .. diff --git a/src/cryptocom/exchange/api.py b/src/cryptocom/exchange/api.py index 8bb339d..17aa007 100644 --- a/src/cryptocom/exchange/api.py +++ b/src/cryptocom/exchange/api.py @@ -1,5 +1,6 @@ import os import json +from re import S import time import hmac import random @@ -7,7 +8,8 @@ import asyncio import hashlib from urllib.parse import urljoin -from .rate_limiter import RateLimiter +from aiolimiter import AsyncLimiter + import aiohttp @@ -32,6 +34,8 @@ class ApiProvider: self.ws_root_url = ws_root_url self.timeout = timeout self.retries = retries + self.limiter = AsyncLimiter(1, 1) + self.last_request = '' self.limits = { # method: (req_limit, period) 'private/create-order': (15, 0.1), @@ -50,9 +54,6 @@ class ApiProvider: 'private/margin/get-order-history': (1, 1) } - # NOTE: do not change this, due to crypto.com rate-limits - # TODO: add more strict settings, req/per second or milliseconds - if not auth_required: return @@ -94,40 +95,48 @@ class ApiProvider: ).hexdigest() return data + def set_limit(self, url): + + if not(url in self.limits.keys()): + if url.startswith('private'): + rate_limit, period = 3, 0.1 + + elif url.startswith('public'): + rate_limit, period = 100, 1 + + else: + raise ApiError(f'Wrong path: {url}') + + else: + rate_limit, period = self.limits[url] + + self.limiter.max_rate = rate_limit + self.limiter.time_period = period + async def request(self, method, path, params=None, data=None, sign=False): original_data = data timeout = aiohttp.ClientTimeout(total=self.timeout) - request_type = path.split('/')[0] - - if not (path in self.limits.keys()) and request_type == 'public': - rate_limit, period = 100, 1 - elif not (path in self.limits.keys()) and request_type == 'private': - rate_limit, period = 3, 0.1 - elif not (path in self.limits.keys()): - raise ApiError(f'Wrong path: {path}') - else: - rate_limit, period = self.limits[path] - rate_limiter = RateLimiter(rate_limit=rate_limit, period=period, - concurrency_limit=1) + if not (path == self.last_request): + self.set_limit(path) + self.last_request = path for count in range(self.retries + 1): if sign: data = self._sign(path, original_data) try: - async with rate_limiter: - async with aiohttp.ClientSession(timeout=timeout) as session: - async with rate_limiter.throttle(): - resp = await session.request( - method, urljoin(self.root_url, path), - params=params, json=data, - headers={'content-type': 'application/json'} - ) - resp_json = await resp.json() - if resp.status != 200: - raise ApiError( - f"Error: {resp_json}. " - f"Status: {resp.status}. Json params: {data}") + async with aiohttp.ClientSession(timeout=timeout) as session: + async with self.limiter: + resp = await session.request( + method, urljoin(self.root_url, path), + params=params, json=data, + headers={'content-type': 'application/json'} + ) + resp_json = await resp.json() + if resp.status != 200: + raise ApiError( + f"Error: {resp_json}. " + f"Status: {resp.status}. Json params: {data}") except aiohttp.ClientConnectorError: raise ApiError(f"Cannot connect to host {self.root_url}") except asyncio.TimeoutError: diff --git a/src/cryptocom/exchange/coins.py b/src/cryptocom/exchange/coins.py index 13d0487..2f80cf1 100644 --- a/src/cryptocom/exchange/coins.py +++ b/src/cryptocom/exchange/coins.py @@ -19,6 +19,8 @@ BAND = Coin("BAND") BAT = Coin("BAT") BCH = Coin("BCH") BNT = Coin("BNT") +BOBA = Coin("BOBA") +BOND = Coin("BOND") BOSON = Coin("BOSON") BRZ = Coin("BRZ") BTC = Coin("BTC") @@ -46,6 +48,7 @@ EFI = Coin("EFI") EGLD = Coin("EGLD") ELON = Coin("ELON") ENJ = Coin("ENJ") +ENS = Coin("ENS") EOS = Coin("EOS") EPS = Coin("EPS") ETC = Coin("ETC") @@ -59,18 +62,23 @@ FTM = Coin("FTM") FXS = Coin("FXS") GALA = Coin("GALA") GHST = Coin("GHST") +GLM = Coin("GLM") GRT = Coin("GRT") GTC = Coin("GTC") GUSD = Coin("GUSD") +HBAR = Coin("HBAR") HNT = Coin("HNT") +HOD = Coin("HOD") HOT = Coin("HOT") HUSD = Coin("HUSD") ICP = Coin("ICP") ICX = Coin("ICX") ILV = Coin("ILV") +IMX = Coin("IMX") INJ = Coin("INJ") IOTX = Coin("IOTX") IQ = Coin("IQ") +JASMY = Coin("JASMY") KAVA = Coin("KAVA") KEEP = Coin("KEEP") KLAY = Coin("KLAY") @@ -93,6 +101,7 @@ NANO = Coin("NANO") NEAR = Coin("NEAR") NEO = Coin("NEO") NKN = Coin("NKN") +NMR = Coin("NMR") NU = Coin("NU") OCEAN = Coin("OCEAN") OGN = Coin("OGN") @@ -105,15 +114,18 @@ PENDLE = Coin("PENDLE") PERP = Coin("PERP") PLA = Coin("PLA") POLY = Coin("POLY") +QI = Coin("QI") QNT = Coin("QNT") QTUM = Coin("QTUM") QUICK = Coin("QUICK") RAD = Coin("RAD") RARI = Coin("RARI") REN = Coin("REN") +REQ = Coin("REQ") RGT = Coin("RGT") RLC = Coin("RLC") RLY = Coin("RLY") +RNDR = Coin("RNDR") RSR = Coin("RSR") RUNE = Coin("RUNE") RVN = Coin("RVN") @@ -143,6 +155,7 @@ USDP = Coin("USDP") USDT = Coin("USDT") VET = Coin("VET") VTHO = Coin("VTHO") +VVS = Coin("VVS") WAVE = Coin("WAVE") WAVES = Coin("WAVES") WAXP = Coin("WAXP") diff --git a/src/cryptocom/exchange/pairs.py b/src/cryptocom/exchange/pairs.py index ba9c3be..98447d8 100644 --- a/src/cryptocom/exchange/pairs.py +++ b/src/cryptocom/exchange/pairs.py @@ -15,6 +15,7 @@ ALGO_USDT = Pair("ALGO_USDT", price_precision=4, quantity_precision=2) ALICE_USDT = Pair("ALICE_USDT", price_precision=3, quantity_precision=3) AMP_USDT = Pair("AMP_USDT", price_precision=5, quantity_precision=1) ANKR_USDT = Pair("ANKR_USDT", price_precision=6, quantity_precision=1) +ARPA_USDC = Pair("ARPA_USDC", price_precision=5, quantity_precision=1) AR_USDC = Pair("AR_USDC", price_precision=3, quantity_precision=3) ATOM_BTC = Pair("ATOM_BTC", price_precision=7, quantity_precision=2) ATOM_CRO = Pair("ATOM_CRO", price_precision=2, quantity_precision=2) @@ -60,6 +61,7 @@ CTSI_USDT = Pair("CTSI_USDT", price_precision=5, quantity_precision=1) DAI_CRO = Pair("DAI_CRO", price_precision=3, quantity_precision=3) DAI_USDC = Pair("DAI_USDC", price_precision=4, quantity_precision=2) DAI_USDT = Pair("DAI_USDT", price_precision=4, quantity_precision=3) +DAR_USDT = Pair("DAR_USDT", price_precision=4, quantity_precision=2) DERC_USDT = Pair("DERC_USDT", price_precision=4, quantity_precision=2) DOGE_BTC = Pair("DOGE_BTC", price_precision=10, quantity_precision=1) DOGE_CRO = Pair("DOGE_CRO", price_precision=4, quantity_precision=2) @@ -80,6 +82,7 @@ ENJ_BTC = Pair("ENJ_BTC", price_precision=9, quantity_precision=2) ENJ_CRO = Pair("ENJ_CRO", price_precision=3, quantity_precision=2) ENJ_USDC = Pair("ENJ_USDC", price_precision=4, quantity_precision=2) ENJ_USDT = Pair("ENJ_USDT", price_precision=5, quantity_precision=1) +ENS_USDT = Pair("ENS_USDT", price_precision=3, quantity_precision=3) EOS_BTC = Pair("EOS_BTC", price_precision=8, quantity_precision=2) EOS_USDT = Pair("EOS_USDT", price_precision=4, quantity_precision=2) EPS_USDT = Pair("EPS_USDT", price_precision=5, quantity_precision=1) @@ -99,21 +102,26 @@ FORTH_USDT = Pair("FORTH_USDT", price_precision=3, quantity_precision=3) FTM_BTC = Pair("FTM_BTC", price_precision=9, quantity_precision=2) FTM_USDT = Pair("FTM_USDT", price_precision=4, quantity_precision=2) FXS_USDC = Pair("FXS_USDC", price_precision=4, quantity_precision=2) +GALA_BTC = Pair("GALA_BTC", price_precision=9, quantity_precision=1) GALA_USDT = Pair("GALA_USDT", price_precision=6, quantity_precision=0) GHST_USDC = Pair("GHST_USDC", price_precision=4, quantity_precision=2) +GLM_USDT = Pair("GLM_USDT", price_precision=5, quantity_precision=1) GRT_CRO = Pair("GRT_CRO", price_precision=3, quantity_precision=2) GRT_USDT = Pair("GRT_USDT", price_precision=5, quantity_precision=2) GTC_USDT = Pair("GTC_USDT", price_precision=3, quantity_precision=3) HNT_USDT = Pair("HNT_USDT", price_precision=3, quantity_precision=3) HOT_CRO = Pair("HOT_CRO", price_precision=5, quantity_precision=0) HOT_USDT = Pair("HOT_USDT", price_precision=6, quantity_precision=0) +ICP_BTC = Pair("ICP_BTC", price_precision=8, quantity_precision=0) ICP_USDT = Pair("ICP_USDT", price_precision=3, quantity_precision=3) ICX_BTC = Pair("ICX_BTC", price_precision=8, quantity_precision=1) ICX_CRO = Pair("ICX_CRO", price_precision=3, quantity_precision=0) ICX_USDT = Pair("ICX_USDT", price_precision=4, quantity_precision=2) +ILV_BTC = Pair("ILV_BTC", price_precision=6, quantity_precision=4) ILV_USDT = Pair("ILV_USDT", price_precision=2, quantity_precision=4) INJ_USDT = Pair("INJ_USDT", price_precision=3, quantity_precision=3) IOTX_USDC = Pair("IOTX_USDC", price_precision=6, quantity_precision=0) +IOTX_USDT = Pair("IOTX_USDT", price_precision=5, quantity_precision=1) IQ_USDT = Pair("IQ_USDT", price_precision=6, quantity_precision=0) KAVA_USDT = Pair("KAVA_USDT", price_precision=4, quantity_precision=2) KEEP_USDT = Pair("KEEP_USDT", price_precision=5, quantity_precision=1) @@ -172,6 +180,7 @@ PENDLE_USDT = Pair("PENDLE_USDT", price_precision=5, quantity_precision=1) PERP_USDT = Pair("PERP_USDT", price_precision=3, quantity_precision=3) PLA_USDT = Pair("PLA_USDT", price_precision=5, quantity_precision=1) POLY_USDC = Pair("POLY_USDC", price_precision=5, quantity_precision=1) +QI_USDT = Pair("QI_USDT", price_precision=5, quantity_precision=1) QNT_USDC = Pair("QNT_USDC", price_precision=2, quantity_precision=4) QNT_USDT = Pair("QNT_USDT", price_precision=2, quantity_precision=4) QTUM_CRO = Pair("QTUM_CRO", price_precision=3, quantity_precision=2) @@ -182,12 +191,16 @@ RARI_USDC = Pair("RARI_USDC", price_precision=3, quantity_precision=3) RARI_USDT = Pair("RARI_USDT", price_precision=3, quantity_precision=3) REN_CRO = Pair("REN_CRO", price_precision=4, quantity_precision=2) REN_USDT = Pair("REN_USDT", price_precision=5, quantity_precision=2) +REQ_USDT = Pair("REQ_USDT", price_precision=5, quantity_precision=1) +RGT_USDC = Pair("RGT_USDC", price_precision=3, quantity_precision=3) RLC_USDT = Pair("RLC_USDT", price_precision=3, quantity_precision=3) RLY_USDT = Pair("RLY_USDT", price_precision=5, quantity_precision=2) +RNDR_USDT = Pair("RNDR_USDT", price_precision=4, quantity_precision=2) RSR_USDT = Pair("RSR_USDT", price_precision=5, quantity_precision=1) RUNE_USDT = Pair("RUNE_USDT", price_precision=3, quantity_precision=3) SAND_USDT = Pair("SAND_USDT", price_precision=5, quantity_precision=2) SC_USDT = Pair("SC_USDT", price_precision=6, quantity_precision=0) +SDN_USDT = Pair("SDN_USDT", price_precision=4, quantity_precision=2) SHIB_USDC = Pair("SHIB_USDC", price_precision=9, quantity_precision=0) SHIB_USDT = Pair("SHIB_USDT", price_precision=9, quantity_precision=0) SKL_USDT = Pair("SKL_USDT", price_precision=5, quantity_precision=2) @@ -197,12 +210,15 @@ SNX_USDT = Pair("SNX_USDT", price_precision=3, quantity_precision=3) SOL_BTC = Pair("SOL_BTC", price_precision=7, quantity_precision=3) SOL_USDC = Pair("SOL_USDC", price_precision=3, quantity_precision=3) SOL_USDT = Pair("SOL_USDT", price_precision=3, quantity_precision=3) +SRM_USDC = Pair("SRM_USDC", price_precision=4, quantity_precision=2) STORJ_USDT = Pair("STORJ_USDT", price_precision=4, quantity_precision=2) STX_USDT = Pair("STX_USDT", price_precision=4, quantity_precision=2) SUSHI_USDT = Pair("SUSHI_USDT", price_precision=3, quantity_precision=3) TFUEL_USDT = Pair("TFUEL_USDT", price_precision=5, quantity_precision=1) THETA_USDT = Pair("THETA_USDT", price_precision=4, quantity_precision=2) TRB_USDT = Pair("TRB_USDT", price_precision=3, quantity_precision=3) +TRIBE_USDT = Pair("TRIBE_USDT", price_precision=4, quantity_precision=2) +TRU_USDT = Pair("TRU_USDT", price_precision=5, quantity_precision=1) UMA_USDT = Pair("UMA_USDT", price_precision=3, quantity_precision=4) UNI_CRO = Pair("UNI_CRO", price_precision=3, quantity_precision=2) UNI_USDC = Pair("UNI_USDC", price_precision=3, quantity_precision=3) @@ -213,9 +229,11 @@ VET_CRO = Pair("VET_CRO", price_precision=4, quantity_precision=0) VET_USDC = Pair("VET_USDC", price_precision=6, quantity_precision=0) VET_USDT = Pair("VET_USDT", price_precision=6, quantity_precision=0) VTHO_USDT = Pair("VTHO_USDT", price_precision=6, quantity_precision=0) +VVS_USDC = Pair("VVS_USDC", price_precision=8, quantity_precision=0) WAVES_USDT = Pair("WAVES_USDT", price_precision=3, quantity_precision=3) WAXP_USDT = Pair("WAXP_USDT", price_precision=5, quantity_precision=1) WBTC_BTC = Pair("WBTC_BTC", price_precision=4, quantity_precision=6) +WBTC_USDC = Pair("WBTC_USDC", price_precision=2, quantity_precision=6) WBTC_USDT = Pair("WBTC_USDT", price_precision=2, quantity_precision=6) XLM_BTC = Pair("XLM_BTC", price_precision=9, quantity_precision=0) XLM_CRO = Pair("XLM_CRO", price_precision=3, quantity_precision=2) @@ -227,6 +245,7 @@ XRP_USDT = Pair("XRP_USDT", price_precision=5, quantity_precision=1) XTZ_BTC = Pair("XTZ_BTC", price_precision=8, quantity_precision=2) XTZ_CRO = Pair("XTZ_CRO", price_precision=3, quantity_precision=2) XTZ_USDT = Pair("XTZ_USDT", price_precision=4, quantity_precision=2) +XYO_USDT = Pair("XYO_USDT", price_precision=6, quantity_precision=0) YFI_BTC = Pair("YFI_BTC", price_precision=4, quantity_precision=6) YFI_CRO = Pair("YFI_CRO", price_precision=2, quantity_precision=6) YFI_USDT = Pair("YFI_USDT", price_precision=2, quantity_precision=6) diff --git a/src/cryptocom/exchange/rate_limiter.py b/src/cryptocom/exchange/rate_limiter.py deleted file mode 100644 index dc58df6..0000000 --- a/src/cryptocom/exchange/rate_limiter.py +++ /dev/null @@ -1,98 +0,0 @@ -import asyncio -import math -import time -import traceback - -from contextlib import asynccontextmanager - - -class RateLimiter: - def __init__(self, - rate_limit: int, - period: float or int, # takes seconds - concurrency_limit: int) -> None: - if not rate_limit or rate_limit < 1: - raise ValueError('rate limit must be non zero positive number') - if not concurrency_limit or concurrency_limit < 1: - raise ValueError('concurrent limit must be non zero positive number') - - self.rate_limit = rate_limit - self.period = period - self.tokens_queue = asyncio.Queue(rate_limit) - self.tokens_consumer_task = asyncio.create_task(self.consume_tokens()) - self.semaphore = asyncio.Semaphore(concurrency_limit) - - async def add_token(self) -> None: - await self.tokens_queue.put(1) - return None - - async def consume_tokens(self): - try: - consumption_rate = self.period / self.rate_limit - last_consumption_time = 0 - - while True: - if self.tokens_queue.empty(): - await asyncio.sleep(consumption_rate) - continue - - current_consumption_time = time.monotonic() - total_tokens = self.tokens_queue.qsize() - tokens_to_consume = self.get_tokens_amount_to_consume( - consumption_rate, - current_consumption_time, - last_consumption_time, - total_tokens - ) - - for _ in range(0, tokens_to_consume): - self.tokens_queue.get_nowait() - - last_consumption_time = time.monotonic() - - await asyncio.sleep(consumption_rate) - except asyncio.CancelledError: - raise - except Exception as e: - raise - - @staticmethod - def get_tokens_amount_to_consume(consumption_rate, current_consumption_time, - last_consumption_time, total_tokens): - time_from_last_consumption = current_consumption_time - last_consumption_time - calculated_tokens_to_consume = math.floor(time_from_last_consumption / consumption_rate) - tokens_to_consume = min(total_tokens, calculated_tokens_to_consume) - - return tokens_to_consume - - @asynccontextmanager - async def throttle(self): - await self.semaphore.acquire() - await self.add_token() - try: - yield - finally: - self.semaphore.release() - - async def __aenter__(self): - return self - - async def __aexit__(self, exc_type, exc_val, exc_tb): - if exc_type: - pass - # print(traceback.format_exc()) - - await self.close() - - async def close(self) -> None: - if self.tokens_consumer_task and not self.tokens_consumer_task.cancelled(): - try: - self.tokens_consumer_task.cancel() - await self.tokens_consumer_task - except asyncio.CancelledError: - # print(traceback.format_exc()) - pass - - except Exception as e: - # print(traceback.format_exc()) - raise diff --git a/tests/test_api.py b/tests/test_api.py index aff2ce8..aad16da 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -57,16 +57,48 @@ async def test_wrong_api_response(): await api.post('account') -# @pytest.mark.asyncio -# async def test_api_rate_limits(): -# api = cro.ApiProvider(from_env=True) -# account = cro.Account(from_env=True) - -# for _ in range(0, 100): -# await account.get_balance() - -# for _ in range(0, 100): -# await account.get_orders_history(cro.pairs.CRO_USDT, page_size=50) - -# for _ in range(0, 100): -# await api.get('public/get-ticker') +@pytest.mark.asyncio +async def test_api_rate_limits(): + api = cro.ApiProvider(from_env=True) + pair = cro.pairs.CRO_USDT + + page = 0 + page_size = 50 + + params = {'page_size': page_size, 'page': page} + + if pair: + params['instrument_name'] = pair.name + + start_time = time.time() + await api.post('private/get-order-history', {'params': params}) + await api.post('private/get-order-history', {'params': params}) + finish_time = (time.time() - start_time) + + assert finish_time > 1 + + start_time = time.time() + await api.post('private/get-order-history', {'params': params}) + await api.post('private/get-order-history', {'params': params}) + await api.post('private/get-order-history', {'params': params}) + await api.post('private/get-order-history', {'params': params}) + + finish_time = time.time() - start_time + assert finish_time > 4 + + start_time = time.time() + await api.get('public/get-instruments') + await api.get('public/get-instruments') + await api.get('public/get-instruments') + await api.get('public/get-instruments') + + finish_time = time.time() - start_time + assert finish_time < 4 + + start_time = time.time() + await api.post('private/get-order-history', {'params': params}) + await api.post('private/get-order-history', {'params': params}) + await api.post('private/get-order-history', {'params': params}) + + finish_time = time.time() - start_time + assert finish_time > 3