Merge pull request #3 from Irishery/working_on_rps

Working on rps

azure-pipelines.yml

@@ -29,7 +29,8 @@ strategy:
       python.version: '3.8'
     Python39:
       python.version: '3.9'
-
+    Python310:
+      python.version: '3.10'
 steps:
   - checkout: self
     persistCredentials: true
@@ -58,6 +59,7 @@ steps:
       git commit -m "[JOB] Updated API pairs and coins"
       git push origin HEAD:master
     displayName: 'Update API structs'
+    condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'))
 
   - script: |
       cd docs && make doctest && cd ..

src/cryptocom/exchange/api.py

@@ -1,5 +1,6 @@
 import os
 import json
+from re import S
 import time
 import hmac
 import random
@@ -7,7 +8,8 @@ import asyncio
 import hashlib
 
 from urllib.parse import urljoin
-from .rate_limiter import RateLimiter
+from aiolimiter import AsyncLimiter
+
 
 import aiohttp
 
@@ -32,6 +34,8 @@ class ApiProvider:
         self.ws_root_url = ws_root_url
         self.timeout = timeout
         self.retries = retries
+        self.limiter = AsyncLimiter(1, 1)
+        self.last_request = ''
         self.limits = {
             # method: (req_limit, period)
             'private/create-order': (15, 0.1),
@@ -50,9 +54,6 @@ class ApiProvider:
             'private/margin/get-order-history': (1, 1)
         }
 
-        # NOTE: do not change this, due to crypto.com rate-limits
-        # TODO: add more strict settings, req/per second or milliseconds
-
         if not auth_required:
             return
 
@@ -94,40 +95,48 @@ class ApiProvider:
         ).hexdigest()
         return data
 
+    def set_limit(self, url):
+
+        if not(url in self.limits.keys()):
+            if url.startswith('private'):
+                rate_limit, period = 3, 0.1
+            
+            elif url.startswith('public'):
+                rate_limit, period = 100, 1
+
+            else:
+                raise ApiError(f'Wrong path: {url}')
+
+        else:
+            rate_limit, period = self.limits[url]
+        
+        self.limiter.max_rate = rate_limit
+        self.limiter.time_period = period
+
     async def request(self, method, path, params=None, data=None, sign=False):
         original_data = data
         timeout = aiohttp.ClientTimeout(total=self.timeout)
-        request_type = path.split('/')[0]
-
-        if not (path in self.limits.keys()) and request_type == 'public':
-            rate_limit, period = 100, 1
-        elif not (path in self.limits.keys()) and request_type == 'private':
-            rate_limit, period = 3, 0.1
-        elif not (path in self.limits.keys()):
-            raise ApiError(f'Wrong path: {path}')
-        else:
-            rate_limit, period = self.limits[path]
 
-        rate_limiter = RateLimiter(rate_limit=rate_limit, period=period, 
+        if not (path == self.last_request):
-                                   concurrency_limit=1)
+            self.set_limit(path)
+            self.last_request = path
 
         for count in range(self.retries + 1):
             if sign:
                 data = self._sign(path, original_data)
             try:
-                async with rate_limiter:
+                async with aiohttp.ClientSession(timeout=timeout) as session:
-                    async with aiohttp.ClientSession(timeout=timeout) as session:
+                    async with self.limiter:
-                        async with rate_limiter.throttle():
+                        resp = await session.request(
-                            resp = await session.request(
+                            method, urljoin(self.root_url, path),
-                                method, urljoin(self.root_url, path),
+                            params=params, json=data,
-                                params=params, json=data,
+                            headers={'content-type': 'application/json'}
-                                headers={'content-type': 'application/json'}
+                        )
-                            )
+                        resp_json = await resp.json()
-                            resp_json = await resp.json()
+                        if resp.status != 200:
-                            if resp.status != 200:
+                            raise ApiError(
-                                raise ApiError(
+                                f"Error: {resp_json}. "
-                                    f"Error: {resp_json}. "
+                                f"Status: {resp.status}. Json params: {data}")
-                                    f"Status: {resp.status}. Json params: {data}")
             except aiohttp.ClientConnectorError:
                 raise ApiError(f"Cannot connect to host {self.root_url}")
             except asyncio.TimeoutError:

src/cryptocom/exchange/coins.py

@@ -19,6 +19,8 @@ BAND = Coin("BAND")
 BAT = Coin("BAT")
 BCH = Coin("BCH")
 BNT = Coin("BNT")
+BOBA = Coin("BOBA")
+BOND = Coin("BOND")
 BOSON = Coin("BOSON")
 BRZ = Coin("BRZ")
 BTC = Coin("BTC")
@@ -46,6 +48,7 @@ EFI = Coin("EFI")
 EGLD = Coin("EGLD")
 ELON = Coin("ELON")
 ENJ = Coin("ENJ")
+ENS = Coin("ENS")
 EOS = Coin("EOS")
 EPS = Coin("EPS")
 ETC = Coin("ETC")
@@ -59,18 +62,23 @@ FTM = Coin("FTM")
 FXS = Coin("FXS")
 GALA = Coin("GALA")
 GHST = Coin("GHST")
+GLM = Coin("GLM")
 GRT = Coin("GRT")
 GTC = Coin("GTC")
 GUSD = Coin("GUSD")
+HBAR = Coin("HBAR")
 HNT = Coin("HNT")
+HOD = Coin("HOD")
 HOT = Coin("HOT")
 HUSD = Coin("HUSD")
 ICP = Coin("ICP")
 ICX = Coin("ICX")
 ILV = Coin("ILV")
+IMX = Coin("IMX")
 INJ = Coin("INJ")
 IOTX = Coin("IOTX")
 IQ = Coin("IQ")
+JASMY = Coin("JASMY")
 KAVA = Coin("KAVA")
 KEEP = Coin("KEEP")
 KLAY = Coin("KLAY")
@@ -93,6 +101,7 @@ NANO = Coin("NANO")
 NEAR = Coin("NEAR")
 NEO = Coin("NEO")
 NKN = Coin("NKN")
+NMR = Coin("NMR")
 NU = Coin("NU")
 OCEAN = Coin("OCEAN")
 OGN = Coin("OGN")
@@ -105,15 +114,18 @@ PENDLE = Coin("PENDLE")
 PERP = Coin("PERP")
 PLA = Coin("PLA")
 POLY = Coin("POLY")
+QI = Coin("QI")
 QNT = Coin("QNT")
 QTUM = Coin("QTUM")
 QUICK = Coin("QUICK")
 RAD = Coin("RAD")
 RARI = Coin("RARI")
 REN = Coin("REN")
+REQ = Coin("REQ")
 RGT = Coin("RGT")
 RLC = Coin("RLC")
 RLY = Coin("RLY")
+RNDR = Coin("RNDR")
 RSR = Coin("RSR")
 RUNE = Coin("RUNE")
 RVN = Coin("RVN")
@@ -143,6 +155,7 @@ USDP = Coin("USDP")
 USDT = Coin("USDT")
 VET = Coin("VET")
 VTHO = Coin("VTHO")
+VVS = Coin("VVS")
 WAVE = Coin("WAVE")
 WAVES = Coin("WAVES")
 WAXP = Coin("WAXP")

src/cryptocom/exchange/pairs.py

@@ -15,6 +15,7 @@ ALGO_USDT = Pair("ALGO_USDT", price_precision=4, quantity_precision=2)
 ALICE_USDT = Pair("ALICE_USDT", price_precision=3, quantity_precision=3)
 AMP_USDT = Pair("AMP_USDT", price_precision=5, quantity_precision=1)
 ANKR_USDT = Pair("ANKR_USDT", price_precision=6, quantity_precision=1)
+ARPA_USDC = Pair("ARPA_USDC", price_precision=5, quantity_precision=1)
 AR_USDC = Pair("AR_USDC", price_precision=3, quantity_precision=3)
 ATOM_BTC = Pair("ATOM_BTC", price_precision=7, quantity_precision=2)
 ATOM_CRO = Pair("ATOM_CRO", price_precision=2, quantity_precision=2)
@@ -60,6 +61,7 @@ CTSI_USDT = Pair("CTSI_USDT", price_precision=5, quantity_precision=1)
 DAI_CRO = Pair("DAI_CRO", price_precision=3, quantity_precision=3)
 DAI_USDC = Pair("DAI_USDC", price_precision=4, quantity_precision=2)
 DAI_USDT = Pair("DAI_USDT", price_precision=4, quantity_precision=3)
+DAR_USDT = Pair("DAR_USDT", price_precision=4, quantity_precision=2)
 DERC_USDT = Pair("DERC_USDT", price_precision=4, quantity_precision=2)
 DOGE_BTC = Pair("DOGE_BTC", price_precision=10, quantity_precision=1)
 DOGE_CRO = Pair("DOGE_CRO", price_precision=4, quantity_precision=2)
@@ -80,6 +82,7 @@ ENJ_BTC = Pair("ENJ_BTC", price_precision=9, quantity_precision=2)
 ENJ_CRO = Pair("ENJ_CRO", price_precision=3, quantity_precision=2)
 ENJ_USDC = Pair("ENJ_USDC", price_precision=4, quantity_precision=2)
 ENJ_USDT = Pair("ENJ_USDT", price_precision=5, quantity_precision=1)
+ENS_USDT = Pair("ENS_USDT", price_precision=3, quantity_precision=3)
 EOS_BTC = Pair("EOS_BTC", price_precision=8, quantity_precision=2)
 EOS_USDT = Pair("EOS_USDT", price_precision=4, quantity_precision=2)
 EPS_USDT = Pair("EPS_USDT", price_precision=5, quantity_precision=1)
@@ -99,21 +102,26 @@ FORTH_USDT = Pair("FORTH_USDT", price_precision=3, quantity_precision=3)
 FTM_BTC = Pair("FTM_BTC", price_precision=9, quantity_precision=2)
 FTM_USDT = Pair("FTM_USDT", price_precision=4, quantity_precision=2)
 FXS_USDC = Pair("FXS_USDC", price_precision=4, quantity_precision=2)
+GALA_BTC = Pair("GALA_BTC", price_precision=9, quantity_precision=1)
 GALA_USDT = Pair("GALA_USDT", price_precision=6, quantity_precision=0)
 GHST_USDC = Pair("GHST_USDC", price_precision=4, quantity_precision=2)
+GLM_USDT = Pair("GLM_USDT", price_precision=5, quantity_precision=1)
 GRT_CRO = Pair("GRT_CRO", price_precision=3, quantity_precision=2)
 GRT_USDT = Pair("GRT_USDT", price_precision=5, quantity_precision=2)
 GTC_USDT = Pair("GTC_USDT", price_precision=3, quantity_precision=3)
 HNT_USDT = Pair("HNT_USDT", price_precision=3, quantity_precision=3)
 HOT_CRO = Pair("HOT_CRO", price_precision=5, quantity_precision=0)
 HOT_USDT = Pair("HOT_USDT", price_precision=6, quantity_precision=0)
+ICP_BTC = Pair("ICP_BTC", price_precision=8, quantity_precision=0)
 ICP_USDT = Pair("ICP_USDT", price_precision=3, quantity_precision=3)
 ICX_BTC = Pair("ICX_BTC", price_precision=8, quantity_precision=1)
 ICX_CRO = Pair("ICX_CRO", price_precision=3, quantity_precision=0)
 ICX_USDT = Pair("ICX_USDT", price_precision=4, quantity_precision=2)
+ILV_BTC = Pair("ILV_BTC", price_precision=6, quantity_precision=4)
 ILV_USDT = Pair("ILV_USDT", price_precision=2, quantity_precision=4)
 INJ_USDT = Pair("INJ_USDT", price_precision=3, quantity_precision=3)
 IOTX_USDC = Pair("IOTX_USDC", price_precision=6, quantity_precision=0)
+IOTX_USDT = Pair("IOTX_USDT", price_precision=5, quantity_precision=1)
 IQ_USDT = Pair("IQ_USDT", price_precision=6, quantity_precision=0)
 KAVA_USDT = Pair("KAVA_USDT", price_precision=4, quantity_precision=2)
 KEEP_USDT = Pair("KEEP_USDT", price_precision=5, quantity_precision=1)
@@ -172,6 +180,7 @@ PENDLE_USDT = Pair("PENDLE_USDT", price_precision=5, quantity_precision=1)
 PERP_USDT = Pair("PERP_USDT", price_precision=3, quantity_precision=3)
 PLA_USDT = Pair("PLA_USDT", price_precision=5, quantity_precision=1)
 POLY_USDC = Pair("POLY_USDC", price_precision=5, quantity_precision=1)
+QI_USDT = Pair("QI_USDT", price_precision=5, quantity_precision=1)
 QNT_USDC = Pair("QNT_USDC", price_precision=2, quantity_precision=4)
 QNT_USDT = Pair("QNT_USDT", price_precision=2, quantity_precision=4)
 QTUM_CRO = Pair("QTUM_CRO", price_precision=3, quantity_precision=2)
@@ -182,12 +191,16 @@ RARI_USDC = Pair("RARI_USDC", price_precision=3, quantity_precision=3)
 RARI_USDT = Pair("RARI_USDT", price_precision=3, quantity_precision=3)
 REN_CRO = Pair("REN_CRO", price_precision=4, quantity_precision=2)
 REN_USDT = Pair("REN_USDT", price_precision=5, quantity_precision=2)
+REQ_USDT = Pair("REQ_USDT", price_precision=5, quantity_precision=1)
+RGT_USDC = Pair("RGT_USDC", price_precision=3, quantity_precision=3)
 RLC_USDT = Pair("RLC_USDT", price_precision=3, quantity_precision=3)
 RLY_USDT = Pair("RLY_USDT", price_precision=5, quantity_precision=2)
+RNDR_USDT = Pair("RNDR_USDT", price_precision=4, quantity_precision=2)
 RSR_USDT = Pair("RSR_USDT", price_precision=5, quantity_precision=1)
 RUNE_USDT = Pair("RUNE_USDT", price_precision=3, quantity_precision=3)
 SAND_USDT = Pair("SAND_USDT", price_precision=5, quantity_precision=2)
 SC_USDT = Pair("SC_USDT", price_precision=6, quantity_precision=0)
+SDN_USDT = Pair("SDN_USDT", price_precision=4, quantity_precision=2)
 SHIB_USDC = Pair("SHIB_USDC", price_precision=9, quantity_precision=0)
 SHIB_USDT = Pair("SHIB_USDT", price_precision=9, quantity_precision=0)
 SKL_USDT = Pair("SKL_USDT", price_precision=5, quantity_precision=2)
@@ -197,12 +210,15 @@ SNX_USDT = Pair("SNX_USDT", price_precision=3, quantity_precision=3)
 SOL_BTC = Pair("SOL_BTC", price_precision=7, quantity_precision=3)
 SOL_USDC = Pair("SOL_USDC", price_precision=3, quantity_precision=3)
 SOL_USDT = Pair("SOL_USDT", price_precision=3, quantity_precision=3)
+SRM_USDC = Pair("SRM_USDC", price_precision=4, quantity_precision=2)
 STORJ_USDT = Pair("STORJ_USDT", price_precision=4, quantity_precision=2)
 STX_USDT = Pair("STX_USDT", price_precision=4, quantity_precision=2)
 SUSHI_USDT = Pair("SUSHI_USDT", price_precision=3, quantity_precision=3)
 TFUEL_USDT = Pair("TFUEL_USDT", price_precision=5, quantity_precision=1)
 THETA_USDT = Pair("THETA_USDT", price_precision=4, quantity_precision=2)
 TRB_USDT = Pair("TRB_USDT", price_precision=3, quantity_precision=3)
+TRIBE_USDT = Pair("TRIBE_USDT", price_precision=4, quantity_precision=2)
+TRU_USDT = Pair("TRU_USDT", price_precision=5, quantity_precision=1)
 UMA_USDT = Pair("UMA_USDT", price_precision=3, quantity_precision=4)
 UNI_CRO = Pair("UNI_CRO", price_precision=3, quantity_precision=2)
 UNI_USDC = Pair("UNI_USDC", price_precision=3, quantity_precision=3)
@@ -213,9 +229,11 @@ VET_CRO = Pair("VET_CRO", price_precision=4, quantity_precision=0)
 VET_USDC = Pair("VET_USDC", price_precision=6, quantity_precision=0)
 VET_USDT = Pair("VET_USDT", price_precision=6, quantity_precision=0)
 VTHO_USDT = Pair("VTHO_USDT", price_precision=6, quantity_precision=0)
+VVS_USDC = Pair("VVS_USDC", price_precision=8, quantity_precision=0)
 WAVES_USDT = Pair("WAVES_USDT", price_precision=3, quantity_precision=3)
 WAXP_USDT = Pair("WAXP_USDT", price_precision=5, quantity_precision=1)
 WBTC_BTC = Pair("WBTC_BTC", price_precision=4, quantity_precision=6)
+WBTC_USDC = Pair("WBTC_USDC", price_precision=2, quantity_precision=6)
 WBTC_USDT = Pair("WBTC_USDT", price_precision=2, quantity_precision=6)
 XLM_BTC = Pair("XLM_BTC", price_precision=9, quantity_precision=0)
 XLM_CRO = Pair("XLM_CRO", price_precision=3, quantity_precision=2)
@@ -227,6 +245,7 @@ XRP_USDT = Pair("XRP_USDT", price_precision=5, quantity_precision=1)
 XTZ_BTC = Pair("XTZ_BTC", price_precision=8, quantity_precision=2)
 XTZ_CRO = Pair("XTZ_CRO", price_precision=3, quantity_precision=2)
 XTZ_USDT = Pair("XTZ_USDT", price_precision=4, quantity_precision=2)
+XYO_USDT = Pair("XYO_USDT", price_precision=6, quantity_precision=0)
 YFI_BTC = Pair("YFI_BTC", price_precision=4, quantity_precision=6)
 YFI_CRO = Pair("YFI_CRO", price_precision=2, quantity_precision=6)
 YFI_USDT = Pair("YFI_USDT", price_precision=2, quantity_precision=6)

src/cryptocom/exchange/rate_limiter.py

@@ -1,98 +0,0 @@
-import asyncio
-import math
-import time
-import traceback
-
-from contextlib import asynccontextmanager
-
-
-class RateLimiter:
-    def __init__(self,
-                 rate_limit: int,
-                 period: float or int,  # takes seconds
-                 concurrency_limit: int) -> None:
-        if not rate_limit or rate_limit < 1:
-            raise ValueError('rate limit must be non zero positive number')
-        if not concurrency_limit or concurrency_limit < 1:
-            raise ValueError('concurrent limit must be non zero positive number')
-
-        self.rate_limit = rate_limit
-        self.period = period
-        self.tokens_queue = asyncio.Queue(rate_limit)
-        self.tokens_consumer_task = asyncio.create_task(self.consume_tokens())
-        self.semaphore = asyncio.Semaphore(concurrency_limit)
-
-    async def add_token(self) -> None:
-        await self.tokens_queue.put(1)
-        return None
-
-    async def consume_tokens(self):
-        try:
-            consumption_rate = self.period / self.rate_limit
-            last_consumption_time = 0
-
-            while True:
-                if self.tokens_queue.empty():
-                    await asyncio.sleep(consumption_rate)
-                    continue
-
-                current_consumption_time = time.monotonic()
-                total_tokens = self.tokens_queue.qsize()
-                tokens_to_consume = self.get_tokens_amount_to_consume(
-                    consumption_rate,
-                    current_consumption_time,
-                    last_consumption_time,
-                    total_tokens
-                )
-
-                for _ in range(0, tokens_to_consume):
-                    self.tokens_queue.get_nowait()
-
-                last_consumption_time = time.monotonic()
-
-                await asyncio.sleep(consumption_rate)
-        except asyncio.CancelledError:
-            raise
-        except Exception as e:
-            raise
-
-    @staticmethod
-    def get_tokens_amount_to_consume(consumption_rate, current_consumption_time,
-                                     last_consumption_time, total_tokens):
-        time_from_last_consumption = current_consumption_time - last_consumption_time
-        calculated_tokens_to_consume = math.floor(time_from_last_consumption / consumption_rate)
-        tokens_to_consume = min(total_tokens, calculated_tokens_to_consume)
-
-        return tokens_to_consume
-
-    @asynccontextmanager
-    async def throttle(self):
-        await self.semaphore.acquire()
-        await self.add_token()
-        try:
-            yield
-        finally:
-            self.semaphore.release()
-
-    async def __aenter__(self):
-        return self
-
-    async def __aexit__(self, exc_type, exc_val, exc_tb):
-        if exc_type:
-            pass
-            # print(traceback.format_exc())
-
-        await self.close()
-
-    async def close(self) -> None:
-        if self.tokens_consumer_task and not self.tokens_consumer_task.cancelled():
-            try:
-                self.tokens_consumer_task.cancel()
-                await self.tokens_consumer_task
-            except asyncio.CancelledError:
-                # print(traceback.format_exc())
-                pass
-
-            except Exception as e:
-                # print(traceback.format_exc())
-                raise

tests/test_api.py

@@ -57,16 +57,48 @@ async def test_wrong_api_response():
         await api.post('account')
 
 
-# @pytest.mark.asyncio
+@pytest.mark.asyncio
-# async def test_api_rate_limits():
+async def test_api_rate_limits():
-#     api = cro.ApiProvider(from_env=True)
+    api = cro.ApiProvider(from_env=True)
-#     account = cro.Account(from_env=True)
+    pair = cro.pairs.CRO_USDT
-
+
-#     for _ in range(0, 100):
+    page = 0
-#         await account.get_balance()
+    page_size = 50
-
+
-#     for _ in range(0, 100):
+    params = {'page_size': page_size, 'page': page}
-#         await account.get_orders_history(cro.pairs.CRO_USDT, page_size=50)
+
-    
+    if pair:
-#     for _ in range(0, 100):
+        params['instrument_name'] = pair.name
-#         await api.get('public/get-ticker')
+
+    start_time = time.time()
+    await api.post('private/get-order-history', {'params': params})
+    await api.post('private/get-order-history', {'params': params})
+    finish_time = (time.time() - start_time)
+
+    assert finish_time > 1
+
+    start_time = time.time()
+    await api.post('private/get-order-history', {'params': params})
+    await api.post('private/get-order-history', {'params': params})
+    await api.post('private/get-order-history', {'params': params})
+    await api.post('private/get-order-history', {'params': params})
+
+    finish_time = time.time() - start_time
+    assert finish_time > 4
+
+    start_time = time.time()
+    await api.get('public/get-instruments')
+    await api.get('public/get-instruments')
+    await api.get('public/get-instruments')
+    await api.get('public/get-instruments')
+
+    finish_time = time.time() - start_time
+    assert finish_time < 4
+
+    start_time = time.time()
+    await api.post('private/get-order-history', {'params': params})
+    await api.post('private/get-order-history', {'params': params})
+    await api.post('private/get-order-history', {'params': params})
+
+    finish_time = time.time() - start_time
+    assert finish_time > 3