Generate binance-markets.json
#1
Hey all,

  I'm sure someone out there has already done this, or perhaps Gekko will eventually acquire the ability to do this.  For now though I've whipped up this little program that will generate binance-markets.json for you.


Code:
/* generate binance-markets.json for Gekko
* Written by George Shearer --- doc@drow.org
* Notes: There's not much error checking here, I just threw this together because I got tired of manually updating the file.
* I hope others can benefit from this.
*
* Thank you Mike van Rossum and all other Gekko contributors
*
* You can build this on most any modern linux platform:
*
* gcc -Wall -s -o genmarkets genmarkets.c -lcurl -ljson-c */


#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <json-c/json.h>
#include <curl/curl.h>

#define SUCCESS 0
#define FAIL    1

#define OUTPUT_FILE   "binance-markets.json"
#define BINANCE_URL   "https://api.binance.com/api/v1/exchangeInfo"

#define MAX_MARKETS 4000
#define MAX_CURRENCIES 50
#define MAX_ASSETS 2000

struct market
{
 char currency[10];
 char asset[10];
 double min_price;
 double min_qty;
 double stepsize;
 double min_notional;
 int currency_precision;
 int asset_precision;
} markets[MAX_MARKETS];

struct currency
{
 char currency[10];
} currencies[MAX_CURRENCIES];

struct asset
{
 char asset[10];
} assets[MAX_ASSETS];

struct curl_fetch_st {
   char *payload;
   size_t size;
};

int total_currencies = 0;
int total_assets = 0;
int total_markets = 0;

/* callback for curl fetch */
size_t curl_callback (void *contents, size_t size, size_t nmemb, void *userp) {
   size_t realsize = size * nmemb;                             /* calculate buffer size */
   struct curl_fetch_st *p = (struct curl_fetch_st *) userp;   /* cast pointer to fetch struct */

   /* expand buffer */
   p->payload = (char *) realloc(p->payload, p->size + realsize + 1);

   /* check buffer */
   if (p->payload == NULL) {
     /* this isn't good */
     fprintf(stderr,"ERROR: Failed to expand buffer in curl_callback");
     /* free buffer */
     free(p->payload);
     /* return */
     return(-1);
   }

   /* copy contents to buffer */
   memcpy(&(p->payload[p->size]), contents, realsize);

   /* set new buffer size */
   p->size += realsize;

   /* ensure null termination */
   p->payload[p->size] = 0;

   /* return size */
   return(realsize);
}

/* fetch and return url body via curl */
CURLcode curl_fetch_url(CURL *ch, const char *url, struct curl_fetch_st *fetch) {
   CURLcode rcode;                   /* curl result code */

   /* init payload */
   fetch->payload = (char *) calloc(1, sizeof(fetch->payload));

   /* check payload */
   if (fetch->payload == NULL) {
       /* log error */
       fprintf(stderr, "ERROR: Failed to allocate payload in curl_fetch_url");
       /* return error */
       return CURLE_FAILED_INIT;
   }

   /* init size */
   fetch->size = 0;

   /* set url to fetch */
   curl_easy_setopt(ch, CURLOPT_URL, url);

   /* set calback function */
   curl_easy_setopt(ch, CURLOPT_WRITEFUNCTION, curl_callback);

   /* pass fetch struct pointer */
   curl_easy_setopt(ch, CURLOPT_WRITEDATA, (void *) fetch);

   /* set default user agent */
   curl_easy_setopt(ch, CURLOPT_USERAGENT, "libcurl-agent/1.0");

   /* set timeout */
   curl_easy_setopt(ch, CURLOPT_TIMEOUT, 5);

   /* enable location redirects */
   curl_easy_setopt(ch, CURLOPT_FOLLOWLOCATION, 1);

   /* set maximum allowed redirects */
   curl_easy_setopt(ch, CURLOPT_MAXREDIRS, 1);

   /* fetch the url */
   rcode = curl_easy_perform(ch);

   /* return */
   return rcode;
}

int
do_curl(const char *url, char **storage)
{
 CURL *curl_handle;
 CURLcode curl_result;

 struct curl_fetch_st curl_fetch;
 struct curl_fetch_st *cf = &curl_fetch;

 curl_handle = curl_easy_init();
 curl_easy_setopt(curl_handle, CURLOPT_TIMEOUT, 20L);
 curl_result = curl_fetch_url(curl_handle, url, cf);
 curl_easy_cleanup(curl_handle);

 /* check return code */
 if (curl_result != CURLE_OK || cf->size < 1)
 {
   fprintf(stderr, "do_curl(): Error grabbing '%s': %s", url, curl_easy_strerror(curl_result));
   return(1);
 }

 /* check payload */
 if (cf->payload != NULL) {
   *storage = cf->payload;
   return(SUCCESS);
 } else {
   fprintf(stderr, "do_curl(): Error storing payload from '%s'\n",url);
   return(FAIL);
 }
}

void
add_currency(char *currency)
{
 int x = 0;
 for(;x < total_currencies && x < MAX_CURRENCIES;x++)
   if(!strcmp(currencies[x].currency, currency))
     return;
 total_currencies++;
 strncpy(currencies[x].currency, currency, 10);
}

void
add_asset(char *asset)
{
 int x = 0;
 for(;x < total_assets && x < MAX_ASSETS;x++)
   if(!strcmp(assets[x].asset, asset))
     return;
 total_assets++;
 strncpy(assets[x].asset, asset, 10);
}

int
get_binance_info(void)
{
 char *data;
 json_object *bjson;
 enum json_tokener_error jerr = json_tokener_success;

 printf("Connecting to %s ... ",BINANCE_URL); fflush(stdout);

 if(do_curl(BINANCE_URL, &data))
 {
   puts("failed");
   return(FAIL);
 }
 puts("success");

 bjson = json_tokener_parse_verbose(data, &jerr);
 free(data);

 if(jerr == json_tokener_success)
 {
   json_object *symbols;

   if((symbols = json_object_object_get(bjson, "symbols"))!=NULL)
   {
     int x;
     
     total_markets = json_object_array_length(symbols);

     for(x = 0; x < total_markets && x < MAX_MARKETS; x++)
     {
       json_object *tmpmkt = json_object_array_get_idx(symbols, x);
       json_object *filters;

       strncpy(markets[x].currency, json_object_get_string(json_object_object_get(tmpmkt, "quoteAsset")), 10);
       markets[x].currency_precision = atoi(json_object_get_string(json_object_object_get(tmpmkt, "quotePrecision")));
       add_currency(markets[x].currency);

       strncpy(markets[x].asset, json_object_get_string(json_object_object_get(tmpmkt, "baseAsset")), 10);
       markets[x].asset_precision = atoi(json_object_get_string(json_object_object_get(tmpmkt, "baseAssetPrecision")));
       add_asset(markets[x].asset);

       if((filters = json_object_object_get(tmpmkt, "filters"))!=NULL)
       {
         int y, numfilters = json_object_array_length(filters);

         for(y = 0; y < numfilters; y++)
         {
           json_object *tmpfilter = json_object_array_get_idx(filters, y);
           const char *filter_type = json_object_get_string(json_object_object_get(tmpfilter, "filterType"));

           if(!strcmp(filter_type, "PRICE_FILTER"))
             markets[x].min_price = atof(json_object_get_string(json_object_object_get(tmpfilter, "minPrice")));
           else if(!strcmp(filter_type, "MIN_NOTIONAL"))
             markets[x].min_notional = atof(json_object_get_string(json_object_object_get(tmpfilter, "minNotional")));
           else if(!strcmp(filter_type, "LOT_SIZE"))
           {
             markets[x].min_qty = atof(json_object_get_string(json_object_object_get(tmpfilter, "minQty")));
             markets[x].stepsize = atof(json_object_get_string(json_object_object_get(tmpfilter, "stepSize")));
           }
         }
       }
     }
     printf("Loaded %d markets\n",total_markets);
   }
 }
 json_object_put(bjson);
 return(SUCCESS);
}

void
gen_gekko_markets(const char *output)
{
 int x = 0;
 FILE *fp = fopen(OUTPUT_FILE, "w");

 fprintf(fp, "{\n  \"currencies\": [");
 for(; x < total_currencies; x++)
 {
   if(x)
     fprintf(fp, ",");
   fprintf(fp, " \"%s\"", currencies[x].currency);
 }

 fprintf(fp, " ],\n  \"assets\": [");
 for(x = 0; x < total_assets; x++)
 {
   if(x)
     fprintf(fp, ",");
   fprintf(fp, " \"%s\"", assets[x].asset);
 }
 fprintf(fp, " ],\n  \"markets\": [\n");

 for(x = 0; x < total_markets; x++)
 {
   char fmt[500], temp[100];

   if(x)
     fprintf(fp, ",\n");

   strcpy(fmt, "    {\n      \"pair\": [\n        \"%s\",\n        \"%s\"\n      ],\n      \"minimalOrder\": {\n        \"amount\": %f,\n        \"price\": %");
   sprintf(temp, "0.%df", markets[x].asset_precision);
   strcat(fmt, temp);
   strcat(fmt, ",\n        \"order\": %f\n      }\n    }");
   fprintf(fp, fmt, markets[x].currency, markets[x].asset, markets[x].min_qty, markets[x].min_price, markets[x].min_notional);
 }

 fprintf(fp, "\n  ]\n}\n");
 fclose(fp);
}

int
main(int argc, char *argv[])
{
 memset((void *)markets, 0, sizeof(struct market) * MAX_MARKETS);
 memset((void *)currencies, 0, sizeof(struct currency) * MAX_CURRENCIES);
 memset((void *)assets, 0, sizeof(struct asset) * MAX_ASSETS);

 if(get_binance_info() == SUCCESS)
 {
   printf("Generating gekko file \"%s\" ...", OUTPUT_FILE); fflush(stdout);
   gen_gekko_markets(OUTPUT_FILE);
   puts("done");
   exit(SUCCESS);
 }
 exit(FAIL);
}
  Reply
#2
Hey! Great stuff, but note that there is a script that comes with Gekko that can also do this, it can be found here: https://github.com/askmike/gekko/tree/de...arketFiles
  Reply
#3
Ahhh! I've been so heads down working on my strategies, I should explore Gekko source tree more. Hehe. Thanks Mike!

Oh, you use tickSize instead of minPrice from Binance API?

Also -- could you PRETTY PLEASE add a mechanism for strategies to know the currency balance before giving advice, and to perhaps also offer advice on how much currency balance to use? Smile
  Reply
#4
Quote:Oh, you use tickSize instead of minPrice from Binance API?

I didn't write that file, it was added by a contributor. Right now I don't know the details unfortunately.

Quote:Also -- could you PRETTY PLEASE add a mechanism for strategies to know the currency balance before giving advice, and to perhaps also offer advice on how much currency balance to use?

Working on this! I am now close to releasing a massive upgrade that has a totally redone event flow (communication between components), this will make it easy to do stuff like this in the future. Keep an eye on PR 2216 (details of event flow are in PR 1850)
  Reply
#5
(06-28-2018, 01:16 PM)askmike Wrote:
Quote:Oh, you use tickSize instead of minPrice from Binance API?

I didn't write that file, it was added by a contributor. Right now I don't know the details unfortunately.

Quote:Also -- could you PRETTY PLEASE add a mechanism for strategies to know the currency balance before giving advice, and to perhaps also offer advice on how much currency balance to use?

Working on this! I am now close to releasing a massive upgrade that has a totally redone event flow (communication between components), this will make it easy to do stuff like this in the future. Keep an eye on PR 2216 (details of event flow are in PR 1850)

Awesome!! I have hacked together a strategy that uses IRC as a "control system" for multiple Gekko bots to lock each other so that you can trade the same assets using the same API key.  If a bot finds a buy opportunity, the others won't give advice until they see the bot that made the buy make a sell, then the process repeats.

It's a hack... but it works. Smile
  Reply
#6
(06-28-2018, 05:02 PM)docdrow Wrote:
(06-28-2018, 01:16 PM)askmike Wrote:
Quote:Oh, you use tickSize instead of minPrice from Binance API?

I didn't write that file, it was added by a contributor. Right now I don't know the details unfortunately.

Quote:Also -- could you PRETTY PLEASE add a mechanism for strategies to know the currency balance before giving advice, and to perhaps also offer advice on how much currency balance to use?

Working on this! I am now close to releasing a massive upgrade that has a totally redone event flow (communication between components), this will make it easy to do stuff like this in the future. Keep an eye on PR 2216 (details of event flow are in PR 1850)

Awesome!! I have hacked together a strategy that uses IRC as a "control system" for multiple Gekko bots to lock each other so that you can trade the same assets using the same API key.  If a bot finds a buy opportunity, the others won't give advice until they see the bot that made the buy make a sell, then the process repeats.

It's a hack... but it works. Smile

Interesting :-D What is the exact goal, you want to trade different assets to one currency on one exchange, so when one bot sold his asset, the currency is "free" for another bot to take it and buy another asset? Or did I misunderstand? I was thinking on s.th. similar...an opportunity to hopp between coins whenever another coin shows better performance. So you have two buy advices and figure out the better one. Do you have a github fork to look at your hack?
  Reply


Forum Jump:


Users browsing this thread: