Gekko Forum
Fix Increase efficiency and shorten order filling time - Printable Version

+- Gekko Forum (https://forum.gekko.wizb.it)
+-- Forum: Gekko (https://forum.gekko.wizb.it/forum-13.html)
+--- Forum: Guides (https://forum.gekko.wizb.it/forum-22.html)
+--- Thread: Fix Increase efficiency and shorten order filling time (/thread-56664.html)

Fix Increase efficiency and shorten order filling time - xFFFFF - 04-05-2018

I found this solution at Gekkos Issues. It bounce last bid/ask price down/up by 0,04% while making order at Binance. Its cheaper solution than market order type. Im lame in Gekkos functions and overall javascript, so it propablly bounce price two times.
Code:
const moment = require('moment');
const _ = require('lodash');

const util = require('../core/util');
const Errors = require('../core/error');
const log = require('../core/log');
const marketData = require('./binance-markets.json');

const Binance = require('binance');

var Trader = function(config) {
  _.bindAll(this);

  if (_.isObject(config)) {
    this.key = config.key;
    this.secret = config.secret;
    this.currency = config.currency.toUpperCase();
    this.asset = config.asset.toUpperCase();
  }

  this.pair = this.asset + this.currency;
  this.name = 'binance';

  this.market = _.find(Trader.getCapabilities().markets, (market) => {
    return market.pair[0] === this.currency && market.pair[1] === this.asset
  });

  this.binance = new Binance.BinanceRest({
    key: this.key,
    secret: this.secret,
    timeout: 15000,
    recvWindow: 60000, // suggested by binance
    disableBeautification: false,
    handleDrift: true,
  });
};

var retryCritical = {
  retries: 10,
  factor: 1.2,
  minTimeout: 1 * 1000,
  maxTimeout: 30 * 1000
};

var retryForever = {
  forever: false,
  factor: 1.2,
  minTimeout: 10 * 1000,
  maxTimeout: 30 * 1000
};

var recoverableErrors = new RegExp(/(SOCKETTIMEDOUT|TIMEDOUT|CONNRESET|CONNREFUSED|NOTFOUND|Error -1021|Response code 429|Response code 5)/);

Trader.prototype.processError = function(funcName, error) {
  if (!error) return undefined;

  if (!error.message || !error.message.match(recoverableErrors)) {
    log.error(`[binance.js] (${funcName}) returned an irrecoverable error: ${error}`);
    return new Errors.AbortError('[binance.js] ' + error.message || error);
  }

  log.debug(`[binance.js] (${funcName}) returned an error, retrying: ${error}`);
  return new Errors.RetryError('[binance.js] ' + error.message || error);
};

Trader.prototype.handleResponse = function(funcName, callback) {
  return (error, body) => {
    if (body && !_.isEmpty(body.code)) {
      error = new Error(`Error ${body.code}: ${body.msg}`);
    }

    return callback(this.processError(funcName, error), body);
  }
};

Trader.prototype.getTrades = function(since, callback, descending) {
  var processResults = function(err, data) {
    if (err) return callback(err);

    var parsedTrades = [];
    _.each(
      data,
      function(trade) {
        parsedTrades.push({
          tid: trade.aggTradeId,
          date: moment(trade.timestamp).unix(),
          price: parseFloat(trade.price),
          amount: parseFloat(trade.quantity),
        });
      },
      this
    );

    if (descending) callback(null, parsedTrades.reverse());
    else callback(undefined, parsedTrades);
  };

  var reqData = {
    symbol: this.pair,
  };

  if (since) {
    var endTs = moment(since)
      .add(1, 'h')
      .valueOf();
    var nowTs = moment().valueOf();

    reqData.startTime = moment(since).valueOf();
    reqData.endTime = endTs > nowTs ? nowTs : endTs;
  }

  let handler = (cb) => this.binance.aggTrades(reqData, this.handleResponse('getTrades', cb));
  util.retryCustom(retryForever, _.bind(handler, this), _.bind(processResults, this));
};

Trader.prototype.getPortfolio = function(callback) {
  var setBalance = function(err, data) {
    log.debug(`[binance.js] entering "setBalance" callback after api call, err: ${err} data: ${JSON.stringify(data)}`)
    if (err) return callback(err);

    var findAsset = function(item) {
      return item.asset === this.asset;
    }
    var assetAmount = parseFloat(_.find(data.balances, _.bind(findAsset, this)).free);

    var findCurrency = function(item) {
      return item.asset === this.currency;
    }
    var currencyAmount = parseFloat(_.find(data.balances, _.bind(findCurrency, this)).free);

    if (!_.isNumber(assetAmount) || _.isNaN(assetAmount)) {
      log.error(
        `Binance did not return portfolio for ${this.asset}, assuming 0.`
      );
      assetAmount = 0;
    }

    if (!_.isNumber(currencyAmount) || _.isNaN(currencyAmount)) {
      log.error(
        `Binance did not return portfolio for ${this.currency}, assuming 0.`
      );
      currencyAmount = 0;
    }

    var portfolio = [
      { name: this.asset, amount: assetAmount },
      { name: this.currency, amount: currencyAmount },
    ];

    return callback(undefined, portfolio);
  };

  let handler = (cb) => this.binance.account({}, this.handleResponse('getPortfolio', cb));
  util.retryCustom(retryForever, _.bind(handler, this), _.bind(setBalance, this));
};

// This uses the base maker fee (0.1%), and does not account for BNB discounts
Trader.prototype.getFee = function(callback) {
  var makerFee = 0.05;
  callback(undefined, makerFee / 100);
};

Trader.prototype.getTicker = function(callback) {
  var setTicker = function(err, data) {
    log.debug(`[binance.js] entering "getTicker" callback after api call, err: ${err} data: ${(data || []).length} symbols`);
    if (err) return callback(err);

    var findSymbol = function(ticker) {
      return ticker.symbol === this.pair;
    }
    var result = _.find(data, _.bind(findSymbol, this));

    var ticker = {
      ask: parseFloat(result.askPrice),
      bid: parseFloat(result.bidPrice),
    };

    callback(undefined, ticker);
  };

  let handler = (cb) => this.binance._makeRequest({}, this.handleResponse('getTicker', cb), 'api/v1/ticker/allBookTickers');
  util.retryCustom(retryForever, _.bind(handler, this), _.bind(setTicker, this));
};

// Effectively counts the number of decimal places, so 0.001 or 0.234 results in 3
Trader.prototype.getPrecision = function(tickSize) {
  if (!isFinite(tickSize)) return 0;
  var e = 1, p = 0;
  while (Math.round(tickSize * e) / e !== tickSize) { e *= 10; p++; }
  return p;
};

Trader.prototype.roundAmount = function(amount, tickSize) {
  var precision = 100000000;
  var t = this.getPrecision(tickSize);

  if(Number.isInteger(t))
    precision = Math.pow(10, t);

  amount *= precision;
  amount = Math.floor(amount);
  amount /= precision;
  return amount;
};

Trader.prototype.getLotSize = function(tradeType, amount, price, callback) {
log.debug(price);

  log.debug(price);
  amount = this.roundAmount(amount, this.market.minimalOrder.amount);
  if (amount < this.market.minimalOrder.amount)
    return callback(undefined, { amount: 0, price: 0 });
log.debug(price);
  price = this.roundAmount(price, this.market.minimalOrder.price)
  price += (tradeType === 'buy') ? (price * .004) : -(price * .004);
  price = Math.max(this.roundAmount(price, 0.00001), 0.00001);



  if (price < this.market.minimalOrder.price)
    return callback(undefined, { amount: 0, price: 0 });

  if (amount * price < this.market.minimalOrder.order)
    return callback(undefined, { amount: 0, price: 0});

  callback(undefined, { amount: amount, price: price });
}


Trader.prototype.addOrder = function(tradeType, amount, price, callback) {
  log.debug(`[binance.js] (addOrder) ${tradeType.toUpperCase()} ${amount} ${this.asset} @${price} ${this.currency}`);
log.debug(price);
price += (tradeType === 'buy') ? (price * .004) : -(price * .004);
  price = Math.max(this.roundAmount(price, 0.00001), 0.00001);
log.debug(price);
  var setOrder = function(err, data) {
    log.debug(`[binance.js] entering "setOrder" callback after api call, err: ${err} data: ${JSON.stringify(data)}`);
    if (err) return callback(err);

    var txid = data.orderId;
    log.debug(`[binance.js] added order with txid: ${txid}`);

    callback(undefined, txid);
  };

  let reqData = {
    symbol: this.pair,
    side: tradeType.toUpperCase(),
    type: 'LIMIT',
    timeInForce: 'GTC', // Good to cancel (I think, not really covered in docs, but is default)
    quantity: amount,
    price: price,
    timestamp: new Date().getTime()
  };

  let handler = (cb) => this.binance.newOrder(reqData, this.handleResponse('addOrder', cb));
  util.retryCustom(retryCritical, _.bind(handler, this), _.bind(setOrder, this));
    log.debug(price);
    

   log.debug(price);
};

Trader.prototype.getOrder = function(order, callback) {
  var get = function(err, data) {
    log.debug(`[binance.js] entering "getOrder" callback after api call, err ${err} data: ${JSON.stringify(data)}`);
    if (err) return callback(err);
log.debug(price);
    var price = parseFloat(data.price);
log.debug(price);
    var amount = parseFloat(data.executedQty);
    // Data.time is a 13 digit millisecon unix time stamp.
    // https://momentjs.com/docs/#/parsing/unix-timestamp-milliseconds/ 
    var date = moment(data.time);

    callback(undefined, { price, amount, date });
  }.bind(this);

  let reqData = {
    symbol: this.pair,
    orderId: order,
  };

  let handler = (cb) => this.binance.queryOrder(reqData, this.handleResponse('getOrder', cb));
  util.retryCustom(retryCritical, _.bind(handler, this), _.bind(get, this));
};

Trader.prototype.buy = function(amount, price, callback) {
log.debug(price);

  this.addOrder('buy', amount, price, callback);
      price += price * .004;
};

Trader.prototype.sell = function(amount, price, callback) {
log.debug(price);

log.debug(price);
  this.addOrder('sell', amount, price, callback);
      price += - price * .004;
};

Trader.prototype.checkOrder = function(order, callback) {
  var check = function(err, data) {
    log.debug(`[binance.js] entering "checkOrder" callback after api call, err ${err} data: ${JSON.stringify(data)}`);
    if (err) return callback(err);

    var stillThere = data.status === 'NEW' || data.status === 'PARTIALLY_FILLED';
    var canceledManually = data.status === 'CANCELED' || data.status === 'REJECTED' || data.status === 'EXPIRED';
    callback(undefined, !stillThere && !canceledManually);
  };

  let reqData = {
    symbol: this.pair,
    orderId: order,
  };

  let handler = (cb) => this.binance.queryOrder(reqData, this.handleResponse('checkOrder', cb));
  util.retryCustom(retryCritical, _.bind(handler, this), _.bind(check, this));
};

Trader.prototype.cancelOrder = function(order, callback) {
  // callback for cancelOrder should be true if the order was already filled, otherwise false
  var cancel = function(err, data) {
    log.debug(`[binance.js] entering "cancelOrder" callback after api call, err ${err} data: ${JSON.stringify(data)}`);
    if (err) {
      if(data && data.msg === 'UNKNOWN_ORDER') {  // this seems to be the response we get when an order was filled
        return callback(true); // tell the thing the order was already filled
      }
      return callback(err);
    }
    callback(undefined);
  };

  let reqData = {
    symbol: this.pair,
    orderId: order,
  };

  let handler = (cb) => this.binance.cancelOrder(reqData, this.handleResponse('cancelOrder', cb));
  util.retryCustom(retryForever, _.bind(handler, this), _.bind(cancel, this));
};

Trader.prototype.initMarkets = function(callback) {

}

Trader.getCapabilities = function() {
  return {
    name: 'Binance',
    slug: 'binance',
    currencies: marketData.currencies,
    assets: marketData.assets,
    markets: marketData.markets,
    requires: ['key', 'secret'],
    providesHistory: 'date',
    providesFullHistory: true,
    tid: 'tid',
    tradable: true,
  };
};

module.exports = Trader;
Only lines 215-216, 223-224, 292 and 300 was changed. This same way You can implement fix on all exchanges. 

UPDATE - BITFINEX EXAMPLE
You must add one line to Trader.prototype.buy and Trader.prototype.sell in bitfinex.js
Example:
Code:
Trader.prototype.buy = function(amount, price, callback) {
  this.submit_order('buy', amount, price, callback);
  price += price * .004;
}

Trader.prototype.sell = function(amount, price, callback) {
  this.submit_order('sell', amount, price, callback);
  price += - price * .004;
}

It plus 0,4% of price to ask/bid price on making order to exchange. So incrase/decrase price a bit. Full changed file:
Code:
const Bitfinex = require("bitfinex-api-node");
const _ = require('lodash');
const moment = require('moment');

const util = require('../core/util');
const Errors = require('../core/error');
const log = require('../core/log');

const marketData = require('./bitfinex-markets.json');

var Trader = function(config) {
  _.bindAll(this);
  if(_.isObject(config)) {
    this.key = config.key;
    this.secret = config.secret;
  }
  this.name = 'Bitfinex';
  this.balance;
  this.price;
  this.asset = config.asset;
  this.currency = config.currency;
  this.pair = this.asset + this.currency;
  this.bitfinex = new Bitfinex(this.key, this.secret, { version: 1 }).rest;
}

var retryCritical = {
  retries: 10,
  factor: 1.2,
  minTimeout: 10 * 1000,
  maxTimeout: 60 * 1000
};

var retryForever = {
  forever: true,
  factor: 1.2,
  minTimeout: 10 * 1000,
  maxTimeout: 300 * 1000
};

// Probably we need to update these string
var recoverableErrors = new RegExp(/(SOCKETTIMEDOUT|TIMEDOUT|CONNRESET|CONNREFUSED|NOTFOUND|429|443|5\d\d)/g);

Trader.prototype.processError = function(funcName, error) {
  if (!error) return undefined;

  if (!error.message.match(recoverableErrors)) {
    log.error(`[bitfinex.js] (${funcName}) returned an irrecoverable error: ${error.message}`);
    return new Errors.AbortError('[bitfinex.js] ' + error.message);
  }

  log.debug(`[bitfinex.js] (${funcName}) returned an error, retrying: ${error.message}`);
  return new Errors.RetryError('[bitfinex.js] ' + error.message);
};

Trader.prototype.handleResponse = function(funcName, callback) {
  return (error, data, body) => {
    return callback(this.processError(funcName, error), data);
  }
};

Trader.prototype.getPortfolio = function(callback) {
  let process = (err, data) => {
    if (err) return callback(err);

    // We are only interested in funds in the "exchange" wallet
    data = data.filter(c => c.type === 'exchange');

    const asset = _.find(data, c => c.currency.toUpperCase() === this.asset);
    const currency = _.find(data, c => c.currency.toUpperCase() === this.currency);

    let assetAmount, currencyAmount;

    if(_.isObject(asset) && _.isNumber(+asset.available) && !_.isNaN(+asset.available))
      assetAmount = +asset.available;
    else {
      log.error(`Bitfinex did not provide ${this.asset} amount, assuming 0`);
      assetAmount = 0;
    }

    if(_.isObject(currency) && _.isNumber(+currency.available) && !_.isNaN(+currency.available))
      currencyAmount = +currency.available;
    else {
      log.error(`Bitfinex did not provide ${this.currency} amount, assuming 0`);
      currencyAmount = 0;
    }

    const portfolio = [
      { name: this.asset, amount: assetAmount },
      { name: this.currency, amount: currencyAmount },
    ];

    callback(undefined, portfolio);
  };

  let handler = (cb) => this.bitfinex.wallet_balances(this.handleResponse('getPortfolio', cb));
  util.retryCustom(retryForever, _.bind(handler, this), _.bind(process, this));
}

Trader.prototype.getTicker = function(callback) {
  let process = (err, data) => {
    if (err) return callback(err);

    // whenever we reach this point we have valid
    // data, the callback is still the same since
    // we are inside the same javascript scope.
    callback(undefined, {bid: +data.bid, ask: +data.ask})
  };
  
  let handler = (cb) => this.bitfinex.ticker(this.pair, this.handleResponse('getTicker', cb));
  util.retryCustom(retryForever, _.bind(handler, this), _.bind(process, this));
}

// This assumes that only limit orders are being placed, so fees are the
// "maker fee" of 0.1%.  It does not take into account volume discounts.
Trader.prototype.getFee = function(callback) {
    var makerFee = 0.1;
    callback(undefined, makerFee / 100);
}

Trader.prototype.submit_order = function(type, amount, price, callback) {
  let process = (err, data) => {
    if (err) return callback(err);

    callback(err, data.order_id);
  }

  amount = Math.floor(amount*100000000)/100000000;
  let handler = (cb) => this.bitfinex.new_order(this.pair,
    amount + '',
    price + '',
    this.name.toLowerCase(),
    type,
    'exchange limit',
    this.handleResponse('submitOrder', cb)
  );

  util.retryCustom(retryCritical, _.bind(handler, this), _.bind(process, this));
}

Trader.prototype.buy = function(amount, price, callback) {
  this.submit_order('buy', amount, price, callback);
  price += price * .004;
}

Trader.prototype.sell = function(amount, price, callback) {
  this.submit_order('sell', amount, price, callback);
  price += - price * .004;
}

Trader.prototype.checkOrder = function(order_id, callback) {
  let process = (err, data) => {
    if (err) return callback(err);

    callback(undefined, !data.is_live);
  }

  let handler = (cb) => this.bitfinex.order_status(order_id, this.handleResponse('checkOrder', cb));
  util.retryCustom(retryCritical, _.bind(handler, this), _.bind(process, this));
}


Trader.prototype.getOrder = function(order_id, callback) {
  let process = (err, data) => {
    if (err) return callback(err);

    var price = parseFloat(data.avg_execution_price);
    var amount = parseFloat(data.executed_amount);
    var date = moment.unix(data.timestamp);

    callback(undefined, {price, amount, date});
  };

  let handler = (cb) => this.bitfinex.order_status(order_id, this.handleResponse('getOrder', cb));
  util.retryCustom(retryCritical, _.bind(handler, this), _.bind(process, this));
}


Trader.prototype.cancelOrder = function(order_id, callback) {
  let process = (err, data) => {
    if (err) return callback(err);

    return callback(undefined);
  }

  let handler = (cb) => this.bitfinex.cancel_order(order_id, this.handleResponse('cancelOrder', cb));
  util.retryCustom(retryForever, _.bind(handler, this), _.bind(process, this));
}

Trader.prototype.getTrades = function(since, callback, descending) {
  let process = (err, data) => {  
    if (err) return callback(err);

    var trades = _.map(data, function(trade) {
      return {
        tid: trade.tid,
        date:  trade.timestamp,
        price: +trade.price,
        amount: +trade.amount
      }
    });

    callback(undefined, descending ? trades : trades.reverse());
  };

  var path = this.pair;
  if(since)
    path += '?limit_trades=2000';

  let handler = (cb) => this.bitfinex.trades(path, this.handleResponse('getTrades', cb));
  util.retryCustom(retryForever, _.bind(handler, this), _.bind(process, this));
}

Trader.getCapabilities = function () {
  return {
    name: 'Bitfinex',
    slug: 'bitfinex',
    currencies: marketData.currencies,
    assets: marketData.assets,
    markets: marketData.markets,
    requires: ['key', 'secret'],
    tid: 'tid',
    providesFullHistory: true,
    providesHistory: 'date',
    tradable: true,
    forceReorderDelay: true
  };
}

module.exports = Trader;


My subjective feelings are that it speeds up filling orders a bit. The more value you put in place .004 the faster theoretically it should be order filled. Im using tickrate 2 sec and orderdelay 15 sec. If someone will compare the method with the traditional one in live mode, please let me know.

This idea can be used to delevop a more intelligent solution that would raise the price until the order is filled. I would be grateful if someone did it Smile