More code refactoring

Fix bugs
Add logging
Remove test code.

gramjs/crypto/AES.js

@@ -46,41 +46,39 @@ class AES {
      * @returns {Buffer}
      */
     static encryptIge(plainText, key, iv) {
-
-        if (plainText.length % 16 !== 0) {
-            let padding = new Uint8Array(16 - plainText.length % 16);
-            plainText = new Uint8Array([
-                ...plainText,
-                ...padding,
-            ]);
+        let padding = plainText.length % 16;
+        if (padding) {
+            plainText = Buffer.concat([plainText, Helpers.generateRandomBytes(16 - padding)]);
         }
+
         let iv1 = iv.slice(0, Math.floor(iv.length / 2));
         let iv2 = iv.slice(Math.floor(iv.length / 2));
+
         let aes = new aesjs.AES(key);
-        let blocksCount = Math.floor(plainText.length / 16);
-        let cipherText = new Array(plainText.length).fill(0);
-        for (let blockIndex = 0; blockIndex < blocksCount; blockIndex++) {
-            let plainTextBlock = plainText.slice(blockIndex * 16, blockIndex * 16 + 16);
+        let cipherText = Buffer.alloc(0);
+        let blockCount = Math.floor(plainText.length / 16);
+
+        for (let blockIndex = 0; blockIndex < blockCount; blockIndex++) {
+            let plainTextBlock = Buffer.from(plainText.slice(blockIndex * 16, blockIndex * 16 + 16));
 
             for (let i = 0; i < 16; i++) {
                 plainTextBlock[i] ^= iv1[i];
             }
-            let cipherTextBlock = aes.encrypt(plainTextBlock);
+            let cipherTextBlock = Buffer.from(aes.encrypt(plainTextBlock));
+
             for (let i = 0; i < 16; i++) {
                 cipherTextBlock[i] ^= iv2[i];
             }
 
-            iv1 = cipherTextBlock.slice(0, 16);
+            iv1 = cipherTextBlock;
             iv2 = plainText.slice(blockIndex * 16, blockIndex * 16 + 16);
-            cipherText = new Uint8Array([
-                ...cipherText.slice(0, blockIndex * 16),
-                ...cipherTextBlock.slice(0, 16),
-                ...cipherText.slice(blockIndex * 16 + 16)
+            cipherText = Buffer.concat([
+                cipherText,
+                cipherTextBlock,
             ]);
         }
-        return Buffer.from(cipherText);
+        return cipherText;
     }
-
 }
 
 module.exports = AES;

gramjs/extensions/BinaryReader.js

@@ -54,7 +54,6 @@ class BinaryReader {
             res = this.stream.readBigUInt64LE(this.offset);
         }
         this.offset += 8;
-        console.log("current offset is ", this.offset);
 
         return res;
     }
@@ -177,9 +176,6 @@ class BinaryReader {
     tgReadObject() {
         let constructorId = this.readInt(false);
         let clazz = tlobjects[constructorId];
-        console.log("class is ", clazz);
-        console.log(this.stream.toString("hex"));
-        console.log(this.offset);
         if (clazz === undefined) {
             /**
              * The class was None, but there's still a
@@ -210,7 +206,6 @@ class BinaryReader {
             }
 
         }
-        console.log(this.tellPosition());
         return clazz.fromReader(this);
 
     }

gramjs/extensions/BinaryWriter.js

@@ -0,0 +1,16 @@
+class BinaryWriter {
+    constructor(stream) {
+        this._stream = stream;
+    }
+
+    write(buffer) {
+        this._stream = Buffer.concat([this._stream, buffer]);
+    }
+
+    getValue() {
+        return this._stream;
+    }
+
+}
+
+module.exports = BinaryWriter;

gramjs/extensions/MessagePacker.js

@@ -1,11 +1,16 @@
 const Helpers = require("../utils/Helpers");
+const MessageContainer = require("../tl/core/MessageContainer");
+const TLMessage = require("../tl/core/TLMessage");
 const {TLRequest} = require("../tl/tlobject");
+const BinaryWriter = require("../extensions/BinaryWriter");
+
 
 class MessagePacker {
-    constructor(state,logger) {
+    constructor(state, logger) {
         this._state = state;
         this._queue = [];
         this._ready = false;
+        this._log = logger;
     }
 
     append(state) {
@@ -28,28 +33,34 @@ class MessagePacker {
             }
         }
         let data;
-        let buffer = [];
+        let buffer = new BinaryWriter(Buffer.alloc(0));
 
         let batch = [];
         let size = 0;
 
-        while (this._queue.length && batch.length <= 100) {
+        while (this._queue.length && batch.length <= MessageContainer.MAXIMUM_LENGTH) {
             let state = this._queue.shift();
-            size += state.length + 12;
-            if (size <= 1044448 - 8) {
-                state.msgId = this._state.writeDataAsMessage(
+            size += state.data.length + TLMessage.SIZE_OVERHEAD;
+            if (size <= MessageContainer.MAXIMUM_SIZE) {
+                let afterId;
+                if (state.after) {
+                    afterId = state.after.msgId;
+                }
+                state.msgId = await this._state.writeDataAsMessage(
                     buffer, state.data, state.request instanceof TLRequest,
-                    state.after.msgId
-                )
+                    afterId
+                );
+
+                this._log.debug(`Assigned msgId = ${state.msgId} to ${state.request.constructor.name}`);
                 batch.push(state);
-                //log
                 continue;
             }
             if (batch.length) {
                 this._queue.unshift(state);
                 break;
             }
-
+            this._log.warning(`Message payload for ${state.request.constructor.name} is too long ${state.data.length} and cannot be sent`);
+            state.promise.reject("Request Payload is too big");
             size = 0;
             continue
         }
@@ -58,10 +69,10 @@ class MessagePacker {
         }
         if (batch.length > 1) {
             data = Buffer.concat([struct.pack(
-                '<Ii', 0x73f1f8dc, batch.length
-            ), buffer[0]]);
-            buffer = [];
-            let containerId = this._state.writeDataAsMessage(
+                '<Ii', MessageContainer.CONSTRUCTOR_ID, batch.length
+            ), buffer.getValue()]);
+
+            let containerId = await this._state.writeDataAsMessage(
                 buffer, data, false
             );
             for (let s of batch) {
@@ -69,7 +80,7 @@ class MessagePacker {
             }
         }
 
-        data = buffer[0];
+        data = buffer.getValue();
         return {batch, data}
     }
 }

gramjs/network/Authenticator.js

@@ -32,7 +32,6 @@ async function doAuthentication(sender) {
     let nonce = Helpers.readBigIntFromBuffer(bytes, false);
 
     let resPQ = await sender.send(new ReqPqMultiRequest({nonce: nonce}));
-    console.log(resPQ);
     if (!(resPQ instanceof ResPQ)) {
         throw new Error(`Step 1 answer was ${resPQ}`)
     }
@@ -48,7 +47,6 @@ async function doAuthentication(sender) {
     bytes = Helpers.generateRandomBytes(32);
     let newNonce = Helpers.readBigIntFromBuffer(bytes);
 
-    console.log(newNonce);
 
     let pqInnerData = new PQInnerData({
             pq: getByteArray(pq),
@@ -83,7 +81,6 @@ async function doAuthentication(sender) {
             encryptedData: cipherText
         }
     ));
-    console.log(serverDhParams);
     if (!(serverDhParams instanceof ServerDHParamsOk || serverDhParams instanceof ServerDHParamsFail)) {
         throw new Error(`Step 2.1 answer was ${serverDhParams}`)
     }
@@ -105,7 +102,7 @@ async function doAuthentication(sender) {
         }
     }
     if (!(serverDhParams instanceof ServerDHParamsOk)) {
-        console.log(`Step 2.2 answer was ${serverDhParams}`);
+        throw new Error(`Step 2.2 answer was ${serverDhParams}`);
     }
 
     // Step 3 sending: Complete DH Exchange
@@ -118,7 +115,6 @@ async function doAuthentication(sender) {
     let plainTextAnswer = AES.decryptIge(
         serverDhParams.encryptedAnswer, key, iv
     );
-    console.log(plainTextAnswer.toString("hex"));
 
     let reader = new BinaryReader(plainTextAnswer);
     reader.read(20); // hash sum
@@ -162,7 +158,6 @@ async function doAuthentication(sender) {
             encryptedData: clientDhEncrypted,
         }
     ));
-    console.log(dhGen);
     let nonceTypes = [DhGenOk, DhGenRetry, DhGenFail];
     if (!(dhGen instanceof nonceTypes[0] || dhGen instanceof nonceTypes[1] || dhGen instanceof nonceTypes[2])) {
         throw new Error(`Step 3.1 answer was ${dhGen}`)
@@ -175,21 +170,16 @@ async function doAuthentication(sender) {
         throw new SecurityError(`Step 3 invalid ${name} server nonce from server`)
 
     }
-    console.log("GAB is ", gab);
     let authKey = new AuthKey(getByteArray(gab));
     let nonceNumber = 1 + nonceTypes.indexOf(dhGen.constructor);
-    console.log("nonce number is ", nonceNumber);
-    console.log("newNonce is ", newNonce);
 
     let newNonceHash = authKey.calcNewNonceHash(newNonce, nonceNumber);
-    console.log("newNonceHash is ", newNonceHash);
     let dhHash = dhGen[`newNonceHash${nonceNumber}`];
-    console.log("dhHash is ", dhHash);
-    /*
+
     if (dhHash !== newNonceHash) {
         throw new SecurityError('Step 3 invalid new nonce hash');
     }
- */
+
     if (!(dhGen instanceof DhGenOk)) {
         throw new Error(`Step 3.2 answer was ${dhGen}`)
     }

gramjs/network/MTProtoPlainSender.js

@@ -37,11 +37,9 @@ class MTProtoPlainSender {
         ]);
 
         await this._connection.send(res);
-        console.log("sent it");
         body = await this._connection.recv();
-        console.log("recived body",body);
         if (body.length < 9) {
-            throw InvalidBufferError(body);
+            throw new InvalidBufferError(body);
         }
         let reader = new BinaryReader(body);
         let authKeyId = reader.readLong();
@@ -67,7 +65,6 @@ class MTProtoPlainSender {
          * the next TLObject without including the padding, but since the
          * reader isn't used for anything else after this, it's unnecessary.
          */
-        console.log("returned object");
         return reader.tgReadObject();
 
     }

gramjs/network/MTProtoSender.js

@@ -4,8 +4,6 @@ const Helpers = require("../utils/Helpers");
 const {MsgsAck} = require("../tl/types");
 const AuthKey = require("../crypto/AuthKey");
 const doAuthentication = require("./Authenticator");
-const AES = require("../crypto/AES");
-const {RPCError} = require("../errors/RPCBaseErrors");
 const RPCResult = require("../tl/core/RPCResult");
 const MessageContainer = require("../tl/core/MessageContainer");
 const GZIPPacked = require("../tl/core/GZIPPacked");
@@ -13,9 +11,6 @@ const TLMessage = require("../tl/core/TLMessage");
 const RequestState = require("./RequestState");
 const format = require('string-format');
 const {TypeNotFoundError} = require("../errors");
-const {BadMessageError} = require("../errors");
-const {InvalidDCError} = require("../errors");
-const {gzip, ungzip} = require('node-gzip');
 const MessagePacker = require("../extensions/MessagePacker");
 const Pong = require("../tl/core/GZIPPacked");
 const BadServerSalt = require("../tl/core/GZIPPacked");
@@ -27,6 +22,9 @@ const FutureSalts = require("../tl/core/GZIPPacked");
 const MsgsStateReq = require("../tl/core/GZIPPacked");
 const MsgResendReq = require("../tl/core/GZIPPacked");
 const MsgsAllInfo = require("../tl/core/GZIPPacked");
+const {SecurityError} = require("../errors/Common");
+const {InvalidBufferError} = require("../errors/Common");
+const {LogOutRequest} = require("../tl/functions/auth");
 //const {tlobjects} = require("../gramjs/tl/alltlobjects");
 format.extend(String.prototype, {});
 
@@ -50,7 +48,7 @@ class MTProtoSender {
      * @param opt
      */
     constructor(authKey, opt = {
-        loggers: null,
+        logger: null,
         retries: 5,
         delay: 1,
         autoReconnect: true,
@@ -60,8 +58,7 @@ class MTProtoSender {
         autoReconnectCallback: null
     }) {
         this._connection = null;
-        this._loggers = opt.loggers;
-        this._log = opt.loggers;
+        this._log = opt.logger;
         this._retries = opt.retries;
         this._delay = opt.delay;
         this._autoReconnect = opt.autoReconnect;
@@ -99,7 +96,7 @@ class MTProtoSender {
          * Note that here we're also storing their ``_RequestState``.
          */
         this._send_queue = new MessagePacker(this._state,
-            this._loggers);
+            this._log);
 
         /**
          * Sent states are remembered until a response is received.
@@ -154,11 +151,8 @@ class MTProtoSender {
             this._log.info('User is already connected!');
             return false;
         }
-        console.log("connecting sender");
         this._connection = connection;
         await this._connect();
-        console.log("finished connecting sender");
-        this._user_connected = true;
         return true;
     }
 
@@ -207,7 +201,7 @@ class MTProtoSender {
         if (!Helpers.isArrayLike(request)) {
             let state = new RequestState(request);
             this._send_queue.append(state);
-            return state;
+            return state.promise;
         } else {
             throw new Error("not supported");
         }
@@ -221,14 +215,12 @@ class MTProtoSender {
      * @private
      */
     async _connect() {
-        //this._log.info('Connecting to {0}...'.replace("{0}", this._connection));
+        this._log.info('Connecting to {0}...'.replace("{0}", this._connection));
         await this._connection.connect();
-        console.log("Connection success");
-        //this._log.debug("Connection success!");
-        console.log("auth key is ", this.authKey);
+        this._log.debug("Connection success!");
         if (!this.authKey._key) {
-            console.log("creating authKey");
             let plain = new MtProtoPlainSender(this._connection, this._loggers);
+            this._log.debug('New auth_key attempt ...');
             let res = await doAuthentication(plain);
             this.authKey.key = res.authKey;
             this._state.time_offset = res.timeOffset;
@@ -244,23 +236,27 @@ class MTProtoSender {
 
             }
 
+        } else {
+            this._log.debug('Already have an auth key ...');
         }
-        //this._log.debug('Starting send loop');
+        this._user_connected = true;
+
+        this._log.debug('Starting send loop');
         this._send_loop_handle = this._send_loop();
 
-        //this._log.debug('Starting receive loop');
+        this._log.debug('Starting receive loop');
         this._recv_loop_handle = this._recv_loop();
 
         // _disconnected only completes after manual disconnection
         // or errors after which the sender cannot continue such
         // as failing to reconnect or any unexpected error.
 
-        //this._log.info('Connection to %s complete!', this._connection)
+        this._log.info('Connection to %s complete!', this._connection)
 
     }
 
 
-    async _disconnected(error = null) {
+    async _disconnect(error = null) {
         if (this._connection === null) {
             this._log.info('Not disconnecting (already have no connection)');
             return
@@ -281,21 +277,24 @@ class MTProtoSender {
      */
     async _send_loop() {
         while (this._user_connected && !this._reconnecting) {
-            if (this._pending_ack) {
-                let ack = new RequestState(new MsgsAck(Array(this._pending_ack)));
+            if (this._pending_ack.size) {
+                let ack = new RequestState(new MsgsAck({msgIds: Array(this._pending_ack)}));
                 this._send_queue.append(ack);
                 this._last_acks.push(ack);
                 this._pending_ack.clear()
             }
-            this._log.debug('Waiting for messages to send...');
+            //this._log.debug('Waiting for messages to send...');
+            this._log.debug("Waiting for messages to send...");
             // TODO Wait for the connection send queue to be empty?
             // This means that while it's not empty we can wait for
             // more messages to be added to the send queue.
-            let {batch, data} = await this._send_queue.get();
-
-            if (!data) {
+            let res = await this._send_queue.get();
+            if (!res) {
                 continue
             }
+            let data = res.data;
+            let batch = res.batch;
+            //this._log.debug(`Encrypting ${batch.length} message(s) in ${data.length} bytes for sending`);
             this._log.debug(`Encrypting ${batch.length} message(s) in ${data.length} bytes for sending`);
 
             data = this._state.encryptMessageData(data);
@@ -306,6 +305,10 @@ class MTProtoSender {
                 this._log.info('Connection closed while sending data');
                 return
             }
+            for (let state of batch) {
+                this._pending_state[state.msgId] = state;
+            }
+            this._log.debug('Encrypted messages put in a queue to be sent');
 
         }
     }
@@ -316,10 +319,12 @@ class MTProtoSender {
 
         while (this._user_connected && !this._reconnecting) {
             //this._log.debug('Receiving items from the network...');
+            this._log.debug('Receiving items from the network...');
             try {
                 body = await this._connection.recv();
             } catch (e) {
                 //this._log.info('Connection closed while receiving data');
+                this._log.debug('Connection closed while receiving data');
                 return
             }
             try {
@@ -329,28 +334,29 @@ class MTProtoSender {
 
                 if (e instanceof TypeNotFoundError) {
                     // Received object which we don't know how to deserialize
-                    //this._log.info(`Type ${e.invalidConstructorId} not found, remaining data ${e.remaining}`);
+                    this._log.info(`Type ${e.invalidConstructorId} not found, remaining data ${e.remaining}`);
+                    continue
                 } else if (e instanceof SecurityError) {
                     // A step while decoding had the incorrect data. This message
                     // should not be considered safe and it should be ignored.
-                    //this._log.warning(`Security error while unpacking a received message: ${e}`);
+                    this._log.warning(`Security error while unpacking a received message: ${e}`);
                     continue
                 } else if (e instanceof InvalidBufferError) {
-                    //this._log.info('Broken authorization key; resetting');
+                    this._log.info('Broken authorization key; resetting');
                     this.authKey.key = null;
                     if (this._authKeyCallback) {
                         this._authKeyCallback(null)
                     }
                     return
                 } else {
-                    //this._log.exception('Unhandled error while receiving data');
+                    this._log.exception('Unhandled error while receiving data');
                     return
                 }
             }
             try {
                 await this._processMessage(message)
             } catch (e) {
-                //this._log.exception('Unhandled error while receiving data');
+                this._log.exception('Unhandled error while receiving data');
                 console.log(e);
             }
         }
@@ -369,7 +375,11 @@ class MTProtoSender {
      */
     async _processMessage(message) {
         this._pending_ack.add(message.msgId);
-        let handler = this._handlers.get(message.obj.CONSTRUCTOR_ID, this.handleUpdate);
+        let handler = this._handlers.get(message.obj.CONSTRUCTOR_ID);
+        if (!handler) {
+            handler = this._handleUpdate
+        }
+
         await handler(message);
     }
 
@@ -449,7 +459,7 @@ class MTProtoSender {
                 );
             } else {
                 let reader = new BinaryReader(RPCResult.body);
-                let result = state.request.readResult(reader);
+                state.resolve(state.request.readResult(reader));
             }
         }
 
@@ -485,7 +495,7 @@ class MTProtoSender {
 
     async _handleUpdate(message) {
         if (message.obj.SUBCLASS_OF_ID !== 0x8af52aac) {  // crc32(b'Updates')
-            this._log.warning(`Note: %s is not an update, not dispatching it ${message.obj}`);
+            this._log.warning(`Note: ${message.obj} is not an update, not dispatching it ${message.obj}`);
             return
         }
         this._log.debug('Handling update %s', message.obj.constructor.name);
@@ -508,7 +518,7 @@ class MTProtoSender {
         let state = this._pending_state.pop(pong.msgId, null);
         // Todo Check result
         if (state) {
-            state.future.set_result(pong)
+            state.resolve(pong)
         }
     }
 
@@ -527,7 +537,7 @@ class MTProtoSender {
         this._state.salt = badSalt.newServerSalt;
         let states = this._popStates(badSalt.badMsgId);
         this._send_queue.extend(states);
-        this._log.debug('%d message(s) will be resent', states.length);
+        this._log.debug(`${states.length} message(s) will be resent`);
     }
 
     /**
@@ -638,7 +648,7 @@ class MTProtoSender {
             let state = this._pending_state[msgId];
             if (state && state.request instanceof LogOutRequest) {
                 delete this._pending_state[msgId];
-                state.future.set_result(true)
+                state.resolve(true)
             }
         }
 
@@ -659,7 +669,7 @@ class MTProtoSender {
         this._log.debug(`Handling future salts for message ${message.msgId}`);
         let state = self._pending_state.pop(message.msgId, null);
         if (state) {
-            state.future.set_result(message.obj)
+            state.resolve(message.obj)
         }
     }
 
@@ -685,327 +695,7 @@ class MTProtoSender {
 
     }
 
-    /**
-     * Adds an update handler (a method with one argument, the received
-     * TLObject) that is fired when there are updates available
-     * @param handler {function}
-     */
-    addUpdateHandler(handler) {
-        let firstHandler = Boolean(this.onUpdateHandlers.length);
-        this.onUpdateHandlers.push(handler);
-        // If this is the first added handler,
-        // we must start receiving updates
-        if (firstHandler) {
-            this.setListenForUpdates(true);
-        }
-    }
-
-    /**
-     * Removes an update handler (a method with one argument, the received
-     * TLObject) that is fired when there are updates available
-     * @param handler {function}
-     */
-    removeUpdateHandler(handler) {
-        let index = this.onUpdateHandlers.indexOf(handler);
-        if (index !== -1) this.onUpdateHandlers.splice(index, 1);
-        if (!Boolean(this.onUpdateHandlers.length)) {
-            this.setListenForUpdates(false);
-
-        }
-    }
-
-    /**
-     *
-     * @param confirmed {boolean}
-     * @returns {number}
-     */
-    generateSequence(confirmed) {
-        if (confirmed) {
-            let result = this.session.sequence * 2 + 1;
-            this.session.sequence += 1;
-            return result;
-        } else {
-            return this.session.sequence * 2;
-        }
-    }
-
-
-    /**
-     *
-     * @param request
-     */
-    async receive(request) {
-        try {
-            //Try until we get an update
-            while (!request.confirmReceived) {
-                let {seq, body} = await this.transport.receive();
-                let {message, remoteMsgId, remoteSequence} = this.decodeMsg(body);
-                console.log("processing msg");
-                await this.processMsg(remoteMsgId, remoteSequence, message, 0, request);
-                console.log("finished processing msg");
-            }
-        } finally {
-            // Todo
-        }
-    }
-
-    // region Low level processing
-    /**
-     * Sends the given packet bytes with the additional
-     * information of the original request.
-     * @param packet
-     * @param request
-     */
-    async sendPacket(packet, request) {
-        request.msgId = this.session.getNewMsgId();
-        // First Calculate plainText to encrypt it
-        let first = Buffer.alloc(8);
-        let second = Buffer.alloc(8);
-        let third = Buffer.alloc(8);
-        let forth = Buffer.alloc(4);
-        let fifth = Buffer.alloc(4);
-        first.writeBigUInt64LE(this.session.salt, 0);
-        second.writeBigUInt64LE(this.session.id, 0);
-        third.writeBigUInt64LE(request.msgId, 0);
-        forth.writeInt32LE(this.generateSequence(request.confirmed), 0);
-        fifth.writeInt32LE(packet.length, 0);
-        let plain = Buffer.concat([
-            first,
-            second,
-            third,
-            forth,
-            fifth,
-            packet
-        ]);
-        let msgKey = Helpers.calcMsgKey(plain);
-        let {key, iv} = Helpers.calcKey(this.session.authKey.key, msgKey, true);
-
-        let cipherText = AES.encryptIge(plain, key, iv);
-
-        //And then finally send the encrypted packet
-
-        first = Buffer.alloc(8);
-        first.writeBigUInt64LE(this.session.authKey.keyId, 0);
-        let cipher = Buffer.concat([
-            first,
-            msgKey,
-            cipherText,
-        ]);
-        await this.transport.send(cipher);
-    }
-
-    /**
-     *
-     * @param body {Buffer}
-     * @returns {{remoteMsgId: number, remoteSequence: BigInt, message: Buffer}}
-     */
-    decodeMsg(body) {
-        if (body.length < 8) {
-            throw Error("Can't decode packet");
-        }
-        let remoteAuthKeyId = body.readBigInt64LE(0);
-        let offset = 8;
-        let msgKey = body.slice(offset, offset + 16);
-        offset += 16;
-        let {key, iv} = Helpers.calcKey(this.session.authKey.key, msgKey, false);
-        let plainText = AES.decryptIge(body.slice(offset, body.length), key, iv);
-        offset = 0;
-        let remoteSalt = plainText.readBigInt64LE(offset);
-        offset += 8;
-        let remoteSessionId = plainText.readBigInt64LE(offset);
-        offset += 8;
-        let remoteMsgId = plainText.readBigInt64LE(offset);
-        offset += 8;
-        let remoteSequence = plainText.readInt32LE(offset);
-        offset += 4;
-        let msgLen = plainText.readInt32LE(offset);
-        offset += 4;
-        let message = plainText.slice(offset, offset + msgLen);
-        return {message, remoteMsgId, remoteSequence}
-    }
-
-    async processMsg(msgId, sequence, reader, offset, request = undefined) {
-        this.needConfirmation.push(msgId);
-        let code = reader.readUInt32LE(offset);
-        console.log("code is ", code);
-        // The following codes are "parsed manually"
-        if (code === 0xf35c6d01) {  //rpc_result, (response of an RPC call, i.e., we sent a request)
-            console.log("got rpc result");
-            return await this.handleRpcResult(msgId, sequence, reader, offset, request);
-        }
 
-        if (code === 0x73f1f8dc) {  //msg_container
-            return this.handleContainer(msgId, sequence, reader, offset, request);
-        }
-        if (code === 0x3072cfa1) {  //gzip_packed
-            return this.handleGzipPacked(msgId, sequence, reader, offset, request);
-        }
-        if (code === 0xedab447b) {  //bad_server_salt
-            return await this.handleBadServerSalt(msgId, sequence, reader, offset, request);
-        }
-        if (code === 0xa7eff811) {  //bad_msg_notification
-            console.log("bad msg notification");
-            return this.handleBadMsgNotification(msgId, sequence, reader, offset);
-        }
-        /**
-         * If the code is not parsed manually, then it was parsed by the code generator!
-         * In this case, we will simply treat the incoming TLObject as an Update,
-         * if we can first find a matching TLObject
-         */
-        console.log("code", code);
-        if (code === 0x9ec20908) {
-            return this.handleUpdate(msgId, sequence, reader, offset);
-        } else {
-
-            if (tlobjects.contains(code)) {
-                return this.handleUpdate(msgId, sequence, reader);
-            }
-        }
-        console.log("Unknown message");
-        return false;
-    }
-
-    // region Message handling
-
-    handleUpdate(msgId, sequence, reader, offset = 0) {
-        let tlobject = Helpers.tgReadObject(reader, offset);
-        for (let handler of this.onUpdateHandlers) {
-            handler(tlobject);
-        }
-        return Float32Array
-    }
-
-    async handleContainer(msgId, sequence, reader, offset, request) {
-        let code = reader.readUInt32LE(offset);
-        offset += 4;
-        let size = reader.readInt32LE(offset);
-        offset += 4;
-        for (let i = 0; i < size; i++) {
-            let innerMsgId = reader.readBigUInt64LE(offset);
-            offset += 8;
-            let innerSequence = reader.readInt32LE(offset);
-            offset += 4;
-            let innerLength = reader.readInt32LE(offset);
-            offset += 4;
-            if (!(await this.processMsg(innerMsgId, sequence, reader, offset, request))) {
-                offset += innerLength;
-            }
-        }
-        return false;
-    }
-
-    async handleBadServerSalt(msgId, sequence, reader, offset, request) {
-        let code = reader.readUInt32LE(offset);
-        offset += 4;
-        let badMsgId = reader.readBigUInt64LE(offset);
-        offset += 8;
-        let badMsgSeqNo = reader.readInt32LE(offset);
-        offset += 4;
-        let errorCode = reader.readInt32LE(offset);
-        offset += 4;
-        let newSalt = reader.readBigUInt64LE(offset);
-        offset += 8;
-        this.session.salt = newSalt;
-
-        if (!request) {
-            throw Error("Tried to handle a bad server salt with no request specified");
-        }
-
-        //Resend
-        await this.send(request, true);
-        return true;
-    }
-
-    handleBadMsgNotification(msgId, sequence, reader, offset) {
-        let code = reader.readUInt32LE(offset);
-        offset += 4;
-        let requestId = reader.readUInt32LE(offset);
-        offset += 4;
-        let requestSequence = reader.readInt32LE(offset);
-        offset += 4;
-        let errorCode = reader.readInt32LE(offset);
-        return new BadMessageError(errorCode);
-    }
-
-    async handleRpcResult(msgId, sequence, reader, offset, request) {
-        if (!request) {
-            throw Error("RPC results should only happen after a request was sent");
-        }
-        let buffer = Buffer.alloc(0);
-        let code = reader.readUInt32LE(offset);
-        offset += 4;
-        let requestId = reader.readUInt32LE(offset);
-        offset += 4;
-        let innerCode = reader.readUInt32LE(offset);
-        offset += 4;
-        if (requestId === request.msgId) {
-            request.confirmReceived = true;
-        }
-
-        if (innerCode === 0x2144ca19) {  // RPC Error
-            console.log("Got an error");
-            let errorCode = reader.readInt32LE(offset);
-            offset += 4;
-            let errorMessage = Helpers.tgReadString(reader, offset);
-            offset = errorMessage.offset;
-            errorMessage = errorMessage.data;
-            let error = new RPCError(errorCode, errorMessage);
-            if (error.mustResend) {
-                request.confirmReceived = false;
-            }
-            if (error.message.startsWith("FLOOD_WAIT_")) {
-                console.log("Should wait {}s. Sleeping until then.".format(error.additionalData));
-                await Helpers.sleep();
-            } else if (error.message.startsWith("PHONE_MIGRATE_")) {
-                throw new InvalidDCError(error.additionalData);
-            } else {
-                throw error;
-            }
-
-        } else {
-            console.log("no errors");
-            if (innerCode === 0x3072cfa1) { //GZip packed
-                console.log("Gzipped data");
-                let res = Helpers.tgReadByte(reader, offset);
-                let unpackedData = await ungzip(res.data);
-                offset = res.offset;
-                res = request.onResponse(unpackedData, offset);
-                buffer = res.data;
-                offset = res.offset;
-            } else {
-                console.log("plain data");
-                offset -= 4;
-                let res = request.onResponse(reader, offset);
-                buffer = res.data;
-                offset = res.offset;
-            }
-        }
-        return {buffer, offset}
-
-    }
-
-    handleGzipPacked(msgId, sequence, reader, offset, request) {
-        throw Error("not implemented");
-        // TODO
-    }
-
-    setListenForUpdates(enabled) {
-
-        if (enabled) {
-            console.log("Enabled updates");
-        } else {
-            console.log("Disabled updates");
-        }
-    }
-
-    updatesListenMethod() {
-        while (true) {
-            let {seq, body} = this.transport.receive();
-            let {message, remoteMsgId, remoteSequence} = this.decodeMsg(body);
-            this.processMsg(remoteMsgId, remoteSequence, message);
-
-        }
-    }
 }
 
 module.exports = MTProtoSender;

gramjs/network/MTProtoState.js

@@ -2,6 +2,10 @@ const struct = require("python-struct");
 const Helpers = require("../utils/Helpers");
 const AES = require("../crypto/AES");
 const BinaryReader = require("../extensions/BinaryReader");
+const GZIPPacked = require("../tl/core/GZIPPacked");
+const {TLMessage} = require("../tl/core");
+const {SecurityError} = require("../errors/Common");
+const {InvalidBufferError} = require("../errors/Common");
 
 class MTProtoState {
     /**
@@ -94,26 +98,22 @@ class MTProtoState {
      * @param buffer
      * @param data
      * @param contentRelated
-     * @param opt
+     * @param afterId
      */
-    writeDataAsMessage(buffer, data, contentRelated, opt = {}) {
+    async writeDataAsMessage(buffer, data, contentRelated, afterId) {
         let msgId = this._getNewMsgId();
         let seqNo = this._getSeqNo(contentRelated);
         let body;
-        if (!opt) {
-            body = GzipPacked.gzipIfSmaller(contentRelated, data);
+        if (!afterId) {
+            body = await GZIPPacked.GZIPIfSmaller(contentRelated, data);
         } else {
-            body = GzipPacked.gzipIfSmaller(contentRelated,
-                new InvokeAfterMsgRequest(opt.afterId, data).toBuffer()
+            body = await GZIPPacked.GZIPIfSmaller(contentRelated,
+                new InvokeAfterMsgRequest(afterId, data).toBuffer()
             );
         }
-
-        buffer = Buffer.from([
-            buffer,
-            struct.pack('<qii', msgId, seqNo, body.length),
-            body,
-        ]);
-        return {msgId, buffer}
+        buffer.write(struct.pack('<qii', msgId.toString(), seqNo, body.length));
+        buffer.write(body);
+        return msgId;
     }
 
     /**
@@ -123,23 +123,25 @@ class MTProtoState {
      */
     encryptMessageData(data) {
         data = Buffer.concat([
-            struct.pack('<qq', this.salt, this.id),
+            struct.pack('<qq', this.salt.toString(), this.id.toString()),
             data,
         ]);
-        let padding = Helpers.generateRandomBytes(-(data.length + 12) % 16 + 12);
+        let padding = Helpers.generateRandomBytes(Helpers.mod(-(data.length + 12), 16) + 12);
         // Being substr(what, offset, length); x = 0 for client
         // "msg_key_large = SHA256(substr(auth_key, 88+x, 32) + pt + padding)"
         let msgKeyLarge = Helpers.sha256(
-            Buffer.concat(([
+            Buffer.concat([
                 this.authKey.key.slice(88, 88 + 32),
                 data,
                 padding
-            ]))
+            ])
         );
         // "msg_key = substr (msg_key_large, 8, 16)"
         let msgKey = msgKeyLarge.slice(8, 24);
+
         let {iv, key} = this._calcKey(this.authKey.key, msgKey, true);
-        let keyId = struct.pack('<Q', this.authKey.keyId);
+
+        let keyId = struct.pack('<Q', this.authKey.keyId.toString());
         return Buffer.concat([
             keyId,
             msgKey,
@@ -159,7 +161,7 @@ class MTProtoState {
      */
     decryptMessageData(body) {
         if (body.length < 8) {
-            throw InvalidBufferError(body);
+            throw new InvalidBufferError(body);
         }
 
         // TODO Check salt,sessionId, and sequenceNumber

gramjs/network/RequestState.js

@@ -1,3 +1,5 @@
+const Helpers = require("../utils/Helpers");
+
 class RequestState {
 
 
@@ -6,9 +8,14 @@ class RequestState {
         this.msgId = null;
         this.request = request;
         this.data = request.bytes;
-        this.after = after
-
+        this.after = after;
+        this.result = null;
+        this.promise = new Promise((resolve, reject) => {
+            this.resolve = resolve;
+            this.reject = reject
+        });
     }
+
 }
 
 module.exports = RequestState;

gramjs/network/connection/Connection.js

@@ -61,19 +61,15 @@ class Connection {
         while (this._sendArray.length !== 0) {
             await Helpers.sleep(1000);
         }
-        console.log("pushed it");
         this._sendArray.push(data);
-        console.log("why am i still here");
     }
 
     async recv() {
         while (this._connected) {
 
             while (this._recvArray.length === 0) {
-                console.log(this._recvArray);
                 await Helpers.sleep(1000);
             }
-            console.log("got result line 76");
             let result = this._recvArray.pop();
 
             if (result) { // null = sentinel value = keep trying
@@ -85,19 +81,32 @@ class Connection {
 
     async _sendLoop() {
         // TODO handle errors
-        while (this._connected) {
-            while (this._sendArray.length === 0) {
-                await Helpers.sleep(1000);
+        try {
+
+
+            while (this._connected) {
+                while (this._sendArray.length === 0) {
+                    await Helpers.sleep(1000);
+                }
+                await this._send(this._sendArray.pop());
+
             }
-            await this._send(this._sendArray.pop());
+        } catch (e) {
+            console.log(e);
+            this._log.info('The server closed the connection while sending')
 
         }
     }
 
     async _recvLoop() {
+        let data;
         while (this._connected) {
-            let data = await this._recv();
-            console.log("ok data is here");
+            try {
+                data = await this._recv();
+            } catch (e) {
+                console.log(e);
+                this._log.info("The server closed the connection")
+            }
             while (this._recvArray.length !== 0) {
                 await Helpers.sleep(1000);
             }
@@ -108,25 +117,24 @@ class Connection {
 
     async _initConn() {
         if (this._codec.tag) {
-            console.log("writing codec");
             await this.socket.write(this._codec.tag);
         }
     }
 
     async _send(data) {
         let encodedPacket = this._codec.encodePacket(data);
-        console.log("sent", encodedPacket.toString("hex"));
         await this.socket.write(encodedPacket);
-        //await this.socket.write(this._codec.encodePacket(data));
 
     }
 
     async _recv() {
-        console.log("receiving");
-        let res = await this._codec.readPacket(this.socket);
-        console.log("read a packet");
-        return res;
+        return await this._codec.readPacket(this.socket);
+    }
+
+    toString() {
+        return `${this._ip}:${this._port}/${this.constructor.name.replace("Connection", "")}`
     }
+
 }
 
 class PacketCodec {

gramjs/network/connection/TCPFull.js

@@ -27,20 +27,20 @@ class FullPacketCodec extends PacketCodec {
      * @returns {Promise<*>}
      */
     async readPacket(reader) {
-        console.log("will read soon");
         let packetLenSeq = await reader.read(8); // 4 and 4
         //process.exit(0);
 
         if (packetLenSeq === undefined) {
+            console.log("connection closed. exiting");
+            process.exit(0)
             throw new Error("closed connection");
+
         }
-        console.log("read packet length", packetLenSeq.toString("hex"));
 
         let res = struct.unpack("<ii", packetLenSeq);
         let packetLen = res[0];
         let seq = res[1];
         let body = await reader.read(packetLen - 8);
-        console.log("body", body.toString("hex"));
         let checksum = struct.unpack("<I", body.slice(-4))[0];
         body = body.slice(0, -4);
 
@@ -48,7 +48,6 @@ class FullPacketCodec extends PacketCodec {
         if (!(validChecksum === checksum)) {
             throw new InvalidChecksumError(checksum, validChecksum);
         }
-        console.log("correct checksum");
         return body;
     }
 }

gramjs/tl/Session.js

@@ -1,6 +1,13 @@
 const Helpers = require("../utils/Helpers");
 const fs = require("fs").promises;
-const {existsSync,readFileSync} = require("fs");
+const {existsSync, readFileSync} = require("fs");
+const AuthKey = require("../crypto/AuthKey");
+BigInt.toJSON = function () {
+    return {fool: this.fool.toString("hex")}
+};
+BigInt.parseJson = function () {
+    return {fool: BigInt("0x" + this.fool)}
+};
 
 class Session {
     constructor(sessionUserId) {
@@ -25,7 +32,16 @@ class Session {
      */
     async save() {
         if (this.sessionUserId) {
-            //await fs.writeFile(`${this.sessionUserId}.session`, JSON.stringify(this));
+            let str = JSON.stringify(this, function (key, value) {
+                if (typeof value === 'bigint') {
+                    return value.toString() + "n";
+                } else {
+                    return value;
+                }
+            });
+
+
+            await fs.writeFile(`${this.sessionUserId}.session`, str);
         }
     }
 
@@ -35,7 +51,32 @@ class Session {
         }
         let filepath = `${sessionUserId}.session`;
         if (existsSync(filepath)) {
-            return JSON.parse(readFileSync(filepath, "utf-8"));
+
+            let ob = JSON.parse(readFileSync(filepath, "utf-8"), function (key, value) {
+                if ((typeof value) == "string" && value.match(/(\d+)n/)) {
+                    return BigInt(value.slice(0, -1));
+                } else {
+                    return value;
+                }
+            });
+
+            let s = Buffer.from("2451F2CA6D89757A4B2A46571C9414A68F3531EF7DDC99B5DDDD33803E8D6FA27CEA148E5B8F0B414074DD6B767A7C64D9FCF2E5EA82F919A6274FF8E7ED392264EEC400DBFDB930CA7F4D992172427B4CD5663006F8C158D0F97F3D845565A6A100BF328548DD7E8213295988B43E7FDC7F5001265AF143DEA7A97BAFF9479C0257B26491F282DAB5CB5408F0A5B2430FE97248DFD26BC7974CE5AC7AB71B1B78C0C5ECCA4B7B704227913DD8937068118A90282A3FF1F2D53550C28F3F45817676A4B8751495D9ACCA323FF714BA7A103237C17EF508506B846C9FD357E9DB8C3FD38AE4E595A7B67D7D48D97AC9D332983025A18F4816104D414952DD35D0", "hex");
+
+
+            let authKey = new AuthKey(s);
+            let session = new Session(ob.sessionUserId);
+            session.serverAddress = ob.serverAddress;
+            session.port = ob.port;
+            //this.serverAddress = "localhost";
+            //this.port = 21;
+            session.authKey = authKey;
+            session.id = ob.id;
+            session.sequence = ob.sequence;
+            session.salt = ob.salt; // Unsigned long
+            session.timeOffset = ob.timeOffset;
+            session.lastMessageId = ob.lastMessageId;
+            session.user = ob.user;
+            return session;
         } else {
             return new Session(sessionUserId);
         }

gramjs/tl/TelegramClient.js

@@ -2,10 +2,13 @@ const Session = require("./Session");
 const doAuthentication = require("../network/Authenticator");
 const MtProtoSender = require("../network/mtprotoSender");
 const MTProtoRequest = require("../tl/MTProtoRequest");
+const {ImportBotAuthorizationRequest} = require("./functions/auth");
 const {ConnectionTCPFull} = require("../network/connection/TCPFull");
+const {TLRequest} = require("./tlobject");
 const {InvokeWithLayerRequest, InitConnectionRequest} = require("./functions/index");
 const {GetConfigRequest} = require("./functions/help");
 const {LAYER} = require("../tl/alltlobjects");
+const log4js = require('log4js');
 
 class TelegramClient {
 
@@ -17,6 +20,7 @@ class TelegramClient {
         this.apiId = apiId;
         this.apiHash = apiHash;
         this._connection = ConnectionTCPFull;
+        this._log = log4js.getLogger("gramjs");
         this._initWith = (x) => {
             return new InvokeWithLayerRequest({
                 layer: LAYER,
@@ -26,7 +30,7 @@ class TelegramClient {
                     systemVersion: "1.8.3",
                     appVersion: "1.8",
                     langCode: "en",
-                    langPack:"en",
+                    langPack: "en",
                     systemLangCode: "en",
                     query: x,
                     proxy: null,
@@ -36,7 +40,9 @@ class TelegramClient {
         this.session = Session.tryLoadOrCreateNew(sessionUserId);
         //These will be set later
         this.dcOptions = null;
-        this._sender = new MtProtoSender(this.session.authKey);
+        this._sender = new MtProtoSender(this.session.authKey, {
+            logger: this._log,
+        });
         this.phoneCodeHashes = Array();
 
     }
@@ -49,11 +55,10 @@ class TelegramClient {
      * @returns {Promise<void>}
      */
     async connect() {
-        let connection = new this._connection(this.session.serverAddress, this.session.port, this.session.dcId, null);
+        let connection = new this._connection(this.session.serverAddress, this.session.port, this.session.dcId, this._log);
         if (!await this._sender.connect(connection)) {
             return;
         }
-        console.log("ok");
         this.session.authKey = this._sender.authKey;
         await this.session.save();
         await this._sender.send(this._initWith(
@@ -62,37 +67,14 @@ class TelegramClient {
 
     }
 
-    /**
-     * Reconnects to the specified DC ID. This is automatically called after an InvalidDCError is raised
-     * @param dc_id {number}
-     */
-    async reconnect_to_dc(dc_id) {
-
-        if (this.dcOptions === undefined || this.dcOptions.length === 0) {
-            throw new Error("Can't reconnect. Stabilise an initial connection first.");
-        }
-        let dc;
-        for (dc of this.dcOptions) {
-            if (dc.id === dc_id) {
-                break;
-            }
-        }
-        await this.transport.close();
-        this.transport = new TcpTransport(dc.ipAddress, dc.port);
-        await this.transport.connect();
-        this.session.server_address = dc.ipAddress;
-        this.session.port = dc.port;
-        this.session.save();
-        await this.connect();
-    }
 
     /**
      * Disconnects from the Telegram server
      * @returns {Promise<void>}
      */
     async disconnect() {
-        if (this.sender) {
-            await this.sender.disconnect();
+        if (this._sender) {
+            await this._sender.disconnect();
         }
     }
 
@@ -102,13 +84,37 @@ class TelegramClient {
      * @returns {Promise}
      */
     async invoke(request) {
-        if (!(request instanceof MTProtoRequest)) {
+        if (!(request instanceof TLRequest)) {
             throw new Error("You can only invoke MTProtoRequests");
         }
-        await this.sender.send(request);
+        let res = await this._sender.send(request);
+        return res;
+    }
+
 
-        await this.sender.receive(request);
-        return request.result;
+    /**
+     * Logs in to Telegram to an existing user or bot account.
+
+     You should only use this if you are not authorized yet.
+
+     This method will send the code if it's not provided.
+
+     .. note::
+
+     In most cases, you should simply use `start()` and not this method.
+
+     * @param args {{botToken: string}}
+     * @returns {Promise<void>}
+     */
+    async signIn(args = {phone: null, code: null, password: null, botToken: null, phoneCodeHash: null}) {
+        let botToken = args.botToken;
+        let request = new ImportBotAuthorizationRequest({
+            flags: 0,
+            botAuthToken: botToken,
+            apiId: this.apiId,
+            apiHash: this.apiHash,
+        });
+        let result = await this.invoke(request);
     }
 }
 

gramjs/tl/core/GZIPPacked.js

@@ -11,9 +11,12 @@ class GZIPPacked extends TLObject {
         this.data = data;
     }
 
-    async GZIPIfSmaller(contentRelated, data) {
+    static async GZIPIfSmaller(contentRelated, data) {
         if (contentRelated && data.length > 512) {
             let gzipped = await (new GZIPPacked(data)).toBytes();
+            if (gzipped.length < data.length) {
+                return gzipped;
+            }
         }
         return data;
     }
@@ -38,4 +41,5 @@ class GZIPPacked extends TLObject {
     }
 
 }
+
 module.exports = GZIPPacked;

gramjs/tl/tlobject.js

@@ -22,7 +22,6 @@ class TLObject {
         }
         let r = [];
         let padding;
-        console.log(data.length)
         if (data.length < 254) {
             padding = (data.length + 1) % 4;
             if (padding !== 0) {
@@ -54,7 +53,6 @@ class TLObject {
         }
         if (dt instanceof Date) {
             dt = Math.floor((Date.now() - dt.getTime()) / 1000);
-            console.log(dt);
         }
         if (typeof dt == "number") {
             return struct.pack('<i', dt)

gramjs/utils/Helpers.js

@@ -71,6 +71,18 @@ class Helpers {
             return buf.readBigUInt64LE(0);
     }
 
+
+    /**
+     * .... really javascript
+     * @param n {number}
+     * @param m {number}
+     * @returns {number}
+     */
+    static mod(n, m) {
+        return ((n % m) + m) % m;
+    }
+
+
     /**
      * Generates a random bytes array
      * @param count

main.js

@@ -1,12 +1,25 @@
 const Helpers = require("./gramjs/utils/Helpers");
 const TelegramClient = require("./gramjs/tl/TelegramClient");
+const {GetConfigRequest} = require("./gramjs/tl/functions/help");
+
+const log4js = require('log4js');
+const logger = log4js.getLogger("gramjs");
+logger.level = 'debug';
+
+
+
 
 (async function () {
     console.log("Loading interactive example...");
     let sessionName = "anon";
-    let apiId = 17349;
-    let apiHash = "344583e45741c457fe1862106095a5eb";
-    let client = new TelegramClient(sessionName,  apiId, apiHash);
+    let apiId = 139938;
+    let apiHash = "284b4948be43b955f7398d2abf355b3f";
+    let client = new TelegramClient(sessionName, apiId, apiHash);
     await client.connect();
+    //let request = new GetConfigRequest();
+    //let res =         await client._sender.send(new GetConfigRequest());
+    //console.log(res)
+    //await client.signIn({botToken: "773348:AAEL_68PNU0ekQhzpjBKj9U5S4WiINq-voY"});
     console.log("You should now be connected.");
-})();
+})();
+

package-lock.json

@@ -300,6 +300,11 @@
       "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-4.4.6.tgz",
       "integrity": "sha512-VisC5TBBhOF+70zjrF9FOiqI2LZOhXK/vAWlOrqyqz3lLa+P8jzJ7L/sg90MHmkSY/brAXWwrmGSZR0tM5yi4g=="
     },
+    "date-format": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/date-format/-/date-format-2.1.0.tgz",
+      "integrity": "sha512-bYQuGLeFxhkxNOF3rcMtiZxvCBAquGzZm6oWA1oZ0g2THUzivaRhv8uOhdr19LmoobSOLoIAxeUK2RdbM8IFTA=="
+    },
     "debug": {
       "version": "4.1.1",
       "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz",
@@ -549,6 +554,16 @@
       "resolved": "https://registry.npmjs.org/flatted/-/flatted-2.0.1.tgz",
       "integrity": "sha512-a1hQMktqW9Nmqr5aktAux3JMNqaucxGcjtjWnZLHX7yyPCmlSV3M54nGYbqT8K+0GhF3NBgmJCc3ma+WOgX8Jg=="
     },
+    "fs-extra": {
+      "version": "8.1.0",
+      "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-8.1.0.tgz",
+      "integrity": "sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g==",
+      "requires": {
+        "graceful-fs": "^4.2.0",
+        "jsonfile": "^4.0.0",
+        "universalify": "^0.1.0"
+      }
+    },
     "fs.realpath": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
@@ -598,6 +613,11 @@
       "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz",
       "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA=="
     },
+    "graceful-fs": {
+      "version": "4.2.2",
+      "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.2.tgz",
+      "integrity": "sha512-IItsdsea19BoLC7ELy13q1iJFNmd7ofZH5+X/pJr90/nRoPEX0DJo1dHDbgtYWOhJhcCgMDTOw84RZ72q6lB+Q=="
+    },
     "has-flag": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
@@ -777,6 +797,14 @@
       "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz",
       "integrity": "sha1-nbe1lJatPzz+8wp1FC0tkwrXJlE="
     },
+    "jsonfile": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz",
+      "integrity": "sha1-h3Gq4HmbZAdrdmQPygWPnBDjPss=",
+      "requires": {
+        "graceful-fs": "^4.1.6"
+      }
+    },
     "leemon": {
       "version": "6.2.0",
       "resolved": "https://registry.npmjs.org/leemon/-/leemon-6.2.0.tgz",
@@ -810,6 +838,18 @@
       "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz",
       "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A=="
     },
+    "log4js": {
+      "version": "5.2.2",
+      "resolved": "https://registry.npmjs.org/log4js/-/log4js-5.2.2.tgz",
+      "integrity": "sha512-Iw4ZjbYTMxSTh1jnXM2brpRIr+psM8/nkUiOHu2gFfd0saoX2NdRB69buMWJJuoIJfU/eTzqKy9rVBr0zQwSGQ==",
+      "requires": {
+        "date-format": "^2.1.0",
+        "debug": "^4.1.1",
+        "flatted": "^2.0.1",
+        "rfdc": "^1.1.4",
+        "streamroller": "^2.2.2"
+      }
+    },
     "long": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz",
@@ -1146,6 +1186,11 @@
         "signal-exit": "^3.0.2"
       }
     },
+    "rfdc": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/rfdc/-/rfdc-1.1.4.tgz",
+      "integrity": "sha512-5C9HXdzK8EAqN7JDif30jqsBzavB7wLpaubisuQIGHWf2gUXSpzy6ArX/+Da8RjFpagWsCn+pIgxTMAmKw9Zug=="
+    },
     "rimraf": {
       "version": "2.6.3",
       "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz",
@@ -1270,6 +1315,16 @@
       "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz",
       "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw="
     },
+    "streamroller": {
+      "version": "2.2.2",
+      "resolved": "https://registry.npmjs.org/streamroller/-/streamroller-2.2.2.tgz",
+      "integrity": "sha512-wizmZ8NNiqeNIYHv8MqBBbSIeNNcsXyoKxbGYBpiFHCjTGlNHqGNGElwrSM3Awg+0j6U96/eFrSnjW+h3aRo0Q==",
+      "requires": {
+        "date-format": "^2.1.0",
+        "debug": "^4.1.1",
+        "fs-extra": "^8.1.0"
+      }
+    },
     "string-format": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/string-format/-/string-format-2.0.0.tgz",
@@ -1386,6 +1441,11 @@
       "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.6.0.tgz",
       "integrity": "sha512-q+MB8nYR1KDLrgr4G5yemftpMC7/QLqVndBmEEdqzmNj5dcFOO4Oo8qlwZE3ULT3+Zim1F8Kq4cBnikNhlCMlg=="
     },
+    "universalify": {
+      "version": "0.1.2",
+      "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz",
+      "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg=="
+    },
     "uri-js": {
       "version": "4.2.2",
       "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz",

package.json

@@ -18,6 +18,7 @@
     "csv-parse": "^4.4.6",
     "fast-csv": "^3.4.0",
     "glob": "^7.1.4",
+    "log4js": "^5.2.2",
     "node-gzip": "^1.1.2",
     "node-rsa": "^1.0.6",
     "promise-socket": "^6.0.2",