Add Download functions

Various fixes

gramjs/Helpers.js

@@ -1,4 +1,5 @@
 const crypto = require('crypto')
+const fs = require('fs')
 
 /**
  * use this instead of ** because of webpack
@@ -158,6 +159,14 @@ function generateKeyDataFromNonce(serverNonce, newNonce) {
     return { key: keyBuffer, iv: ivBuffer }
 }
 
+/**
+ * ensures that the parent directory exists
+ * @param filePath
+ */
+function ensureParentDirExists(filePath) {
+    fs.mkdirSync(filePath, { recursive: true })
+}
+
 /**
  * Calculates the SHA1 digest for the given data
  * @param data
@@ -258,4 +267,5 @@ module.exports = {
     getRandomInt,
     sleep,
     isArrayLike,
+    ensureParentDirExists,
 }

gramjs/Utils.js

@@ -130,6 +130,26 @@ function getInputChannel(entity) {
     _raiseCastFail(entity, 'InputChannel')
 }
 
+/**
+ *     Adds the JPG header and footer to a stripped image.
+ Ported from https://github.com/telegramdesktop/tdesktop/blob/bec39d89e19670eb436dc794a8f20b657cb87c71/Telegram/SourceFiles/ui/image/image.cpp#L225
+
+ * @param stripped{Buffer}
+ * @returns {Buffer}
+ */
+function strippedPhotoToJpg(stripped) {
+    // Note: Changes here should update _stripped_real_length
+    if (stripped.length < 3 || stripped[0] !== 1) {
+        return stripped
+    }
+    const header = Buffer.from('ffd8ffe000104a46494600010100000100010000ffdb004300281c1e231e19282321232d2b28303c64413c37373c7b585d4964918099968f808c8aa0b4e6c3a0aadaad8a8cc8ffcbdaeef5ffffff9bc1fffffffaffe6fdfff8ffdb0043012b2d2d3c353c76414176f8a58ca5f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8ffc00011080000000003012200021101031101ffc4001f0000010501010101010100000000000000000102030405060708090a0bffc400b5100002010303020403050504040000017d01020300041105122131410613516107227114328191a1082342b1c11552d1f02433627282090a161718191a25262728292a3435363738393a434445464748494a535455565758595a636465666768696a737475767778797a838485868788898a92939495969798999aa2a3a4a5a6a7a8a9aab2b3b4b5b6b7b8b9bac2c3c4c5c6c7c8c9cad2d3d4d5d6d7d8d9dae1e2e3e4e5e6e7e8e9eaf1f2f3f4f5f6f7f8f9faffc4001f0100030101010101010101010000000000000102030405060708090a0bffc400b51100020102040403040705040400010277000102031104052131061241510761711322328108144291a1b1c109233352f0156272d10a162434e125f11718191a262728292a35363738393a434445464748494a535455565758595a636465666768696a737475767778797a82838485868788898a92939495969798999aa2a3a4a5a6a7a8a9aab2b3b4b5b6b7b8b9bac2c3c4c5c6c7c8c9cad2d3d4d5d6d7d8d9dae2e3e4e5e6e7e8e9eaf2f3f4f5f6f7f8f9faffda000c03010002110311003f00', 'hex')
+    const footer = Buffer.from('ffd9', 'hex')
+    header[164] = stripped[1]
+    header[166] = stripped[2]
+    return Buffer.concat([header, stripped.slice(3), footer])
+}
+
+
 /**
  Similar to :meth:`get_input_peer`, but for :tl:`InputUser`'s alone.
 
@@ -177,6 +197,53 @@ function getInputUser(entity) {
     _raiseCastFail(entity, 'InputUser')
 }
 
+function getInputLocation(location) {
+    try {
+        if (!location.SUBCLASS_OF_ID) {
+            throw new Error()
+        }
+        if (location.SUBCLASS_OF_ID === 0x1523d462) {
+            return { dcId: null, inputLocation: location }
+        }
+    } catch (e) {
+        _raiseCastFail(location, 'InputFileLocation')
+    }
+    if (location instanceof types.Message) {
+        location = location.media
+    }
+
+    if (location instanceof types.MessageMediaDocument) {
+        location = location.document
+    } else if (location instanceof types.MessageMediaPhoto) {
+        location = location.photo
+    }
+
+    if (location instanceof types.Document) {
+        return {
+            dcId: location.dcId, inputLocation: new types.InputDocumentFileLocation({
+                id: location.id,
+                accessHash: location.accessHash,
+                fileReference: location.fileReference,
+                thumbSize: '', // Presumably to download one of its thumbnails
+            }),
+        }
+    } else if (location instanceof types.Photo) {
+        return {
+            dcId: location.dcId, inputLocation: new types.InputPhotoFileLocation({
+                id: location.id,
+                accessHash: location.accessHash,
+                fileReference: location.fileReference,
+                thumbSize: location.sizes[location.sizes.length - 1].type,
+            }),
+        }
+    }
+
+    if (location instanceof types.FileLocationToBeDeprecated) {
+        throw new Error('Unavailable location cannot be used as input')
+    }
+    _raiseCastFail(location, 'InputFileLocation')
+}
+
 /**
  Similar to :meth:`get_input_peer`, but for dialogs
  * @param dialog
@@ -432,6 +499,26 @@ function rtrim(s, mask) {
     return s
 }
 
+/**
+ * Gets the appropriated part size when uploading or downloading files,
+ * given an initial file size.
+ * @param fileSize
+ * @returns {Number}
+ */
+function getAppropriatedPartSize(fileSize) {
+    if (fileSize <= 104857600) { // 100MB
+        return 128
+    }
+    if (fileSize <= 786432000) { // 750MB
+        return 256
+    }
+    if (fileSize <= 1572864000) { // 1500MB
+        return 512
+    }
+
+    throw new Error('File size too large')
+}
+
 /**
  * Gets the display name for the given :tl:`User`,
  :tl:`Chat` or :tl:`Channel`. Returns an empty string otherwise
@@ -488,4 +575,6 @@ module.exports = {
     getDisplayName,
     resolveId,
     isListLike,
+    getAppropriatedPartSize,
+    getInputLocation, strippedPhotoToJpg,
 }

gramjs/client/TelegramClient.js

@@ -13,13 +13,19 @@ const { LAYER } = require('../tl/AllTLObjects')
 const { functions, types } = require('../tl')
 const { computeCheck } = require('../Password')
 const MTProtoSender = require('../network/MTProtoSender')
+const Helpers = require('../Helpers')
 const { ConnectionTCPObfuscated } = require('../network/connection/TCPObfuscated')
-
+const { BinaryWriter } = require('../extensions')
 const DEFAULT_DC_ID = 4
 const DEFAULT_IPV4_IP = '149.154.167.51'
 const DEFAULT_IPV6_IP = '[2001:67c:4e8:f002::a]'
 const DEFAULT_PORT = 443
 
+// Chunk sizes for upload.getFile must be multiples of the smallest size
+const MIN_CHUNK_SIZE = 4096
+const MAX_CHUNK_SIZE = 512 * 1024
+
+
 class TelegramClient {
     static DEFAULT_OPTIONS = {
         connection: ConnectionTCPObfuscated,
@@ -182,6 +188,7 @@ class TelegramClient {
 
     async _getDC(dcId, cdn = false) {
         if (!this._config) {
+            console.log('no config getting new')
             this._config = await this.invoke(new functions.help.GetConfigRequest())
         }
         if (cdn && !this._cdnConfig) {
@@ -214,8 +221,6 @@ class TelegramClient {
         if (request.CONSTRUCTOR_ID in this._floodWaitedRequests) {
             const due = this._floodWaitedRequests[request.CONSTRUCTOR_ID]
             const diff = Math.round(due - new Date().getTime() / 1000)
-            console.log('diff is ', diff)
-            console.log('limit is ', this.floodSleepLimit)
             if (diff <= 3) { // Flood waits below 3 seconds are 'ignored'
                 delete this._floodWaitedRequests[request.CONSTRUCTOR_ID]
             } else if (diff <= this.floodSleepLimit) {
@@ -234,6 +239,7 @@ class TelegramClient {
         for (attempt = 0; attempt < this._requestRetries; attempt++) {
             try {
                 const promise = this._sender.send(request)
+                console.log('promise is ', promise)
                 const result = await promise
                 this.session.processEntities(result)
                 this._entityCache.add(result)
@@ -258,6 +264,7 @@ class TelegramClient {
                     if (shouldRaise && await this.isUserAuthorized()) {
                         throw e
                     }
+                    await Helpers.sleep(1000)
                     await this._switchDC(e.newDc)
                 } else {
                     throw e
@@ -474,6 +481,7 @@ class TelegramClient {
      * @private
      */
     _onLogin(user) {
+        console.log('on login')
         this._bot = Boolean(user.bot)
         this._authorized = true
         return user
@@ -642,6 +650,17 @@ class TelegramClient {
 
 
     // users region
+
+
+
+
+
+
+
+
+
+
+
     /**
      Turns the given entity into its input entity version.
 
@@ -847,6 +866,302 @@ class TelegramClient {
         return sender
     }
 
+    // end region
+
+    // download region
+
+
+    async downloadFile(inputLocation, file, args = {
+        partSizeKb: null,
+        fileSize: null,
+        progressCallback: null,
+        dcId: null,
+    }) {
+        if (!args.partSizeKb) {
+            if (!args.fileSize) {
+                args.partSizeKb = 64
+            } else {
+                args.partSizeKb = utils.getAppropriatedPartSize(args.fileSize)
+            }
+        }
+        const partSize = parseInt(args.partSizeKb * 1024)
+        if (partSize % MIN_CHUNK_SIZE !== 0) {
+            throw new Error('The part size must be evenly divisible by 4096')
+        }
+        const inMemory = !file || file === Buffer
+        let f
+        if (inMemory) {
+            f = new BinaryWriter(Buffer.alloc(0))
+        } else {
+            throw new Error('not supported')
+        }
+        const res = utils.getInputLocation(inputLocation)
+        let exported = res.dcId && this.session.dcId !== res.dc
+        let sender
+        if (exported) {
+            try {
+                sender = await this._borrowExportedSender(res.dcId)
+            } catch (e) {
+                if (e instanceof errors.DcIdInvalidError) {
+                    // Can't export a sender for the ID we are currently in
+                    sender = this._sender
+                    exported = false
+                } else {
+                    throw e
+                }
+            }
+        } else {
+            sender = this._sender
+        }
+
+        this._log.info(`Downloading file in chunks of ${partSize} bytes`)
+
+        try {
+            let offset = 0
+            // eslint-disable-next-line no-constant-condition
+            while (true) {
+                let result
+                try {
+                    result = await sender.send(new functions.upload.GetFileRequest({
+                        location: res.inputLocation,
+                        offset: offset,
+                        limit: partSize,
+                    }))
+                    if (result instanceof types.upload.FileCdnRedirect) {
+                        throw new Error('not implemented')
+                    }
+                } catch (e) {
+                    if (e instanceof errors.FileMigrateError) {
+                        this._log.info('File lives in another DC')
+                        sender = await this._borrowExportedSender(e.newDc)
+                        exported = true
+                        continue
+                    } else {
+                        throw e
+                    }
+                }
+                offset += partSize
+
+                if (!result.bytes.length) {
+                    if (inMemory) {
+                        return f.getValue()
+                    } else {
+                        // Todo implement
+                    }
+                }
+                this._log.debug(`Saving ${result.bytes.length} more bytes`)
+                f.write(result.bytes)
+                if (args.progressCallback) {
+                    await args.progressCallback(f.getValue().length, args.fileSize)
+                }
+            }
+        } finally {
+            // TODO
+        }
+    }
+
+    async downloadMedia(message, file, args = {
+        thumb: null,
+        progressCallback: null,
+    }) {
+        let date
+        let media
+        if (message instanceof types.Message) {
+            date = message.date
+            media = message.media
+        } else {
+            date = new Date().getTime()
+            media = message
+        }
+        if (typeof media == 'string') {
+            throw new Error('not implemented')
+        }
+
+        if (media instanceof types.MessageMediaWebPage) {
+            if (media.webpage instanceof types.WebPage) {
+                media = media.webpage.document || media.webpage.photo
+            }
+        }
+        if (media instanceof types.MessageMediaPhoto || media instanceof types.Photo) {
+            console.log('is a photo')
+            return await this._downloadPhoto(media, file, date, args.thumb, args.progressCallback)
+        } else if (media instanceof types.MessageMediaDocument || media instanceof types.Document) {
+            return await this._downloadDocument(media, file, date, args.thumb, args.progressCallback)
+        } else if (media instanceof types.MessageMediaContact && args.thumb == null) {
+            return this._downloadContact(media, file)
+        } else if ((media instanceof types.WebDocument || media instanceof types.WebDocumentNoProxy) && args.thumb == null) {
+            return await this._downloadWebDocument(media, file, args.progressCallback)
+        }
+    }
+
+    async downloadProfilePhoto(entity, file, downloadBig = true) {
+        // ('User', 'Chat', 'UserFull', 'ChatFull')
+        const ENTITIES = [0x2da17977, 0xc5af5d94, 0x1f4661b9, 0xd49a2697]
+        // ('InputPeer', 'InputUser', 'InputChannel')
+        // const INPUTS = [0xc91c90b6, 0xe669bf46, 0x40f202fd]
+        // Todo account for input methods
+        const thumb = downloadBig ? -1 : 0
+
+        let photo
+        if (!(entity.SUBCLASS_OF_ID in ENTITIES)) {
+            photo = entity
+        } else {
+            if (!entity.photo) {
+                // Special case: may be a ChatFull with photo:Photo
+                if (!entity.chatPhoto) {
+                    return null
+                }
+
+                return await this._downloadPhoto(
+                    entity.chatPhoto, file, null, thumb, null,
+                )
+            }
+            photo = entity.photo
+        }
+        let dcId
+        let which
+        let loc
+        if (photo instanceof types.UserProfilePhoto || photo instanceof types.ChatPhoto) {
+            dcId = photo.dcId
+            which = downloadBig ? photo.photoBig : photo.photoSmall
+            loc = new types.InputPeerPhotoFileLocation({
+                peer: await this.getInputEntity({ entity: entity }),
+                localId: which.localId,
+                volumeId: which.volumeId,
+                big: downloadBig,
+            })
+        } else {
+            // It doesn't make any sense to check if `photo` can be used
+            // as input location, because then this method would be able
+            // to "download the profile photo of a message", i.e. its
+            // media which should be done with `download_media` instead.
+            return null
+        }
+        file = file ? file : Buffer
+        try {
+            const result = await this.downloadFile(loc, file, {
+                dcId: dcId,
+            })
+            return result
+        } catch (e) {
+            if (e instanceof errors.LocationInvalidError) {
+                const ie = await this.getInputEntity(entity)
+                if (ie instanceof types.InputPeerChannel) {
+                    const full = await this.invoke(new functions.channels.GetFullChannelRequest({
+                        channel: ie,
+                    }))
+                    return await this._downloadPhoto(full.fullChat.chatPhoto, file, null, null, thumb)
+                } else {
+                    return null
+                }
+            } else {
+                throw e
+            }
+        }
+
+
+    }
+
+    _getThumb(thumbs, thumb) {
+        if (thumb === null || thumb === undefined) {
+            return thumbs[thumbs.length - 1]
+        } else if (typeof thumb === 'number') {
+            return thumbs[thumb]
+        } else if (thumb instanceof types.PhotoSize ||
+            thumb instanceof types.PhotoCachedSize ||
+            thumb instanceof types.PhotoStrippedSize) {
+            return thumb
+        } else {
+            return null
+        }
+    }
+
+    _downloadCachedPhotoSize(size, file) {
+        // No need to download anything, simply write the bytes
+        let data
+        if (size instanceof types.PhotoStrippedSize) {
+            data = utils.strippedPhotoToJpg(size.bytes)
+        } else {
+            data = size.bytes
+        }
+        return data
+    }
+
+    async _downloadPhoto(photo, file, date, thumb, progressCallback) {
+        if (photo instanceof types.MessageMediaPhoto) {
+            photo = photo.photo
+        }
+        if (!(photo instanceof types.Photo)) {
+            return
+        }
+        const size = this._getThumb(photo.sizes, thumb)
+        if (!size || (size instanceof types.PhotoSizeEmpty)) {
+            return
+        }
+
+        file = file ? file : Buffer
+        if (size instanceof types.PhotoCachedSize || size instanceof types.PhotoStrippedSize) {
+            return this._downloadCachedPhotoSize(size, file)
+        }
+
+        const result = await this.downloadFile(
+            new types.InputPhotoFileLocation({
+                id: photo.id,
+                accessHash: photo.accessHash,
+                fileReference: photo.fileReference,
+                thumbSize: size.type,
+            }),
+            file,
+            {
+                fileSize: size.size,
+                progressCallback: progressCallback,
+            },
+        )
+        return result
+    }
+
+    async _downloadDocument(document, file, date, thumb, progressCallback) {
+        if (document instanceof types.MessageMediaPhoto) {
+            document = document.document
+        }
+        if (!(document instanceof types.Document)) {
+            return
+        }
+        let size
+        file = file ? file : Buffer
+
+        if (thumb === null || thumb === undefined) {
+            size = null
+        } else {
+            size = this._getThumb(document.thumbs, thumb)
+            if (size instanceof types.PhotoCachedSize || size instanceof types.PhotoStrippedSize) {
+                return this._downloadCachedPhotoSize(size, file)
+            }
+        }
+        const result = await this.downloadFile(
+            new types.InputDocumentFileLocation({
+                id: document.id,
+                accessHash: document.accessHash,
+                fileReference: document.fileReference,
+                thumbSize: size ? size.type : '',
+            }),
+            file,
+            {
+                fileSize: size ? size.size : document.size,
+                progressCallback: progressCallback,
+            },
+        )
+        return result
+    }
+
+    _downloadContact(media, file) {
+        throw new Error('not implemented')
+    }
+
+    // endregion
+    async _downloadWebDocument(media, file, progressCallback) {
+        throw new Error('not implemented')
+    }
 }
 
 module.exports = TelegramClient

gramjs/extensions/index.js

@@ -4,7 +4,7 @@ const BinaryReader = require('./BinaryReader')
 const PromisedWebSockets = require('./PromisedWebSockets')
 const MessagePacker = require('./MessagePacker')
 const AsyncQueue = require('./AsyncQueue')
-const PromisedNetSocket = require('./PromisedNetSocket')
+const PromisedNetSocket = require('./PromisedNetSockets')
 module.exports = {
     BinaryWriter,
     BinaryReader,