Partially update tlobject generation

gramjs_generator/generators/index.js

@@ -1,10 +1,10 @@
 const { generateErrors } = require('./errors');
-const { generateTlobjects, cleanTlobjects } = require('./tlobject');
+const { generateTLObjects, cleanTLObjects } = require('./tlobject');
 const { generateDocs } = require('./docs');
 
 module.exports = {
     generateErrors,
-    generateTlobjects,
-    cleanTlobjects,
+    generateTLObjects,
+    cleanTLObjects,
     generateDocs,
 };

gramjs_generator/generators/tlobject.js

@@ -1,7 +1,8 @@
 const fs = require('fs');
 const util = require('util');
-const { crc32 } = require('crc');
+const {crc32} = require('crc');
 const SourceBuilder = require('../sourcebuilder');
+const {snakeToCamelCase} = require("../utils");
 
 const AUTO_GEN_NOTICE =
     "/*! File generated by TLObjects' generator. All changes will be ERASED !*/";
@@ -60,7 +61,7 @@ const writeModules = (
     typeConstructors
 ) => {
     // namespace_tlobjects: {'namespace', [TLObject]}
-    fs.mkdirSync(outDir, { recursive: true });
+    fs.mkdirSync(outDir, {recursive: true});
 
     for (const [ns, tlobjects] of Object.entries(namespaceTlobjects)) {
         const file = `${outDir}/${ns === 'null' ? 'index' : ns}.js`;
@@ -211,6 +212,37 @@ const writeModules = (
     }
 };
 
+const writeReadResult = (tlobject, builder) => {
+    // Only requests can have a different response that's not their
+    // serialized body, that is, we'll be setting their .result.
+    //
+    // The default behaviour is reading a TLObject too, so no need
+    // to override it unless necessary.
+    if (!tlobject.isFunction)
+        return;
+
+    // https://core.telegram.org/mtproto/serialize#boxed-and-bare-types
+    // TL;DR; boxed types start with uppercase always, so we can use
+    // this to check whether everything in it is boxed or not.
+    //
+    // Currently only un-boxed responses are Vector<int>/Vector<long>.
+    // If this weren't the case, we should check upper case after
+    // max(index('<'), index('.')) (and if it is, it's boxed, so return).
+    let m = tlobject.result.match(/Vector<(int|long)>/);
+    if (!m) {
+        return
+    }
+    builder.endBlock();
+    builder.writeln('static readResult(reader){');
+    builder.writeln('reader.readInt()  // Vector ID');
+    builder.writeln('let temp = [];');
+    builder.writeln('for (let i=0;i<reader.readInt();i++){');
+    builder.writeln('temp.push(reader.read%s', m[0]);
+    builder.writeln("}");
+    builder.writeln('return temp');
+
+};
+
 /**
  * Writes the source code corresponding to the given TLObject
  * by making use of the ``builder`` `SourceBuilder`.
@@ -222,14 +254,16 @@ const writeModules = (
 const writeSourceCode = (tlobject, kind, builder, typeConstructors) => {
     writeClassConstructor(tlobject, kind, typeConstructors, builder);
     writeResolve(tlobject, builder);
-    writeToJson(tlobject, builder);
+    //writeToJson(tlobject, builder);
     writeToBytes(tlobject, builder);
     builder.currentIndent--;
+    writeFromReader(tlobject, builder);
+    writeReadResult(tlobject, builder);
     builder.writeln('}');
-    // writeFromReader(tlobject, builder);
-    // writeReadResult(tlobject, builder);
+
 };
 
+
 const writeClassConstructor = (tlobject, kind, typeConstructors, builder) => {
     builder.writeln();
     builder.writeln();
@@ -291,9 +325,9 @@ const writeClassConstructor = (tlobject, kind, typeConstructors, builder) => {
         // inferred are those called 'random_id'
         else if (arg.name === 'random_id') {
             // Endianness doesn't really matter, and 'big' is shorter
-            let code = `int.from_bytes(os.urandom(${
+            let code = `int.from_bytes(Helpers.generateRandomBytes(${
                 arg.type === 'long' ? 8 : 4
-            }), 'big', signed=True)`;
+            }))`;
 
             if (arg.isVector) {
                 // Currently for the case of "messages.forwardMessages"
@@ -308,7 +342,7 @@ const writeClassConstructor = (tlobject, kind, typeConstructors, builder) => {
             }
 
             builder.writeln(
-                `this.random_id = random_id !== null ? random_id : ${code}`
+                `this.randomId = randomId !== null ? randomId : ${code}`
             );
         } else {
             throw new Error(`Cannot infer a value for ${arg}`);
@@ -367,8 +401,8 @@ const writeResolve = (tlobject, builder) => {
         builder.endBlock();
     }
 };
-
-const writeToJson = (tlobject, builder) => {
+/**
+ const writeToJson = (tlobject, builder) => {
     builder.writeln('toJson() {');
     builder.writeln('return {');
 
@@ -411,7 +445,7 @@ const writeToJson = (tlobject, builder) => {
     builder.currentIndent--;
     builder.writeln('}');
 };
-
+ */
 const writeToBytes = (tlobject, builder) => {
     builder.writeln('get bytes() {');
 
@@ -424,7 +458,6 @@ const writeToBytes = (tlobject, builder) => {
             if (!repeatedArgs[arg.flagIndex]) {
                 repeatedArgs[arg.flagIndex] = [];
             }
-
             repeatedArgs[arg.flagIndex].push(arg);
         }
     }
@@ -448,7 +481,6 @@ const writeToBytes = (tlobject, builder) => {
                 "throw new Error('%s parameters must all be false-y (like null) or all me true-y');",
                 ra.map(a => a.name).join(', ')
             );
-
             builder.endBlock();
         }
     }
@@ -459,12 +491,13 @@ const writeToBytes = (tlobject, builder) => {
             .padStart(8, `0`),
         `hex`
     )
-        .readUInt32LE()
+        .readUInt32LE(0)
         .toString(16)
         .padStart(8, `0`);
 
-    builder.writeln('return parseInt([');
+    builder.writeln('return Buffer.concat([');
     builder.currentIndent++;
+
     builder.writeln("'%s',", bytes);
 
     for (const arg of tlobject.args) {
@@ -479,15 +512,27 @@ const writeToBytes = (tlobject, builder) => {
 };
 
 // writeFromReader
+const writeFromReader = (tlobject, builder) => {
+    builder.writeln("static fromReader(reader) {");
+    for (const arg of tlobject.args) {
+        writeArgReadCode(builder, arg, tlobject.args, "_" + arg.name);
+    }
+    let temp = [];
+    for (let a of tlobject.realArgs) {
+        temp.push(`${a.name}:_${a.name}`)
+    }
+    builder.writeln("return this({%s})", temp.join(",\n\t"));
+    builder.writeln("}");
+};
 // writeReadResult
 
 /**
  * Writes the .__bytes__() code for the given argument
- * :param builder: The source code builder
- * :param arg: The argument to write
- * :param args: All the other arguments in TLObject same __bytes__.
+ * @param builder: The source code builder
+ * @param arg: The argument to write
+ * @param args: All the other arguments in TLObject same __bytes__.
  *              This is required to determine the flags value
- * :param name: The name of the argument. Defaults to "self.argname"
+ * @param name: The name of the argument. Defaults to "self.argname"
  *              This argument is an option because it's required when
  *              writing Vectors<>
  */
@@ -512,12 +557,12 @@ const writeArgToBytes = (builder, arg, args, name = null) => {
             // so we need an extra join here. Note that empty vector flags
             // should NOT be sent either!
             builder.write(
-                "%s === null || %s === false ? b'' : b''.join([",
+                "%s === null || %s === false ? Buffer.alloc(0) :Buffer.concat([",
                 name,
                 name
             );
         } else {
-            builder.write("%s === null || %s === false ? b'' : [", name, name);
+            builder.write("%s === null || %s === false ? Buffer.alloc(0) : [", name, name);
         }
     }
 
@@ -530,8 +575,7 @@ const writeArgToBytes = (builder, arg, args, name = null) => {
 
         // Cannot unpack the values for the outer tuple through *[(
         // since that's a Python >3.5 feature, so add another join.
-        builder.write('Buffer.concat(%s.map(x => ', name);
-
+        builder.write('Buffer.concat([%s.map(x => ', name);
         // Temporary disable .is_vector, not to enter this if again
         // Also disable .is_flag since it's not needed per element
         const oldFlag = arg.isFlag;
@@ -540,7 +584,7 @@ const writeArgToBytes = (builder, arg, args, name = null) => {
         arg.isVector = true;
         arg.isFlag = oldFlag;
 
-        builder.write('))', name);
+        builder.write(')]),', name);
     } else if (arg.flagIndicator) {
         // Calculate the flags with those items which are not None
         if (!args.some(f => f.isFlag)) {
@@ -566,9 +610,9 @@ const writeArgToBytes = (builder, arg, args, name = null) => {
     } else if (arg.type === 'long') {
         builder.write("struct.pack('<q', %s)", name);
     } else if (arg.type === 'int128') {
-        builder.write("%s.to_bytes(16, 'little', signed=True)", name);
+        builder.write("BigIntBuffer.toBufferLE(BigInt(%s,16))", name);
     } else if (arg.type === 'int256') {
-        builder.write("%s.to_bytes(32, 'little', signed=True)", name);
+        builder.write("BigIntBuffer.toBufferLE(BigInt(%s,32))", name);
     } else if (arg.type === 'double') {
         builder.write("struct.pack('<d', %s)", name);
     } else if (arg.type === 'string') {
@@ -583,7 +627,7 @@ const writeArgToBytes = (builder, arg, args, name = null) => {
         builder.write('this.serializeDatetime(%s)', name);
     } else {
         // Else it may be a custom type
-        builder.write('bytes(%s)', name);
+        builder.write('%s.bytes', name);
 
         // If the type is not boxed (i.e. starts with lowercase) we should
         // not serialize the constructor ID (so remove its first 4 bytes).
@@ -606,8 +650,123 @@ const writeArgToBytes = (builder, arg, args, name = null) => {
     return true;
 };
 
+
+/**
+ * Writes the read code for the given argument, setting the
+ * arg.name variable to its read value.
+ * @param builder The source code builder
+ * @param arg The argument to write
+ * @param args All the other arguments in TLObject same on_send.
+ * This is required to determine the flags value
+ * @param name The name of the argument. Defaults to "self.argname"
+ * This argument is an option because it's required when
+ * writing Vectors<>
+ */
+const writeArgReadCode = (builder, arg, args, name) => {
+    if (arg.genericDefinition) {
+        return // Do nothing, this only specifies a later type
+    }
+    //The argument may be a flag, only write that flag was given!
+    let wasFlag = false;
+    if (arg.isFlag) {
+        // Treat 'true' flags as a special case, since they're true if
+        // they're set, and nothing else needs to actually be read.
+        if (arg.type === "true") {
+            builder.writeln("let %s = Boolean(flags & %s)", name, 1 << arg.flagIndex);
+            return;
+        }
+
+        wasFlag = true;
+        builder.writeln("if (flags & %s)", 1 << arg.flagIndex);
+        // Temporary disable .is_flag not to enter this if
+        // again when calling the method recursively
+        arg.isFlag = false;
+    }
+
+    if (arg.isVector) {
+        if (arg.useVectorId) {
+            // We have to read the vector's constructor ID
+            builder.writeln("reader.readInt()");
+        }
+        builder.writeln("let %s = []", name);
+        builder.writeln("for (let i=0;i<reader.readInt();i++){");
+        // Temporary disable .is_vector, not to enter this if again
+        arg.isVector = false;
+        writeArgReadCode(builder, arg, args, "_x");
+        builder.writeln("%s.push(_x)", name);
+        arg.isVector = true;
+
+    } else if (arg.flagIndicator) {
+        //Read the flags, which will indicate what items we should read next
+        builder.writeln("let flags = reader.readInt()");
+        builder.writeln();
+    } else if (arg.type === "int") {
+        builder.writeln("let %s = reader.readInt()", name)
+    } else if (arg.type === "long") {
+        builder.writeln("let %s = reader.readInt()", name);
+    } else if (arg.type === "int128") {
+        builder.writeln('let %s = reader.readLargeInt(bits=128)', name);
+    } else if (arg.type === "int256") {
+        builder.writeln('let %s = reader.readLargeInt(bits=256)', name);
+    } else if (arg.type === "double") {
+        builder.writeln('let %s = reader.readDouble()', name);
+    } else if (arg.type === "string") {
+        builder.writeln('let %s = reader.tgReadString()', name);
+    } else if (arg.type === "Bool") {
+        builder.writeln('let %s = reader.tgReadBool()', name);
+    } else if (arg.type === "true") {
+        builder.writeln('let %s = true', name);
+    } else if (arg.type === "bytes") {
+        builder.writeln('let %s = reader.tgReadBytes()', name);
+    } else if (arg.type === "date") {
+        builder.writeln('let %s = reader.tgReadDate()', name);
+    } else {
+        // Else it may be a custom type
+        if (!arg.skipConstructorId) {
+            builder.writeln('let %s = reader.tgReadObject()', name);
+        } else {
+            // Import the correct type inline to avoid cyclic imports.
+            // There may be better solutions so that we can just access
+            // all the types before the files have been parsed, but I
+            // don't know of any.
+            let sepIndex = arg.type.indexOf(".");
+            let ns, t;
+            if (sepIndex === -1) {
+                ns = ".";
+                t = arg.type;
+            } else {
+                ns = "." + arg.type.slice(0, sepIndex);
+                t = arg.type.slice(sepIndex + 1);
+            }
+            let className = snakeToCamelCase(t);
+
+            // There would be no need to import the type if we're in the
+            // file with the same namespace, but since it does no harm
+            // and we don't have information about such thing in the
+            // method we just ignore that case.
+            builder.writeln('const %s = require("%S")', className, ns);
+            builder.writeln("let %s = %s.fromReader(reader)", name, className);
+        }
+    }
+
+    // End vector and flag blocks if required (if we opened them before)
+    if (arg.isVector) {
+        builder.endBlock();
+    }
+
+    if (wasFlag) {
+        builder.currentIndent -= 1;
+        builder.writeln("else");
+        builder.writeln("let %s = null", name);
+        builder.currentIndent -= 1;
+        // Restore .isFlag;
+        arg.isFlag = true;
+    }
+};
+
+
 const writePatched = (outDir, namespaceTlobjects) => {
-    fs.mkdirSync(outDir, { recursive: true });
+    fs.mkdirSync(outDir, {recursive: true});
 
     for (const [ns, tlobjects] of Object.entries(namespaceTlobjects)) {
         const file = `${outDir}/${ns === 'null' ? 'index' : ns}.js`;
@@ -632,9 +791,9 @@ const writePatched = (outDir, namespaceTlobjects) => {
             builder.writeln(`this.SUBCLASS_OF_ID = 0x${crc32(t.result)}`);
             builder.endBlock();
 
-            writeToJson(t, builder);
+            //writeToJson(t, builder);
             writeToBytes(t, builder);
-            // writeFromReader(t, builder);
+            writeFromReader(t, builder);
 
             builder.writeln();
             builder.writeln(
@@ -648,7 +807,7 @@ const writePatched = (outDir, namespaceTlobjects) => {
     }
 };
 
-const writeAllTlobjects = (tlobjects, layer, builder) => {
+const writeAllTLObjects = (tlobjects, layer, builder) => {
     builder.writeln(AUTO_GEN_NOTICE);
     builder.writeln();
 
@@ -687,11 +846,13 @@ const writeAllTlobjects = (tlobjects, layer, builder) => {
     builder.endBlock(true);
 };
 
-const generateTlobjects = (tlobjects, layer, importDepth, outputDir) => {
+const generateTLObjects = (tlobjects, layer, importDepth, outputDir) => {
+    // Group everything by {namespace :[tlobjects]} to generate index.js
     const namespaceFunctions = {};
     const namespaceTypes = {};
     const namespacePatched = {};
 
+    // Group {type: [constructors]} to generate the documentation
     const typeConstructors = {};
 
     for (const tlobject of tlobjects) {
@@ -737,16 +898,16 @@ const generateTlobjects = (tlobjects, layer, importDepth, outputDir) => {
         namespaceTypes,
         typeConstructors
     );
-    writePatched(`${outputDir}/pathced`, namespacePatched);
+    writePatched(`${outputDir}/patched`, namespacePatched);
 
     const filename = `${outputDir}/alltlobjects.js`;
     const stream = fs.createWriteStream(filename);
     const builder = new SourceBuilder(stream);
 
-    writeAllTlobjects(tlobjects, layer, builder);
+    writeAllTLObjects(tlobjects, layer, builder);
 };
 
-const cleanTlobjects = outputDir => {
+const cleanTLObjects = outputDir => {
     for (let d in ['functions', 'types', 'patched']) {
         d = `${outputDir}/d`;
 
@@ -774,6 +935,6 @@ const writeModuleExports = (tlobjects, builder) => {
 };
 
 module.exports = {
-    generateTlobjects,
-    cleanTlobjects,
+    generateTLObjects,
+    cleanTLObjects,
 };

gramjs_generator/sourcebuilder.js

@@ -19,7 +19,7 @@ class SourceBuilder {
      * by the current indentation level
      */
     indent() {
-        this.write(' '.repeat(this.currentIndent * this.indentSize));
+        this.write(' '.repeat(Math.abs(this.currentIndent * this.indentSize)));
     }
 
     /**

index.js

@@ -51,9 +51,9 @@ const generate = (which, action = 'gen') => {
 
     const {
         generateErrors,
-        generateTlobjects,
+        generateTLObjects,
         generateDocs,
-        cleanTlobjects,
+        cleanTLObjects,
     } = require('./gramjs_generator/generators');
 
     const [layer] = TLOBJECT_IN_TLS.map(findLayer).filter(Boolean);
@@ -96,9 +96,9 @@ const generate = (which, action = 'gen') => {
         console.log(action, 'TLObjects...');
 
         if (clean) {
-            cleanTlobjects(TLOBJECT_OUT);
+            cleanTLObjects(TLOBJECT_OUT);
         } else {
-            generateTlobjects(tlobjects, layer, IMPORT_DEPTH, TLOBJECT_OUT);
+            generateTLObjects(tlobjects, layer, IMPORT_DEPTH, TLOBJECT_OUT);
         }
     }