Skip to content
Snippets Groups Projects
Commit 00b6ffbe authored by Robin VAN DE MERGHEL's avatar Robin VAN DE MERGHEL :computer:
Browse files

Adding more python support

parent b1adff3e
No related branches found
No related tags found
No related merge requests found
Show whitespace changes
Inline Side-by-side
classes/Flex.js
+ 62
− 7
View file @ 00b6ffbe
......@@ -2,6 +2,9 @@
//
// Modules
const fs = require("fs");
/**
* @param {String} string
* @returns {Array} occurences
......@@ -62,7 +65,7 @@ class Flex {
this.config.tokens.forEach(token2 => {
// Checking if the token are imbricated
if (token.global.includes(token2.global) && token !== token2) {
if (token.val.includes(token2.val) && token !== token2) {
// Adding the token to the imbricated tokens array
// We have to get the offset of the token
......@@ -71,7 +74,7 @@ class Flex {
// The offset of "in" is 2 (3-1)
// But there could be more than one occurence of the sub-token
var offset = token.global.everyOccurence(token2.global);
var offset = token.val.everyOccurence(token2.val);
imbricatedTokens.push({ token: token, subtoken: token2, offset: offset });
......@@ -82,6 +85,7 @@ class Flex {
});
// Returning the imbricated tokens array
return imbricatedTokens;
}
......@@ -101,7 +105,7 @@ class Flex {
this.config.tokens.forEach(token => {
// Checking if the token is present in the string
tokens[token.global] = string.everyOccurence(token.global);
tokens[token.type] = string.everyOccurence(token.val);
});
......@@ -127,17 +131,18 @@ class Flex {
// Looping through the imbricated tokens
imbricatedTokens.forEach(imbricatedToken => {
for (var i=0;i<tokens[imbricatedToken.token.global].length;i++) {
for (var i=0;i<tokens[imbricatedToken.token.type].length;i++) {
// For each offset, we check if the subtoken is present
for (var j=0;j<imbricatedToken.offset.length;j++) {
// If the subtoken is present, we remove it from the tokens array
let wantedToken = tokens[imbricatedToken.token.global][i] + imbricatedToken.offset[j];
if (tokens[imbricatedToken.subtoken.global].includes(wantedToken) && tokensCopy[imbricatedToken.subtoken.global].includes(wantedToken)) {
let wantedToken = tokens[imbricatedToken.token.type][i] + imbricatedToken.offset[j];
if (tokens[imbricatedToken.subtoken.type].includes(wantedToken) && tokensCopy[imbricatedToken.subtoken.type].includes(wantedToken)) {
// Removing the token from the tokens array
tokensCopy[imbricatedToken.subtoken.global].splice(tokensCopy[imbricatedToken.subtoken.global].indexOf(wantedToken), 1);
tokensCopy[imbricatedToken.subtoken.type].splice(tokensCopy[imbricatedToken.subtoken.type].indexOf(wantedToken), 1);
}
......@@ -155,6 +160,56 @@ class Flex {
}
tokenizeFile(file) {
// Reading the file
let content = fs.readFileSync(file, "utf8");
// Tokenizing the file
// For each line, we check for tokens
let lines = content.split("\n");
let tokens = [];
for (var i=0;i<lines.length;i++) {
// Checking for tokens
tokens.push(this.checkTokens(lines[i]));
}
// Tokens array is like following :
// [
// { -> line 1
// "comments": [ -> token 1 ]
// "print": [ -> token 2 ]
// },...
// ]
// There could be empty sub-arrays
// We remove them
for (var i=0;i<tokens.length;i++) {
// Looping through the tokens
for (var token in tokens[i]) {
// Checking if the token is empty
if (tokens[i][token].length === 0) {
// Removing the token
delete tokens[i][token];
}
}
}
// Returning the tokens array
return tokens;
}
}
// Export
......
config.json
+ 539
− 27
View file @ 00b6ffbe
{
"python": {
"tokens": [
{
"val": "#",
"type": "text-modifier-comment",
"global": "comment"
},
{
"val": "print",
"global": "print",
"type": "function"
"type": "print-keyword",
"global": "print"
},
{
"val": "(",
"global": "(",
"type": "paren"
"type": "opened-parenthesis-punctuation",
"global": "opened-parenthesis"
},
{
"val": ")",
"global": ")",
"type": "paren"
"type": "closed-parenthesis-punctuation",
"global": "closed-parenthesis"
},
{
"val": "{",
"type": "opened-brace-punctuation",
"global": "opened-brace"
},
{
"val": "}",
"type": "closed-brace-punctuation",
"global": "closed-brace"
},
{
"val": "[",
"type": "opened-bracket-punctuation",
"global": "opened-bracket"
},
{
"val": "]",
"type": "closed-bracket-punctuation",
"global": "closed-bracket"
},
{
"val": "if",
"type": "conditional-keyword-if",
"global": "if"
},
{
"val": "elif",
"type": "conditional-keyword-else-if",
"global": "elif"
},
{
"val": "else",
"type": "conditional-keyword-else",
"global": "else"
},
{
"val": "for",
"type": "loop-keyword-for",
"global": "for"
},
{
"val": "while",
"type": "loop-keyword-while",
"global": "while"
},
{
"val": "in",
"global": "in",
"type": "keyword"
"type": "in-keyword",
"global": "in"
},
{
"val": "not",
"type": "conditional-keyword-not",
"global": "not"
},
{
"val": "and",
"type": "conditional-keyword-and",
"global": "and"
},
{
"val": "or",
"type": "conditional-keyword-or",
"global": "or"
},
{
"val": "def",
"type": "function-keyword-def",
"global": "def"
},
{
"val": "class",
"type": "class-keyword-class",
"global": "class"
},
{
"val": "return",
"type": "return-keyword",
"global": "return"
},
{
"val": "continue",
"type": "continue-keyword",
"global": "continue"
},
{
"val": "break",
"type": "break-keyword",
"global": "break"
},
{
"val": "pass",
"type": "pass-keyword",
"global": "pass"
},
{
"val": "import",
"type": "import-keyword-import",
"global": "import"
},
{
"val": "from",
"type": "import-keyword-from",
"global": "from"
},
{
"val": "as",
"type": "import-keyword-as",
"global": "as"
},
{
"val": "try",
"type": "try-keyword-try",
"global": "try"
},
{
"val": "except",
"type": "try-keyword-except",
"global": "except"
},
{
"val": "finally",
"type": "try-keyword-finally",
"global": "finally"
},
{
"val": "raise",
"type": "try-keyword-raise",
"global": "raise"
},
{
"val": "i",
"global": "i",
"type": "variable"
"val": "with",
"type": "with-keyword-with",
"global": "with"
},
{
"val": "yield",
"type": "yield-keyword",
"global": "yield"
},
{
"val": "del",
"type": "del-keyword",
"global": "del"
},
{
"val": "global",
"type": "global-keyword",
"global": "global"
},
{
"val": "nonlocal",
"type": "nonlocal-keyword",
"global": "nonlocal"
},
{
"val": "assert",
"type": "assert-keyword",
"global": "assert"
},
{
"val": "lambda",
"type": "lambda-keyword",
"global": "lambda"
},
{
"val": "is",
"type": "conditional-keyword-is",
"global": "is"
},
{
"val": "None",
"type": "constant",
"global": "None"
},
{
"val": "True",
"type": "constant",
"global": "True"
},
{
"val": "False",
"type": "constant",
"global": "False"
},
{
"val": "self",
"type": "self-keyword",
"global": "self"
},
{
"val": "super",
"type": "super-keyword",
"global": "super"
},
{
"val": "int",
"type": "type-modifier-int",
"global": "int"
},
{
"val": "float",
"type": "type-modifier-float",
"global": "float"
},
{
"val": "str",
"type": "type-modifier-str",
"global": "str"
},
{
"val": "bool",
"type": "type-modifier-bool",
"global": "bool"
},
{
"val": "list",
"type": "type-modifier-list",
"global": "list"
},
{
"val": "tuple",
"type": "type-modifier-tuple",
"global": "tuple"
},
{
"val": "dict",
"type": "type-modifier-dict",
"global": "dict"
},
{
"val": "set",
"type": "type-modifier-set",
"global": "set"
},
{
"val": "range",
"type": "array-constructor-range",
"global": "range"
},
{
"val": "len",
"type": "array-length-getter",
"global": "length"
},
{
"val": "sum",
"type": "array-sum",
"global": "sum"
},
{
"val": "max",
"type": "array-max",
"global": "max"
},
{
"val": "min",
"type": "array-min",
"global": "min"
},
{
"val": "input",
"type": "user-input",
"global": "input"
},
{
"val": "open",
"type": "file-handling-open",
"global": "open"
},
{
"val": "close",
"type": "file-handling-close",
"global": "close"
},
{
"val": "read",
"type": "file-handling-read",
"global": "read"
},
{
"val": "write",
"type": "file-handling-write",
"global": "write"
},
{
"val": "readline",
"type": "file-handling-readline",
"global": "readline"
},
{
"val": "readlines",
"type": "file-handling-readlines",
"global": "readlines"
},
{
"val": "writelines",
"type": "file-handling-writelines",
"global": "writelines"
},
{
"val": "seek",
"type": "file-handling-seek",
"global": "seek"
},
{
"val": "tell",
"type": "file-handling-tell",
"global": "tell"
},
{
"val": "flush",
"type": "file-handling-flush",
"global": "flush"
},
{
"val": "isinstance",
"type": "isinstance",
"global": "isinstance"
},
{
"val": "issubclass",
"type": "issubclass",
"global": "issubclass"
},
{
"val": "abs",
"type": "abs",
"global": "abs"
},
{
"val": "all",
"type": "all",
"global": "all"
},
{
"val": "any",
"type": "any",
"global": "any"
},
{
"val": "ascii",
"type": "ascii",
"global": "ascii"
},
{
"val": "bin",
"type": "bin",
"global": "bin"
},
{
"val": "bool",
"type": "bool",
"global": "bool"
},
{
"val": "=",
"type": "assignment-operator-equal",
"global": "assignment"
},
{
"val": "+=",
"type": "assignment-operator-plus-equal",
"global": "assignment"
},
{
"val": "-=",
"type": "assignment-operator-minus-equal",
"global": "assignment"
},
{
"val": "*=",
"type": "assignment-operator-multiply-equal",
"global": "assignment"
},
{
"val": "/=",
"type": "assignment-operator-divide-equal",
"global": "assignment"
},
{
"val": "//=",
"type": "assignment-operator-floor-divide-equal",
"global": "assignment"
},
{
"val": "%=",
"type": "assignment-operator-modulo-equal",
"global": "assignment"
},
{
"val": "**=",
"type": "assignment-operator-power-equal",
"global": "assignment"
},
{
"val": "&=",
"type": "assignment-operator-bitwise-and-equal",
"global": "assignment"
},
{
"val": "|=",
"type": "assignment-operator-bitwise-or-equal",
"global": "assignment"
},
{
"val": "^=",
"type": "assignment-operator-bitwise-xor-equal",
"global": "assignment"
},
{
"val": "<<=",
"type": "assignment-operator-left-shift-equal",
"global": "assignment"
},
{
"val": ">>=",
"type": "assignment-operator-right-shift-equal",
"global": "assignment"
},
{
"val": "==",
"type": "comparison-operator-equal",
"global": "comparison"
},
{
"val": "!=",
"type": "comparison-operator-not-equal",
"global": "comparison"
},
{
"val": ">",
"type": "comparison-operator-greater-than",
"global": "comparison"
},
{
"val": ">=",
"type": "comparison-operator-greater-than-equal",
"global": "comparison"
},
{
"val": "<",
"type": "comparison-operator-less-than",
"global": "comparison"
},
{
"val": "<=",
"type": "comparison-operator-less-than-equal",
"global": "comparison"
},
{
"val": "+",
"type": "arithmetic-operator-plus",
"global": "arithmetic"
},
{
"val": "-",
"type": "arithmetic-operator-minus",
"global": "arithmetic"
},
{
"val": "*",
"type": "arithmetic-operator-multiply",
"global": "arithmetic"
},
{
"val": "/",
"type": "arithmetic-operator-divide",
"global": "arithmetic"
},
{
"val": "//",
"type": "arithmetic-operator-floor-divide",
"global": "arithmetic"
},
{
"val": "%",
"type": "arithmetic-operator-modulo",
"global": "arithmetic"
},
{
"val": "**",
"type": "arithmetic-operator-power",
"global": "arithmetic"
},
{
"val": "~",
"type": "bitwise-operator-invert",
"global": "bitwise"
},
{
"val": "&",
"type": "bitwise-operator-and",
"global": "bitwise"
},
{
"val": "|",
"type": "bitwise-operator-or",
"global": "bitwise"
},
{
"val": "^",
"type": "bitwise-operator-xor",
"global": "bitwise"
},
{
"val": "<<",
"type": "bitwise-operator-left-shift",
"global": "bitwise"
},
{
"val": ">>",
"type": "bitwise-operator-right-shift",
"global": "bitwise"
}
]
}
}
\ No newline at end of file
index.js
+ 4
− 2
View file @ 00b6ffbe
......@@ -19,10 +19,12 @@ const FlexEvent = require("./classes/FlexEvent");
const config = require("./config.json");
// Creating the Flex client
const flex = new Flex(config);
const flex = new Flex(config.python);
// console.log(flex.checkImbricatedTokens());
console.log(flex.checkTokens("for i in range(10): print(i)"));
// console.log(flex.checkTokens("for i in range(10): print(i)"));
console.log(flex.tokenizeFile(path.join(__dirname, "/sample/main.py")));
// Loading commands
// TODO
......
sample/main.py 0 → 100644
+ 4
− 0
View file @ 00b6ffbe
# u = 0
for i in range(10):
print(u)
u = 3 * u - 10 + 1 == 2
\ No newline at end of file
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment