diff --git a/include/lib/fex/lexer.h b/include/lib/fex/lexer.h index 9b22976d..d4d65271 100644 --- a/include/lib/fex/lexer.h +++ b/include/lib/fex/lexer.h @@ -4,6 +4,7 @@ #include #include #include +#include namespace fex { @@ -89,9 +90,7 @@ namespace fex Lexer() = default; ~Lexer() = default; - std::vector LexFile(const std::string &path); - std::vector LexString(const std::string &data); - void LexFileDumpTokens(const std::string &path, const std::string &out); + std::vector LexFile(const QString &path); private: std::vector Lex(); diff --git a/include/lib/fex/parser.h b/include/lib/fex/parser.h index 6a6b9e43..b73dd81e 100644 --- a/include/lib/fex/parser.h +++ b/include/lib/fex/parser.h @@ -21,7 +21,7 @@ namespace fex std::vector ParseTopLevelArrays(std::vector tokens); std::map ParseTopLevelObjects(std::vector tokens); - std::map ReadDefines(const std::string &filename, std::vector matching); + std::map ReadDefines(const QString &filename, std::vector matching); private: int EvaluateExpression(std::vector tokens); diff --git a/src/core/parseutil.cpp b/src/core/parseutil.cpp index 9664fdc7..2c357776 100644 --- a/src/core/parseutil.cpp +++ b/src/core/parseutil.cpp @@ -596,7 +596,7 @@ bool ParseUtil::gameStringToBool(QString gameString, bool * ok) { QMap> ParseUtil::readCStructs(const QString &filename, const QString &label, const QHash memberMap) { QString filePath = this->root + "/" + filename; auto cParser = fex::Parser(); - auto tokens = fex::Lexer().LexFile(filePath.toStdString()); + auto tokens = fex::Lexer().LexFile(filePath); auto structs = cParser.ParseTopLevelObjects(tokens); QMap> structMaps; for (auto it = structs.begin(); it != structs.end(); it++) { diff --git a/src/lib/fex/lexer.cpp b/src/lib/fex/lexer.cpp index 2dd4b249..e8545f2e 100644 --- a/src/lib/fex/lexer.cpp +++ b/src/lib/fex/lexer.cpp @@ -3,6 +3,7 @@ #include #include #include +#include namespace fex { @@ -155,48 +156,26 @@ namespace fex return Token(Token::Type::kDefine, filename_, line_number_); } - std::vector Lexer::LexString(const std::string &data) + std::vector Lexer::LexFile(const QString &path) { - filename_ = "string literal"; - line_number_ = 1; - index_ = 0; - data_ = data; - - return Lex(); - } - - std::vector Lexer::LexFile(const std::string &path) - { - filename_ = path; + filename_ = path.toStdString(); line_number_ = 1; - std::ifstream file; - file.open(path); + // Note: Using QFile instead of ifstream to handle encoding differences between platforms + // (specifically to handle accented characters on Windows) + QFile file(path); + file.open(QIODevice::ReadOnly); - std::stringstream stream; - stream << file.rdbuf(); + const QByteArray data = file.readAll(); index_ = 0; - data_ = stream.str(); + data_ = data.toStdString(); file.close(); return Lex(); } - void Lexer::LexFileDumpTokens(const std::string &path, const std::string &out) - { - std::ofstream file; - file.open(out); - - for (Token token : LexFile(path)) - { - file << token.ToString() << std::endl; - } - - file.close(); - } - std::vector Lexer::Lex() { std::vector tokens; diff --git a/src/lib/fex/parser.cpp b/src/lib/fex/parser.cpp index bb5c90a8..2e2a6f3e 100644 --- a/src/lib/fex/parser.cpp +++ b/src/lib/fex/parser.cpp @@ -337,7 +337,7 @@ namespace fex return DefineStatement(identifer, value); } - std::map Parser::ReadDefines(const std::string &filename, std::vector matching) + std::map Parser::ReadDefines(const QString &filename, std::vector matching) { std::map out;