Add by-name and recursive define evaluation

include/core/parseutil.h

@@ -48,16 +48,15 @@ public:
     void invalidateTextFile(const QString &path);
     static int textFileLineCount(const QString &path);
     QList<QStringList> parseAsm(const QString &filename);
-    int evaluateDefine(const QString&, const QMap<QString, int>&);
     QStringList readCArray(const QString &filename, const QString &label);
     QMap<QString, QStringList> readCArrayMulti(const QString &filename);
     QMap<QString, QString> readNamedIndexCArray(const QString &text, const QString &label);
     QString readCIncbin(const QString &text, const QString &label);
     QMap<QString, QString> readCIncbinMulti(const QString &filepath);
     QStringList readCIncbinArray(const QString &filename, const QString &label);
-    QMap<QString, int> readCDefines(const QString &filename, const QStringList &prefixes, QMap<QString, int> = { });
+    QMap<QString, int> readCDefinesByPrefix(const QString &filename, const QStringList &prefixes);
+    QMap<QString, int> readCDefinesByName(const QString &filename, const QStringList &defineNames);
     QStringList readCDefineNames(const QString&, const QStringList&);
-    QStringList readCDefineNamesByValue(const QString&, const QStringList&, const QMap<QString, int>& = { });
     QMap<QString, QHash<QString, QString>> readCStructs(const QString &, const QString & = "", const QHash<int, QString> = { });
     QList<QStringList> getLabelMacros(const QList<QStringList>&, const QString&);
     QStringList getLabelValues(const QList<QStringList>&, const QString&);
@@ -90,14 +89,16 @@ private:
     QString file;
     QString curDefine;
     QHash<QString, QStringList> errorMap;
-    QString readCDefinesFile(const QString &filename);
+    int evaluateDefine(const QString&, const QString &, QMap<QString, int>*, QMap<QString, QString>*);
-    QList<Token> tokenizeExpression(QString expression, const QMap<QString, int> &knownIdentifiers);
+    QList<Token> tokenizeExpression(QString, QMap<QString, int>*, QMap<QString, QString>*);
     QList<Token> generatePostfix(const QList<Token> &tokens);
     int evaluatePostfix(const QList<Token> &postfix);
     void recordError(const QString &message);
     void recordErrors(const QStringList &errors);
     void logRecordedErrors();
     QString createErrorMessage(const QString &message, const QString &expression);
+    QString readCDefinesFile(const QString &filename);
+    QMap<QString, int> readCDefines(const QString &filename, const QStringList &searchText, bool fullMatch);
 
     static const QRegularExpression re_incScriptLabel;
     static const QRegularExpression re_globalIncScriptLabel;

include/project.h

@@ -177,7 +177,6 @@ public:
     bool readTilesetLabels();
     bool readTilesetProperties();
     bool readTilesetMetatileLabels();
-    bool readMaxMapDataSize();
     bool readRegionMapSections();
     bool readItemNames();
     bool readFlagNames();

src/core/parseutil.cpp

@@ -106,16 +106,31 @@ QList<QStringList> ParseUtil::parseAsm(const QString &filename) {
     return parsed;
 }
 
-int ParseUtil::evaluateDefine(const QString &define, const QMap<QString, int> &knownDefines) {
+// 'identifier' is the name of the #define to evaluate, e.g. 'FOO' in '#define FOO (BAR+1)'
-    QList<Token> tokens = tokenizeExpression(define, knownDefines);
+// 'expression' is the text of the #define to evaluate, e.g. '(BAR+1)' in '#define FOO (BAR+1)'
+// 'knownValues' is a pointer to a map of identifier->values for defines that have already been evaluated.
+// 'unevaluatedExpressions' is a pointer to a map of identifier->expressions for defines that have not been evaluated. If this map contains any
+//   identifiers found in 'expression' then this function will be called recursively to evaluate that define first.
+// This function will maintain the passed maps appropriately as new #defines are evaluated.
+int ParseUtil::evaluateDefine(const QString &identifier, const QString &expression, QMap<QString, int> *knownValues, QMap<QString, QString> *unevaluatedExpressions) {
+    if (unevaluatedExpressions->contains(identifier))
+        unevaluatedExpressions->remove(identifier);
+
+    if (knownValues->contains(identifier))
+        return knownValues->value(identifier);
+
+    QList<Token> tokens = tokenizeExpression(expression, knownValues, unevaluatedExpressions);
     QList<Token> postfixExpression = generatePostfix(tokens);
-    return evaluatePostfix(postfixExpression);
+    int value = evaluatePostfix(postfixExpression);
+
+    knownValues->insert(identifier, value);
+    return value;
 }
 
-QList<Token> ParseUtil::tokenizeExpression(QString expression, const QMap<QString, int> &knownIdentifiers) {
+QList<Token> ParseUtil::tokenizeExpression(QString expression, QMap<QString, int> *knownValues, QMap<QString, QString> *unevaluatedExpressions) {
     QList<Token> tokens;
 
-    QStringList tokenTypes = (QStringList() << "hex" << "decimal" << "identifier" << "operator" << "leftparen" << "rightparen");
+    static const QStringList tokenTypes = {"hex", "decimal", "identifier", "operator", "leftparen", "rightparen"};
     static const QRegularExpression re("^(?<hex>0x[0-9a-fA-F]+)|(?<decimal>[0-9]+)|(?<identifier>[a-zA-Z_0-9]+)|(?<operator>[+\\-*\\/<>|^%]+)|(?<leftparen>\\()|(?<rightparen>\\))");
 
     expression = expression.trimmed();
@@ -129,10 +144,14 @@ QList<Token> ParseUtil::tokenizeExpression(QString expression, const QMap<QStrin
             QString token = match.captured(tokenType);
             if (!token.isEmpty()) {
                 if (tokenType == "identifier") {
-                    if (knownIdentifiers.contains(token)) {
+                    if (unevaluatedExpressions->contains(token)) {
+                        // This expression depends on a define we know of but haven't evaluated. Evaluate it now
+                        evaluateDefine(token, unevaluatedExpressions->value(token), knownValues, unevaluatedExpressions);
+                    }
+                    if (knownValues->contains(token)) {
                         // Any errors encountered when this identifier was evaluated should be recorded for this expression as well.
                         recordErrors(this->errorMap.value(token));
-                        QString actualToken = QString("%1").arg(knownIdentifiers.value(token));
+                        QString actualToken = QString("%1").arg(knownValues->value(token));
                         expression = expression.replace(0, token.length(), actualToken);
                         token = actualToken;
                         tokenType = "decimal";
@@ -357,50 +376,72 @@ QString ParseUtil::readCDefinesFile(const QString &filename)
     return this->text;
 }
 
-QMap<QString, int> ParseUtil::readCDefines(const QString &filename,
+// Read all the define names and their expressions in the specified file, then evaluate the ones matching the search text (and any they depend on).
-                                           const QStringList &prefixes,
+// If 'fullMatch' is true, 'searchText' is a list of exact define names to evaluate and return.
-                                           QMap<QString, int> allDefines)
+// If 'fullMatch' is false, 'searchText' is a list of prefixes or regexes for define names to evaluate and return.
+QMap<QString, int> ParseUtil::readCDefines(const QString &filename, const QStringList &searchText, bool fullMatch)
 {
-    QMap<QString, int> filteredDefines;
+    QMap<QString, int> filteredValues;
 
     this->text = this->readCDefinesFile(filename);
     if (this->text.isEmpty()) {
-        return filteredDefines;
+        return filteredValues;
     }
 
-    allDefines.insert("FALSE", 0);
+    // Extract all the define names and expressions
-    allDefines.insert("TRUE", 1);
+    QMap<QString, QString> allExpressions;
-
+    QMap<QString, QString> filteredExpressions;
     static const QRegularExpression re("#define\\s+(?<defineName>\\w+)[^\\S\\n]+(?<defineValue>.+)");
     QRegularExpressionMatchIterator iter = re.globalMatch(this->text);
-    this->errorMap.clear();
     while (iter.hasNext()) {
         QRegularExpressionMatch match = iter.next();
-        QString name = match.captured("defineName");
+        const QString name = match.captured("defineName");
-        QString expression = match.captured("defineValue");
+        const QString expression = match.captured("defineValue");
+        // If name matches the search text record it for evaluation.
+        for (auto s : searchText) {
+            if ((fullMatch && name == s) || (!fullMatch && (name.startsWith(s) || QRegularExpression(s).match(name).hasMatch()))) {
+                filteredExpressions.insert(name, expression);
+                break;
+            }
+        }
+        allExpressions.insert(name, expression);
+    }
+
+    QMap<QString, int> allValues;
+    allValues.insert("FALSE", 0);
+    allValues.insert("TRUE", 1);
+
+    // Evaluate defines
+    this->errorMap.clear();
+    while (!filteredExpressions.isEmpty()) {
+        const QString name = filteredExpressions.firstKey();
+        const QString expression = filteredExpressions.take(name);
         if (expression == " ") continue;
         this->curDefine = name;
-        int value = evaluateDefine(expression, allDefines);
+        filteredValues.insert(name, evaluateDefine(name, expression, &allValues, &allExpressions));
-        allDefines.insert(name, value);
+        logRecordedErrors(); // Only log errors for defines that Porymap is looking for
-        for (QString prefix : prefixes) {
-            if (name.startsWith(prefix) || QRegularExpression(prefix).match(name).hasMatch()) {
-                // Only log errors for defines that Porymap is looking for
-                logRecordedErrors();
-                filteredDefines.insert(name, value);
     }
-        }
+    return filteredValues;
-    }
+}
-    return filteredDefines;
+
+// Find and evaluate an unknown list of defines with a known name prefix.
+QMap<QString, int> ParseUtil::readCDefinesByPrefix(const QString &filename, const QStringList &prefixes) {
+    return this->readCDefines(filename, prefixes, false);
+}
+
+// Find and evaluate a specific set of defines with known names.
+QMap<QString, int> ParseUtil::readCDefinesByName(const QString &filename, const QStringList &names) {
+    return this->readCDefines(filename, names, true);
 }
 
 // Similar to readCDefines, but for cases where we only need to show a list of define names.
-// We can skip evaluating each define (and by extension skip reporting any errors from this process).
+// We can skip reading/evaluating any expressions (and by extension skip reporting any errors from this process).
 QStringList ParseUtil::readCDefineNames(const QString &filename, const QStringList &prefixes) {
-    QStringList filteredDefines;
+    QStringList filteredNames;
 
     this->text = this->readCDefinesFile(filename);
     if (this->text.isEmpty()) {
-        return filteredDefines;
+        return filteredNames;
     }
 
     static const QRegularExpression re("#define\\s+(?<defineName>\\w+)[^\\S\\n]+");
@@ -410,26 +451,11 @@ QStringList ParseUtil::readCDefineNames(const QString &filename, const QStringLi
         QString name = match.captured("defineName");
         for (QString prefix : prefixes) {
             if (name.startsWith(prefix) || QRegularExpression(prefix).match(name).hasMatch()) {
-                filteredDefines.append(name);
+                filteredNames.append(name);
             }
         }
     }
-    return filteredDefines;
+    return filteredNames;
-}
-
-QStringList ParseUtil::readCDefineNamesByValue(const QString &filename,
-                                               const QStringList &prefixes,
-                                               const QMap<QString, int> &knownDefines)
-{
-    QMap<QString, int> defines = readCDefines(filename, prefixes, knownDefines);
-
-    // The defines should be sorted by their underlying value, not alphabetically or in parse order.
-    // Reverse the map and read out the resulting keys in order.
-    QMultiMap<int, QString> definesInverse;
-    for (QString defineName : defines.keys()) {
-        definesInverse.insert(defines[defineName], defineName);
-    }
-    return definesInverse.values();
 }
 
 QStringList ParseUtil::readCArray(const QString &filename, const QString &label) {

src/mainwindow.cpp

@@ -936,7 +936,6 @@ bool MainWindow::loadDataStructures() {
                 && project->readTilesetLabels()
                 && project->readTilesetMetatileLabels()
                 && project->readFieldmapMasks()
-                && project->readMaxMapDataSize()
                 && project->readHealLocations()
                 && project->readMiscellaneousConstants()
                 && project->readSpeciesIconPaths()

src/project.cpp

@@ -1505,7 +1505,8 @@ bool Project::readTilesetMetatileLabels() {
     QString metatileLabelsFilename = projectConfig.getFilePath(ProjectFilePath::constants_metatile_labels);
     fileWatcher.addPath(root + "/" + metatileLabelsFilename);
 
-    QMap<QString, int> defines = parser.readCDefines(metatileLabelsFilename, QStringList() << "METATILE_");
+    static const QStringList prefixes = {"METATILE_"};
+    QMap<QString, int> defines = parser.readCDefinesByPrefix(metatileLabelsFilename, prefixes);
 
     for (QString label : defines.keys()) {
         QString tilesetName = findMetatileLabelsTileset(label);
@@ -1861,11 +1862,19 @@ bool Project::readTilesetLabels() {
     return success;
 }
 
+// TODO: Names to config
 bool Project::readTilesetProperties() {
-    QStringList definePrefixes{ "\\bNUM_" };
+    static const QStringList names = {
+        "NUM_TILES_IN_PRIMARY",
+        "NUM_TILES_TOTAL",
+        "NUM_METATILES_IN_PRIMARY",
+        "NUM_PALS_IN_PRIMARY",
+        "NUM_PALS_TOTAL",
+        "MAX_MAP_DATA_SIZE",
+    };
     QString filename = projectConfig.getFilePath(ProjectFilePath::constants_fieldmap);
     fileWatcher.addPath(root + "/" + filename);
-    QMap<QString, int> defines = parser.readCDefines(filename, definePrefixes);
+    QMap<QString, int> defines = parser.readCDefinesByName(filename, names);
 
     auto it = defines.find("NUM_TILES_IN_PRIMARY");
     if (it != defines.end()) {
@@ -1907,16 +1916,42 @@ bool Project::readTilesetProperties() {
         logWarn(QString("Value for tileset property 'NUM_PALS_TOTAL' not found. Using default (%1) instead.")
                 .arg(Project::num_pals_total));
     }
+
+    it = defines.find("MAX_MAP_DATA_SIZE");
+    if (it != defines.end()) {
+        int min = getMapDataSize(1, 1);
+        if (it.value() >= min) {
+            Project::max_map_data_size = it.value();
+            calculateDefaultMapSize();
+        } else {
+            // must be large enough to support a 1x1 map
+            logWarn(QString("Value for map property 'MAX_MAP_DATA_SIZE' is %1, must be at least %2. Using default (%3) instead.")
+                    .arg(it.value())
+                    .arg(min)
+                    .arg(Project::max_map_data_size));
+        }
+    }
+    else {
+        logWarn(QString("Value for map property 'MAX_MAP_DATA_SIZE' not found. Using default (%1) instead.")
+                .arg(Project::max_map_data_size));
+    }
+
     return true;
 }
 
+// TODO: Update for config
 // Read data masks for Blocks and metatile attributes.
 bool Project::readFieldmapMasks() {
-    // We're looking for the suffix "_MASK". Technically our "prefix" is the whole define.
+    static const QStringList searchNames = {
-    QStringList definePrefixes{ "\\b\\w+_MASK" };
+        ProjectConfig::metatileIdMaskName,
+        ProjectConfig::collisionMaskName,
+        ProjectConfig::elevationMaskName,
+        ProjectConfig::behaviorMaskName,
+        ProjectConfig::layerTypeMaskName,
+    };
     QString globalFieldmap = projectConfig.getFilePath(ProjectFilePath::global_fieldmap);
     fileWatcher.addPath(root + "/" + globalFieldmap);
-    QMap<QString, int> defines = parser.readCDefines(globalFieldmap, definePrefixes);
+    QMap<QString, int> defines = parser.readCDefinesByName(globalFieldmap, searchNames);
 
     // These mask values are accessible via the settings editor for users who don't have these defines.
     // If users do have the defines we disable them in the settings editor and direct them to their project files.
@@ -1986,40 +2021,14 @@ bool Project::readFieldmapMasks() {
     return true;
 }
 
-bool Project::readMaxMapDataSize() {
-    QStringList definePrefixes{ "\\bMAX_" };
-    QString filename = projectConfig.getFilePath(ProjectFilePath::constants_fieldmap); // already in fileWatcher from readTilesetProperties
-    QMap<QString, int> defines = parser.readCDefines(filename, definePrefixes);
-
-    auto it = defines.find("MAX_MAP_DATA_SIZE");
-    if (it != defines.end()) {
-        int min = getMapDataSize(1, 1);
-        if (it.value() >= min) {
-            Project::max_map_data_size = it.value();
-            calculateDefaultMapSize();
-        } else {
-            // must be large enough to support a 1x1 map
-            logWarn(QString("Value for map property 'MAX_MAP_DATA_SIZE' is %1, must be at least %2. Using default (%3) instead.")
-                    .arg(it.value())
-                    .arg(min)
-                    .arg(Project::max_map_data_size));
-        }
-    }
-    else {
-        logWarn(QString("Value for map property 'MAX_MAP_DATA_SIZE' not found. Using default (%1) instead.")
-                .arg(Project::max_map_data_size));
-    }
-    return true;
-}
-
 bool Project::readRegionMapSections() {
     this->mapSectionNameToValue.clear();
     this->mapSectionValueToName.clear();
 
-    QStringList prefixes = (QStringList() << "\\bMAPSEC_");
+    static const QStringList prefixes = {"\\bMAPSEC_"};
     QString filename = projectConfig.getFilePath(ProjectFilePath::constants_region_map_sections);
     fileWatcher.addPath(root + "/" + filename);
-    this->mapSectionNameToValue = parser.readCDefines(filename, prefixes);
+    this->mapSectionNameToValue = parser.readCDefinesByPrefix(filename, prefixes);
     if (this->mapSectionNameToValue.isEmpty()) {
         logError(QString("Failed to read region map sections from %1.").arg(filename));
         return false;
@@ -2034,10 +2043,10 @@ bool Project::readRegionMapSections() {
 // Read the constants to preserve any "unused" heal locations when writing the file later
 bool Project::readHealLocationConstants() {
     this->healLocationNameToValue.clear();
-    QStringList prefixes{ "\\bSPAWN_", "\\bHEAL_LOCATION_" };
+    static const QStringList prefixes = {"\\bSPAWN_", "\\bHEAL_LOCATION_"};
     QString constantsFilename = projectConfig.getFilePath(ProjectFilePath::constants_heal_locations);
     fileWatcher.addPath(root + "/" + constantsFilename);
-    this->healLocationNameToValue = parser.readCDefines(constantsFilename, prefixes);
+    this->healLocationNameToValue = parser.readCDefinesByPrefix(constantsFilename, prefixes);
     // No need to check if empty, not finding any heal location constants is ok
     return true;
 }
@@ -2276,10 +2285,10 @@ bool Project::readMetatileBehaviors() {
     this->metatileBehaviorMap.clear();
     this->metatileBehaviorMapInverse.clear();
 
-    QStringList prefixes("\\bMB_");
+    static const QStringList prefixes = {"\\bMB_"};
     QString filename = projectConfig.getFilePath(ProjectFilePath::constants_metatile_behaviors);
     fileWatcher.addPath(root + "/" + filename);
-    this->metatileBehaviorMap = parser.readCDefines(filename, prefixes);
+    this->metatileBehaviorMap = parser.readCDefinesByPrefix(filename, prefixes);
     if (this->metatileBehaviorMap.isEmpty()) {
         logError(QString("Failed to read metatile behaviors from %1.").arg(filename));
         return false;
@@ -2318,10 +2327,10 @@ bool Project::readSongNames() {
 }
 
 bool Project::readObjEventGfxConstants() {
-    QStringList objEventGfxPrefixes("\\bOBJ_EVENT_GFX_");
+    static const QStringList prefixes = {"\\bOBJ_EVENT_GFX_"};
     QString filename = projectConfig.getFilePath(ProjectFilePath::constants_obj_events);
     fileWatcher.addPath(root + "/" + filename);
-    this->gfxDefines = parser.readCDefines(filename, objEventGfxPrefixes);
+    this->gfxDefines = parser.readCDefinesByPrefix(filename, prefixes);
     if (this->gfxDefines.isEmpty()) {
         logError(QString("Failed to read object event graphics constants from %1.").arg(filename));
         return false;
@@ -2329,20 +2338,20 @@ bool Project::readObjEventGfxConstants() {
     return true;
 }
 
+// TODO: Names to config
 bool Project::readMiscellaneousConstants() {
     miscConstants.clear();
     if (userConfig.getEncounterJsonActive()) {
         QString filename = projectConfig.getFilePath(ProjectFilePath::constants_pokemon);
         fileWatcher.addPath(root + "/" + filename);
-        QMap<QString, int> pokemonDefines = parser.readCDefines(filename, { "MIN_", "MAX_" });
+        QMap<QString, int> pokemonDefines = parser.readCDefinesByName(filename, {"MIN_LEVEL", "MAX_LEVEL"});
         miscConstants.insert("max_level_define", pokemonDefines.value("MAX_LEVEL") > pokemonDefines.value("MIN_LEVEL") ? pokemonDefines.value("MAX_LEVEL") : 100);
         miscConstants.insert("min_level_define", pokemonDefines.value("MIN_LEVEL") < pokemonDefines.value("MAX_LEVEL") ? pokemonDefines.value("MIN_LEVEL") : 1);
     }
 
     QString filename = projectConfig.getFilePath(ProjectFilePath::constants_global);
     fileWatcher.addPath(root + "/" + filename);
-    QStringList definePrefixes("\\bOBJECT_");
+    QMap<QString, int> defines = parser.readCDefinesByName(filename, {"OBJECT_EVENT_TEMPLATES_COUNT"});
-    QMap<QString, int> defines = parser.readCDefines(filename, definePrefixes);
 
     auto it = defines.find("OBJECT_EVENT_TEMPLATES_COUNT");
     if (it != defines.end()) {