From 6ba5787cec151411fa725db5d1b6d143bb4f4cac Mon Sep 17 00:00:00 2001 From: Bryant Mairs Date: Tue, 17 Apr 2012 19:13:53 -0700 Subject: [PATCH] Refactored the LogCompressor. It wasn't properly handling certain message logging files that mixed messages (Issue #70). One note is that this logger should work correctly as I've tested it, but could have some edge cases. It is pickier about the file format that it will sparse, but I don't know if the error-checking stuff from the old code would have actually worked. This can also be easily re-added. This code is also much faster than the old stuff. From what I could tell it scanned through the log file at least twice, I think three times. It also copied a lot of data into memory instead of reading, processing, and writing one line at a time, so memory use should be much lower. Some memory leaks from the old code were also refactored out, so lifetime memory use should be down. --- src/LogCompressor.cc | 505 +++++++++++++++---------------------------- src/LogCompressor.h | 28 +-- 2 files changed, 192 insertions(+), 341 deletions(-) diff --git a/src/LogCompressor.cc b/src/LogCompressor.cc index 53afa4dfd..e1b01a39c 100644 --- a/src/LogCompressor.cc +++ b/src/LogCompressor.cc @@ -1,328 +1,177 @@ -/*=================================================================== -QGroundControl Open Source Ground Control Station - -(c) 2009, 2010 QGROUNDCONTROL PROJECT - -This file is part of the QGROUNDCONTROL project - - QGROUNDCONTROL is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - QGROUNDCONTROL is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with QGROUNDCONTROL. If not, see . - -======================================================================*/ - -/** - * @file - * @brief Implementation of class LogCompressor - * @author Lorenz Meier - * - */ - -#include -#include -#include -#include -#include -#include "LogCompressor.h" - -#include - -/** - * It will only get active upon calling startCompression() - */ -LogCompressor::LogCompressor(QString logFileName, QString outFileName, int uasid) : - logFileName(logFileName), - outFileName(outFileName), - running(true), - currentDataLine(0), - dataLines(1), - uasid(uasid), - holeFillingEnabled(true) -{ -} - -void LogCompressor::run() -{ - QString separator = "\t"; - QString fileName = logFileName; - QFile file(fileName); - QFile outfile(outFileName); - QStringList* keys = new QStringList(); - QList times;// = new QList(); - QList finalTimes; - - //qDebug() << "LOG COMPRESSOR: Starting" << fileName; - - if (!file.exists() || !file.open(QIODevice::ReadOnly | QIODevice::Text)) { - //qDebug() << "LOG COMPRESSOR: INPUT FILE DOES NOT EXIST"; - emit logProcessingStatusChanged(tr("Log Compressor: Cannot start/compress log file, since input file %1 is not readable").arg(QFileInfo(fileName).absoluteFilePath())); - return; - } - - // Check if file is writeable - if (outFileName == ""/* || !QFileInfo(outfile).isWritable()*/) { - //qDebug() << "LOG COMPRESSOR: OUTPUT FILE DOES NOT EXIST" << outFileName; - emit logProcessingStatusChanged(tr("Log Compressor: Cannot start/compress log file, since output file %1 is not writable").arg(QFileInfo(outFileName).absoluteFilePath())); - return; - } - - // Find all keys - QTextStream in(&file); - - // Search only a certain region, assuming that not more - // than N dimensions at H Hertz can be send - const unsigned int keySearchLimit = 15000; - // e.g. 500 Hz * 30 values or - // e.g. 100 Hz * 150 values - - unsigned int keyCounter = 0; - while (!in.atEnd() && keyCounter < keySearchLimit) { - QString line = in.readLine(); - // Accumulate map of keys - // Data field name is at position 2 - QString key = line.split(separator).at(2); - if (!keys->contains(key)) - { - keys->append(key); - } - keyCounter++; - } - keys->sort(); - - QString header = ""; - QString spacer = ""; - for (int i = 0; i < keys->length(); i++) { - header += keys->at(i) + separator; - spacer += separator; - } - - emit logProcessingStatusChanged(tr("Log compressor: Dataset contains dimension: ") + header); - - // Find all times - //in.reset(); - file.reset(); - in.reset(); - in.resetStatus(); - bool ok; - while (!in.atEnd()) { - QString line = in.readLine(); - // Accumulate map of keys - // Data field name is at position 2b - quint64 time = static_cast(line.split(separator).at(0)).toLongLong(&ok); - if (ok) { - times.append(time); - } - } - - qSort(times); - - qint64 lastTime = -1; - - // Create lines - QStringList* outLines = new QStringList(); - for (int i = 0; i < times.length(); i++) { - // Cast to signed on purpose, 64 bit timestamp still long enough - if (static_cast(times.at(i)) != lastTime) { - outLines->append(QString("%1").arg(times.at(i)) + separator + spacer); - lastTime = static_cast(times.at(i)); - finalTimes.append(times.at(i)); - //qDebug() << "ADDED:" << outLines->last(); - } - } - - dataLines = finalTimes.length(); - - emit logProcessingStatusChanged(tr("Log compressor: Now processing %1 log lines").arg(finalTimes.length())); - - // Fill in the values for all keys - file.reset(); - QTextStream data(&file); - int linecounter = 0; - quint64 lastTimeIndex = 0; - bool failed = false; - - while (!data.atEnd()) { - linecounter++; - currentDataLine = linecounter; - QString line = data.readLine(); - QStringList parts = line.split(separator); - // Get time - quint64 time = static_cast(parts.first()).toLongLong(&ok); - QString field = parts.at(2); - int fieldIndex = keys->indexOf(field); - QString value = parts.at(3); -// // Enforce NaN if no value is present -// if (value.length() == 0 || value == "" || value == " " || value == "\t" || value == "\n") { -// // Hole filling disabled, fill with NaN -// value = "NaN"; -// } - // Get matching output line - - // Constraining the search area might result in not finding a key, - // but it significantly reduces the time needed for the search - // setting a window of 100 entries means that a 1 Hz data point - // can still be located - quint64 offsetLimit = 100; - quint64 offset; - qint64 index = -1; - failed = false; - - // Search the index until it is valid (!= -1) - // or the start of the list has been reached (failed) - while (index == -1 && !failed) { - if (lastTimeIndex > offsetLimit) { - offset = lastTimeIndex - offsetLimit; - } else { - offset = 0; - } - - index = finalTimes.indexOf(time, offset); - if (index == -1) { - if (offset == 0) { - emit logProcessingStatusChanged(tr("Log compressor: Timestamp %1 not found in dataset, ignoring log line %2").arg(time).arg(linecounter)); - qDebug() << "Completely failed finding value"; - //continue; - failed = true; - } else { - emit logProcessingStatusChanged(tr("Log compressor: Timestamp %1 not found in dataset, restarting search.").arg(time)); - offsetLimit*=2; - } - } - } - - if (dataLines > 100) if (index % (dataLines/100) == 0) emit logProcessingStatusChanged(tr("Log compressor: Processed %1% of %2 lines").arg(index/(float)dataLines*100, 0, 'f', 2).arg(dataLines)); - - if (!failed) { - // When the algorithm reaches here the correct index was found - lastTimeIndex = index; - QString outLine = outLines->at(index); - QStringList outParts = outLine.split(separator); - // Replace measurement placeholder with current value - outParts.replace(fieldIndex+1, value); - outLine = outParts.join(separator); - outLines->replace(index, outLine); - } - } - - /////////////////////////// - // HOLE FILLING - - // If hole filling is enabled, run again through the whole file and replace holes - if (holeFillingEnabled) - { - // Build up the fill values - initialize to NaN - QStringList fillValues; - int fillCount = keys->count(); - for (int i = 0; i< fillCount; ++i) - { - fillValues.append("NaN"); - } - - // Run through all lines and replace with fill values - for (int index = 0; index < outLines->count(); ++index) - { - QString line = outLines->at(index); - //qDebug() << "LINE" << line; - QStringList fields = line.split(separator, QString::SkipEmptyParts); - // The fields line contains the timestamp - // index of the data fields therefore runs from 1 to n-1 - int fieldCount = fields.count(); - for (int i = 1; i < fillCount+1; ++i) - { - if (fieldCount <= i) fields.append(""); - - // Allow input data to be screwed up - if (fields.at(i) == "\t" || fields.at(i) == " " || fields.at(i) == "\n") - { - // Remove invalid data - if (fieldCount > fillCount+1) - { - // This field has a seperator value and is too much - //qDebug() << "REMOVED INVALID INPUT DATA"; - fields.removeAt(i); - } - // Continue on invalid data - continue; - } - - // Check if this is NaN - if (fields.at(i) == 0 || fields.at(i) == "") - { - // Value was empty, replace it - fields.replace(i, fillValues[i-1]); - //qDebug() << "FILL" << fillValues.at(i-1); - } - else - { - // Value was not NaN, use it as - // new fill value - fillValues.replace(i-1, fields[i]); - } - } - outLines->replace(index, fields.join(separator)); - } - } - - // Add header, write out file - file.close(); - - if (outFileName == logFileName) { - QFile::remove(file.fileName()); - outfile.setFileName(file.fileName()); - - } - if (!outfile.open(QIODevice::WriteOnly | QIODevice::Text)) - return; - outfile.write(QString(QString("timestamp_ms") + separator + header.replace(" ", "_") + QString("\n")).toLatin1()); - emit logProcessingStatusChanged(tr("Log Compressor: Writing output to file %1").arg(QFileInfo(outFileName).absoluteFilePath())); - - // File output - for (int i = 0; i < outLines->length(); i++) { - //qDebug() << outLines->at(i); - outfile.write(QString(outLines->at(i) + "\n").toLatin1()); - - } - - currentDataLine = 0; - dataLines = 1; - delete keys; - emit logProcessingStatusChanged(tr("Log compressor: Finished processing file: %1").arg(outfile.fileName())); - qDebug() << "Done with logfile processing"; - emit finishedFile(outfile.fileName()); - running = false; -} - -/** - * @param holeFilling If hole filling is enabled, the compressor tries to fill empty data fields with previous - * values from the same variable (or NaN, if no previous value existed) - */ -void LogCompressor::startCompression(bool holeFilling) -{ - // Set hole filling - holeFillingEnabled = holeFilling; - start(); -} - -bool LogCompressor::isFinished() -{ - return !running; -} - -int LogCompressor::getCurrentLine() -{ - return currentDataLine; -} - -int LogCompressor::getDataLines() -{ - return dataLines; -} +/*=================================================================== +QGroundControl Open Source Ground Control Station + +(c) 2009, 2010 QGROUNDCONTROL PROJECT + +This file is part of the QGROUNDCONTROL project + + QGROUNDCONTROL is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + QGROUNDCONTROL is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with QGROUNDCONTROL. If not, see . + +======================================================================*/ + +/** + * @file + * @brief Implementation of class LogCompressor. This class reads in a file containing messages and translates it into a tab-delimited CSV file. + * @author Lorenz Meier + * + */ + +#include +#include +#include +#include +#include +#include +#include "LogCompressor.h" + +#include + +/** + * Initializes all the variables necessary for a compression run. This won't actually happen + * until startCompression(...) is called. + */ +LogCompressor::LogCompressor(QString logFileName, QString outFileName, QString delimiter) : + logFileName(logFileName), + outFileName(outFileName), + running(true), + currentDataLine(0), + holeFillingEnabled(true), + delimiter(delimiter) +{ +} + +void LogCompressor::run() +{ + // Verify that the input file is useable + QFile infile(logFileName); + if (!infile.exists() || !infile.open(QIODevice::ReadOnly | QIODevice::Text)) { + emit logProcessingStatusChanged(tr("Log Compressor: Cannot start/compress log file, since input file %1 is not readable").arg(QFileInfo(infile.fileName()).absoluteFilePath())); + return; + } + + // Verify that the output file is useable + QTemporaryFile outTmpFile; + if (!outTmpFile.open()) { + emit logProcessingStatusChanged(tr("Log Compressor: Cannot start/compress log file, since output file %1 is not writable").arg(QFileInfo(outTmpFile.fileName()).absoluteFilePath())); + return; + } + + + // First we search the input file through keySearchLimit number of lines + // looking for variables. This is neccessary before CSV files require + // the same number of fields for every line. + const unsigned int keySearchLimit = 15000; + unsigned int keyCounter = 0; + QTextStream in(&infile); + QMap messageMap; + while (!in.atEnd() && keyCounter < keySearchLimit) { + QString messageName = in.readLine().split(delimiter).at(2); + messageMap.insert(messageName, 0); + ++keyCounter; + } + + // Now update each key with its index in the output string. These are + // all offset by one to account for the first field: timestamp_ms. + QMap::iterator i = messageMap.constBegin(); + int j; + for (i = messageMap.begin(), j = 1; i != messageMap.end(); ++i, ++j) { + i.value() = j; + } + + // Open the output file and write the header line to it + QStringList headerList(messageMap.keys()); + QString headerLine = "timestamp_ms" + delimiter + headerList.join(delimiter) + "\n"; + outTmpFile.write(headerLine.toLocal8Bit()); + + emit logProcessingStatusChanged(tr("Log compressor: Dataset contains dimension: ") + headerLine); + + // Reset our position in the input file before we start the main processing loop. + infile.reset(); + in.reset(); + in.resetStatus(); + + // Template list stores a list for populating with data as it's parsed from messages. + QStringList templateList; + for (int i = 0; i < headerList.size() + 1; ++i) { + templateList << (holeFillingEnabled?"NaN":""); + } + QStringList filledList(templateList); + QStringList currentLine = in.readLine().split(delimiter); + currentDataLine = 1; + while (!in.atEnd()) { + // We only overwrite data from the last time set if we aren't doing a zero-order hold + if (!holeFillingEnabled) { + filledList = templateList; + } + // Populate this time set with the data from this first message + filledList.replace(0, currentLine.at(0)); + filledList.replace(messageMap.value(currentLine.at(2)), currentLine.at(3)); + + // Continue searching for messages in the same time set and adding that data + // to the current time set if appropriate. + while (!in.atEnd()) { + QStringList newLine = in.readLine().split(delimiter); + ++currentDataLine; + + if (newLine.at(0) == currentLine.at(0)) { + QString currentDataName = newLine.at(2); + QString currentDataValue = newLine.at(3); + filledList.replace(messageMap.value(currentDataName), currentDataValue); + } else { + currentLine = newLine; + break; + } + } + + // Write this current time set out to the file + QString output = filledList.join(delimiter) + "\n"; + outTmpFile.write(output.toLocal8Bit()); + } + + // We're now done with the source file + infile.close(); + + // Make sure we remove the source file before replacing it. + QFile::remove(outFileName); + outTmpFile.copy(outFileName); + outTmpFile.close(); + emit logProcessingStatusChanged(tr("Log Compressor: Writing output to file %1").arg(QFileInfo(outFileName).absoluteFilePath())); + + // Clean up and update the status before we return. + currentDataLine = 0; + emit logProcessingStatusChanged(tr("Log compressor: Finished processing file: %1").arg(outFileName)); + emit finishedFile(outFileName); + qDebug() << "Done with logfile processing"; + running = false; +} + +/** + * @param holeFilling If hole filling is enabled, the compressor tries to fill empty data fields with previous + * values from the same variable (or NaN, if no previous value existed) + */ +void LogCompressor::startCompression(bool holeFilling) +{ + holeFillingEnabled = holeFilling; + start(); +} + +bool LogCompressor::isFinished() +{ + return !running; +} + +int LogCompressor::getCurrentLine() +{ + return currentDataLine; +} \ No newline at end of file diff --git a/src/LogCompressor.h b/src/LogCompressor.h index f9a0b8553..138c5a59c 100644 --- a/src/LogCompressor.h +++ b/src/LogCompressor.h @@ -8,29 +8,31 @@ class LogCompressor : public QThread Q_OBJECT public: /** @brief Create the log compressor. It will only get active upon calling startCompression() */ - LogCompressor(QString logFileName, QString outFileName="", int uasid = 0); + LogCompressor(QString logFileName, QString outFileName="", QString delimiter="\t"); /** @brief Start the compression of a raw, line-based logfile into a CSV file */ void startCompression(bool holeFilling=false); bool isFinished(); - int getDataLines(); int getCurrentLine(); protected: - void run(); - QString logFileName; - QString outFileName; - bool running; - int currentDataLine; - int dataLines; - int uasid; - bool holeFillingEnabled; ///< Enables the filling of holes in the dataset with the previous value (or NaN if none exists) + void run(); ///< This function actually performs the compression. It's an overloaded function from QThread + QString logFileName; ///< The input file name. + QString outFileName; ///< The output file name. If blank defaults to logFileName + bool running; ///< True when the startCompression() function is operating. + int currentDataLine; ///< The current line of data that is being processed. Only relevant when running==true + QString delimiter; ///< Delimiter between fields in the output file. Defaults to tab ('\t') + bool holeFillingEnabled; ///< Enables the filling of holes in the dataset with the previous value (or NaN if none exists) signals: + /** @brief This signal is emitted when there is a change in the status of the parsing algorithm. For instance if an error is encountered. + * @param status A status message + */ + void logProcessingStatusChanged(QString status); + /** @brief This signal is emitted once a logfile has been finished writing - * @param fileName The name out the output (CSV) file + * @param fileName The name of the output (CSV) file */ - void logProcessingStatusChanged(QString); void finishedFile(QString fileName); }; -#endif // LOGCOMPRESSOR_H +#endif // LOGCOMPRESSOR_H \ No newline at end of file -- 2.22.0