diff --git a/src/LogCompressor.cc b/src/LogCompressor.cc index 53afa4dfdf30b3babdb4c95da6bc146e0ce00162..e1b01a39c3ffd42cb79bc8317af39e6bb1a9c749 100644 --- a/src/LogCompressor.cc +++ b/src/LogCompressor.cc @@ -1,328 +1,177 @@ -/*=================================================================== -QGroundControl Open Source Ground Control Station - -(c) 2009, 2010 QGROUNDCONTROL PROJECT - -This file is part of the QGROUNDCONTROL project - - QGROUNDCONTROL is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - QGROUNDCONTROL is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with QGROUNDCONTROL. If not, see . - -======================================================================*/ - -/** - * @file - * @brief Implementation of class LogCompressor - * @author Lorenz Meier - * - */ - -#include -#include -#include -#include -#include -#include "LogCompressor.h" - -#include - -/** - * It will only get active upon calling startCompression() - */ -LogCompressor::LogCompressor(QString logFileName, QString outFileName, int uasid) : - logFileName(logFileName), - outFileName(outFileName), - running(true), - currentDataLine(0), - dataLines(1), - uasid(uasid), - holeFillingEnabled(true) -{ -} - -void LogCompressor::run() -{ - QString separator = "\t"; - QString fileName = logFileName; - QFile file(fileName); - QFile outfile(outFileName); - QStringList* keys = new QStringList(); - QList times;// = new QList(); - QList finalTimes; - - //qDebug() << "LOG COMPRESSOR: Starting" << fileName; - - if (!file.exists() || !file.open(QIODevice::ReadOnly | QIODevice::Text)) { - //qDebug() << "LOG COMPRESSOR: INPUT FILE DOES NOT EXIST"; - emit logProcessingStatusChanged(tr("Log Compressor: Cannot start/compress log file, since input file %1 is not readable").arg(QFileInfo(fileName).absoluteFilePath())); - return; - } - - // Check if file is writeable - if (outFileName == ""/* || !QFileInfo(outfile).isWritable()*/) { - //qDebug() << "LOG COMPRESSOR: OUTPUT FILE DOES NOT EXIST" << outFileName; - emit logProcessingStatusChanged(tr("Log Compressor: Cannot start/compress log file, since output file %1 is not writable").arg(QFileInfo(outFileName).absoluteFilePath())); - return; - } - - // Find all keys - QTextStream in(&file); - - // Search only a certain region, assuming that not more - // than N dimensions at H Hertz can be send - const unsigned int keySearchLimit = 15000; - // e.g. 500 Hz * 30 values or - // e.g. 100 Hz * 150 values - - unsigned int keyCounter = 0; - while (!in.atEnd() && keyCounter < keySearchLimit) { - QString line = in.readLine(); - // Accumulate map of keys - // Data field name is at position 2 - QString key = line.split(separator).at(2); - if (!keys->contains(key)) - { - keys->append(key); - } - keyCounter++; - } - keys->sort(); - - QString header = ""; - QString spacer = ""; - for (int i = 0; i < keys->length(); i++) { - header += keys->at(i) + separator; - spacer += separator; - } - - emit logProcessingStatusChanged(tr("Log compressor: Dataset contains dimension: ") + header); - - // Find all times - //in.reset(); - file.reset(); - in.reset(); - in.resetStatus(); - bool ok; - while (!in.atEnd()) { - QString line = in.readLine(); - // Accumulate map of keys - // Data field name is at position 2b - quint64 time = static_cast(line.split(separator).at(0)).toLongLong(&ok); - if (ok) { - times.append(time); - } - } - - qSort(times); - - qint64 lastTime = -1; - - // Create lines - QStringList* outLines = new QStringList(); - for (int i = 0; i < times.length(); i++) { - // Cast to signed on purpose, 64 bit timestamp still long enough - if (static_cast(times.at(i)) != lastTime) { - outLines->append(QString("%1").arg(times.at(i)) + separator + spacer); - lastTime = static_cast(times.at(i)); - finalTimes.append(times.at(i)); - //qDebug() << "ADDED:" << outLines->last(); - } - } - - dataLines = finalTimes.length(); - - emit logProcessingStatusChanged(tr("Log compressor: Now processing %1 log lines").arg(finalTimes.length())); - - // Fill in the values for all keys - file.reset(); - QTextStream data(&file); - int linecounter = 0; - quint64 lastTimeIndex = 0; - bool failed = false; - - while (!data.atEnd()) { - linecounter++; - currentDataLine = linecounter; - QString line = data.readLine(); - QStringList parts = line.split(separator); - // Get time - quint64 time = static_cast(parts.first()).toLongLong(&ok); - QString field = parts.at(2); - int fieldIndex = keys->indexOf(field); - QString value = parts.at(3); -// // Enforce NaN if no value is present -// if (value.length() == 0 || value == "" || value == " " || value == "\t" || value == "\n") { -// // Hole filling disabled, fill with NaN -// value = "NaN"; -// } - // Get matching output line - - // Constraining the search area might result in not finding a key, - // but it significantly reduces the time needed for the search - // setting a window of 100 entries means that a 1 Hz data point - // can still be located - quint64 offsetLimit = 100; - quint64 offset; - qint64 index = -1; - failed = false; - - // Search the index until it is valid (!= -1) - // or the start of the list has been reached (failed) - while (index == -1 && !failed) { - if (lastTimeIndex > offsetLimit) { - offset = lastTimeIndex - offsetLimit; - } else { - offset = 0; - } - - index = finalTimes.indexOf(time, offset); - if (index == -1) { - if (offset == 0) { - emit logProcessingStatusChanged(tr("Log compressor: Timestamp %1 not found in dataset, ignoring log line %2").arg(time).arg(linecounter)); - qDebug() << "Completely failed finding value"; - //continue; - failed = true; - } else { - emit logProcessingStatusChanged(tr("Log compressor: Timestamp %1 not found in dataset, restarting search.").arg(time)); - offsetLimit*=2; - } - } - } - - if (dataLines > 100) if (index % (dataLines/100) == 0) emit logProcessingStatusChanged(tr("Log compressor: Processed %1% of %2 lines").arg(index/(float)dataLines*100, 0, 'f', 2).arg(dataLines)); - - if (!failed) { - // When the algorithm reaches here the correct index was found - lastTimeIndex = index; - QString outLine = outLines->at(index); - QStringList outParts = outLine.split(separator); - // Replace measurement placeholder with current value - outParts.replace(fieldIndex+1, value); - outLine = outParts.join(separator); - outLines->replace(index, outLine); - } - } - - /////////////////////////// - // HOLE FILLING - - // If hole filling is enabled, run again through the whole file and replace holes - if (holeFillingEnabled) - { - // Build up the fill values - initialize to NaN - QStringList fillValues; - int fillCount = keys->count(); - for (int i = 0; i< fillCount; ++i) - { - fillValues.append("NaN"); - } - - // Run through all lines and replace with fill values - for (int index = 0; index < outLines->count(); ++index) - { - QString line = outLines->at(index); - //qDebug() << "LINE" << line; - QStringList fields = line.split(separator, QString::SkipEmptyParts); - // The fields line contains the timestamp - // index of the data fields therefore runs from 1 to n-1 - int fieldCount = fields.count(); - for (int i = 1; i < fillCount+1; ++i) - { - if (fieldCount <= i) fields.append(""); - - // Allow input data to be screwed up - if (fields.at(i) == "\t" || fields.at(i) == " " || fields.at(i) == "\n") - { - // Remove invalid data - if (fieldCount > fillCount+1) - { - // This field has a seperator value and is too much - //qDebug() << "REMOVED INVALID INPUT DATA"; - fields.removeAt(i); - } - // Continue on invalid data - continue; - } - - // Check if this is NaN - if (fields.at(i) == 0 || fields.at(i) == "") - { - // Value was empty, replace it - fields.replace(i, fillValues[i-1]); - //qDebug() << "FILL" << fillValues.at(i-1); - } - else - { - // Value was not NaN, use it as - // new fill value - fillValues.replace(i-1, fields[i]); - } - } - outLines->replace(index, fields.join(separator)); - } - } - - // Add header, write out file - file.close(); - - if (outFileName == logFileName) { - QFile::remove(file.fileName()); - outfile.setFileName(file.fileName()); - - } - if (!outfile.open(QIODevice::WriteOnly | QIODevice::Text)) - return; - outfile.write(QString(QString("timestamp_ms") + separator + header.replace(" ", "_") + QString("\n")).toLatin1()); - emit logProcessingStatusChanged(tr("Log Compressor: Writing output to file %1").arg(QFileInfo(outFileName).absoluteFilePath())); - - // File output - for (int i = 0; i < outLines->length(); i++) { - //qDebug() << outLines->at(i); - outfile.write(QString(outLines->at(i) + "\n").toLatin1()); - - } - - currentDataLine = 0; - dataLines = 1; - delete keys; - emit logProcessingStatusChanged(tr("Log compressor: Finished processing file: %1").arg(outfile.fileName())); - qDebug() << "Done with logfile processing"; - emit finishedFile(outfile.fileName()); - running = false; -} - -/** - * @param holeFilling If hole filling is enabled, the compressor tries to fill empty data fields with previous - * values from the same variable (or NaN, if no previous value existed) - */ -void LogCompressor::startCompression(bool holeFilling) -{ - // Set hole filling - holeFillingEnabled = holeFilling; - start(); -} - -bool LogCompressor::isFinished() -{ - return !running; -} - -int LogCompressor::getCurrentLine() -{ - return currentDataLine; -} - -int LogCompressor::getDataLines() -{ - return dataLines; -} +/*=================================================================== +QGroundControl Open Source Ground Control Station + +(c) 2009, 2010 QGROUNDCONTROL PROJECT + +This file is part of the QGROUNDCONTROL project + + QGROUNDCONTROL is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + QGROUNDCONTROL is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with QGROUNDCONTROL. If not, see . + +======================================================================*/ + +/** + * @file + * @brief Implementation of class LogCompressor. This class reads in a file containing messages and translates it into a tab-delimited CSV file. + * @author Lorenz Meier + * + */ + +#include +#include +#include +#include +#include +#include +#include "LogCompressor.h" + +#include + +/** + * Initializes all the variables necessary for a compression run. This won't actually happen + * until startCompression(...) is called. + */ +LogCompressor::LogCompressor(QString logFileName, QString outFileName, QString delimiter) : + logFileName(logFileName), + outFileName(outFileName), + running(true), + currentDataLine(0), + holeFillingEnabled(true), + delimiter(delimiter) +{ +} + +void LogCompressor::run() +{ + // Verify that the input file is useable + QFile infile(logFileName); + if (!infile.exists() || !infile.open(QIODevice::ReadOnly | QIODevice::Text)) { + emit logProcessingStatusChanged(tr("Log Compressor: Cannot start/compress log file, since input file %1 is not readable").arg(QFileInfo(infile.fileName()).absoluteFilePath())); + return; + } + + // Verify that the output file is useable + QTemporaryFile outTmpFile; + if (!outTmpFile.open()) { + emit logProcessingStatusChanged(tr("Log Compressor: Cannot start/compress log file, since output file %1 is not writable").arg(QFileInfo(outTmpFile.fileName()).absoluteFilePath())); + return; + } + + + // First we search the input file through keySearchLimit number of lines + // looking for variables. This is neccessary before CSV files require + // the same number of fields for every line. + const unsigned int keySearchLimit = 15000; + unsigned int keyCounter = 0; + QTextStream in(&infile); + QMap messageMap; + while (!in.atEnd() && keyCounter < keySearchLimit) { + QString messageName = in.readLine().split(delimiter).at(2); + messageMap.insert(messageName, 0); + ++keyCounter; + } + + // Now update each key with its index in the output string. These are + // all offset by one to account for the first field: timestamp_ms. + QMap::iterator i = messageMap.constBegin(); + int j; + for (i = messageMap.begin(), j = 1; i != messageMap.end(); ++i, ++j) { + i.value() = j; + } + + // Open the output file and write the header line to it + QStringList headerList(messageMap.keys()); + QString headerLine = "timestamp_ms" + delimiter + headerList.join(delimiter) + "\n"; + outTmpFile.write(headerLine.toLocal8Bit()); + + emit logProcessingStatusChanged(tr("Log compressor: Dataset contains dimension: ") + headerLine); + + // Reset our position in the input file before we start the main processing loop. + infile.reset(); + in.reset(); + in.resetStatus(); + + // Template list stores a list for populating with data as it's parsed from messages. + QStringList templateList; + for (int i = 0; i < headerList.size() + 1; ++i) { + templateList << (holeFillingEnabled?"NaN":""); + } + QStringList filledList(templateList); + QStringList currentLine = in.readLine().split(delimiter); + currentDataLine = 1; + while (!in.atEnd()) { + // We only overwrite data from the last time set if we aren't doing a zero-order hold + if (!holeFillingEnabled) { + filledList = templateList; + } + // Populate this time set with the data from this first message + filledList.replace(0, currentLine.at(0)); + filledList.replace(messageMap.value(currentLine.at(2)), currentLine.at(3)); + + // Continue searching for messages in the same time set and adding that data + // to the current time set if appropriate. + while (!in.atEnd()) { + QStringList newLine = in.readLine().split(delimiter); + ++currentDataLine; + + if (newLine.at(0) == currentLine.at(0)) { + QString currentDataName = newLine.at(2); + QString currentDataValue = newLine.at(3); + filledList.replace(messageMap.value(currentDataName), currentDataValue); + } else { + currentLine = newLine; + break; + } + } + + // Write this current time set out to the file + QString output = filledList.join(delimiter) + "\n"; + outTmpFile.write(output.toLocal8Bit()); + } + + // We're now done with the source file + infile.close(); + + // Make sure we remove the source file before replacing it. + QFile::remove(outFileName); + outTmpFile.copy(outFileName); + outTmpFile.close(); + emit logProcessingStatusChanged(tr("Log Compressor: Writing output to file %1").arg(QFileInfo(outFileName).absoluteFilePath())); + + // Clean up and update the status before we return. + currentDataLine = 0; + emit logProcessingStatusChanged(tr("Log compressor: Finished processing file: %1").arg(outFileName)); + emit finishedFile(outFileName); + qDebug() << "Done with logfile processing"; + running = false; +} + +/** + * @param holeFilling If hole filling is enabled, the compressor tries to fill empty data fields with previous + * values from the same variable (or NaN, if no previous value existed) + */ +void LogCompressor::startCompression(bool holeFilling) +{ + holeFillingEnabled = holeFilling; + start(); +} + +bool LogCompressor::isFinished() +{ + return !running; +} + +int LogCompressor::getCurrentLine() +{ + return currentDataLine; +} \ No newline at end of file diff --git a/src/LogCompressor.h b/src/LogCompressor.h index f9a0b855393be81a36acd7cb41b76923c43c5f3d..138c5a59c6698b947d5541a00b3eb36a873af102 100644 --- a/src/LogCompressor.h +++ b/src/LogCompressor.h @@ -8,29 +8,31 @@ class LogCompressor : public QThread Q_OBJECT public: /** @brief Create the log compressor. It will only get active upon calling startCompression() */ - LogCompressor(QString logFileName, QString outFileName="", int uasid = 0); + LogCompressor(QString logFileName, QString outFileName="", QString delimiter="\t"); /** @brief Start the compression of a raw, line-based logfile into a CSV file */ void startCompression(bool holeFilling=false); bool isFinished(); - int getDataLines(); int getCurrentLine(); protected: - void run(); - QString logFileName; - QString outFileName; - bool running; - int currentDataLine; - int dataLines; - int uasid; - bool holeFillingEnabled; ///< Enables the filling of holes in the dataset with the previous value (or NaN if none exists) + void run(); ///< This function actually performs the compression. It's an overloaded function from QThread + QString logFileName; ///< The input file name. + QString outFileName; ///< The output file name. If blank defaults to logFileName + bool running; ///< True when the startCompression() function is operating. + int currentDataLine; ///< The current line of data that is being processed. Only relevant when running==true + QString delimiter; ///< Delimiter between fields in the output file. Defaults to tab ('\t') + bool holeFillingEnabled; ///< Enables the filling of holes in the dataset with the previous value (or NaN if none exists) signals: + /** @brief This signal is emitted when there is a change in the status of the parsing algorithm. For instance if an error is encountered. + * @param status A status message + */ + void logProcessingStatusChanged(QString status); + /** @brief This signal is emitted once a logfile has been finished writing - * @param fileName The name out the output (CSV) file + * @param fileName The name of the output (CSV) file */ - void logProcessingStatusChanged(QString); void finishedFile(QString fileName); }; -#endif // LOGCOMPRESSOR_H +#endif // LOGCOMPRESSOR_H \ No newline at end of file