Skip to content

Q
qgroundcontrol
Commit 27f2a93b authored by Lorenz Meier's avatar Lorenz Meier
Browse files
Options

Merge pull request #84 from Susurrus/datalogger_refactoring 

Datalogger refactoring
parents 33b2346e 6ba5787c
Hide whitespace changes
Inline Side-by-side
Showing with 195 additions and 344 deletions
src/LogCompressor.cc
View file @ 27f2a93b
/*===================================================================
QGroundControl Open Source Ground Control Station
(c) 2009, 2010 QGROUNDCONTROL PROJECT <http://www.qgroundcontrol.org>
This file is part of the QGROUNDCONTROL project
QGROUNDCONTROL is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
QGROUNDCONTROL is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with QGROUNDCONTROL. If not, see <http://www.gnu.org/licenses/>.
======================================================================*/
/**
* @file
* @brief Implementation of class LogCompressor
* @author Lorenz Meier <mavteam@student.ethz.ch>
*
*/
#include <QFile>
#include <QTextStream>
#include <QStringList>
#include <QFileInfo>
#include <QList>
#include "LogCompressor.h"
#include <QDebug>
/**
* It will only get active upon calling startCompression()
*/
LogCompressor::LogCompressor(QString logFileName, QString outFileName, int uasid) :
logFileName(logFileName),
outFileName(outFileName),
running(true),
currentDataLine(0),
dataLines(1),
uasid(uasid),
holeFillingEnabled(true)
{
}
void LogCompressor::run()
{
QString separator = "\t";
QString fileName = logFileName;
QFile file(fileName);
QFile outfile(outFileName);
QStringList* keys = new QStringList();
QList<quint64> times;// = new QList<quint64>();
QList<quint64> finalTimes;
//qDebug() << "LOG COMPRESSOR: Starting" << fileName;
if (!file.exists() || !file.open(QIODevice::ReadOnly | QIODevice::Text)) {
//qDebug() << "LOG COMPRESSOR: INPUT FILE DOES NOT EXIST";
emit logProcessingStatusChanged(tr("Log Compressor: Cannot start/compress log file, since input file %1 is not readable").arg(QFileInfo(fileName).absoluteFilePath()));
return;
}
// Check if file is writeable
if (outFileName == ""/* || !QFileInfo(outfile).isWritable()*/) {
//qDebug() << "LOG COMPRESSOR: OUTPUT FILE DOES NOT EXIST" << outFileName;
emit logProcessingStatusChanged(tr("Log Compressor: Cannot start/compress log file, since output file %1 is not writable").arg(QFileInfo(outFileName).absoluteFilePath()));
return;
}
// Find all keys
QTextStream in(&file);
// Search only a certain region, assuming that not more
// than N dimensions at H Hertz can be send
const unsigned int keySearchLimit = 15000;
// e.g. 500 Hz * 30 values or
// e.g. 100 Hz * 150 values
unsigned int keyCounter = 0;
while (!in.atEnd() && keyCounter < keySearchLimit) {
QString line = in.readLine();
// Accumulate map of keys
// Data field name is at position 2
QString key = line.split(separator).at(2);
if (!keys->contains(key))
{
keys->append(key);
}
keyCounter++;
}
keys->sort();
QString header = "";
QString spacer = "";
for (int i = 0; i < keys->length(); i++) {
header += keys->at(i) + separator;
spacer += separator;
}
emit logProcessingStatusChanged(tr("Log compressor: Dataset contains dimension: ") + header);
// Find all times
//in.reset();
file.reset();
in.reset();
in.resetStatus();
bool ok;
while (!in.atEnd()) {
QString line = in.readLine();
// Accumulate map of keys
// Data field name is at position 2b
quint64 time = static_cast<QString>(line.split(separator).at(0)).toLongLong(&ok);
if (ok) {
times.append(time);
}
}
qSort(times);
qint64 lastTime = -1;
// Create lines
QStringList* outLines = new QStringList();
for (int i = 0; i < times.length(); i++) {
// Cast to signed on purpose, 64 bit timestamp still long enough
if (static_cast<qint64>(times.at(i)) != lastTime) {
outLines->append(QString("%1").arg(times.at(i)) + separator + spacer);
lastTime = static_cast<qint64>(times.at(i));
finalTimes.append(times.at(i));
//qDebug() << "ADDED:" << outLines->last();
}
}
dataLines = finalTimes.length();
emit logProcessingStatusChanged(tr("Log compressor: Now processing %1 log lines").arg(finalTimes.length()));
// Fill in the values for all keys
file.reset();
QTextStream data(&file);
int linecounter = 0;
quint64 lastTimeIndex = 0;
bool failed = false;
while (!data.atEnd()) {
linecounter++;
currentDataLine = linecounter;
QString line = data.readLine();
QStringList parts = line.split(separator);
// Get time
quint64 time = static_cast<QString>(parts.first()).toLongLong(&ok);
QString field = parts.at(2);
int fieldIndex = keys->indexOf(field);
QString value = parts.at(3);
// // Enforce NaN if no value is present
// if (value.length() == 0 || value == "" || value == " " || value == "\t" || value == "\n") {
// // Hole filling disabled, fill with NaN
// value = "NaN";
// }
// Get matching output line
// Constraining the search area might result in not finding a key,
// but it significantly reduces the time needed for the search
// setting a window of 100 entries means that a 1 Hz data point
// can still be located
quint64 offsetLimit = 100;
quint64 offset;
qint64 index = -1;
failed = false;
// Search the index until it is valid (!= -1)
// or the start of the list has been reached (failed)
while (index == -1 && !failed) {
if (lastTimeIndex > offsetLimit) {
offset = lastTimeIndex - offsetLimit;
} else {
offset = 0;
}
index = finalTimes.indexOf(time, offset);
if (index == -1) {
if (offset == 0) {
emit logProcessingStatusChanged(tr("Log compressor: Timestamp %1 not found in dataset, ignoring log line %2").arg(time).arg(linecounter));
qDebug() << "Completely failed finding value";
//continue;
failed = true;
} else {
emit logProcessingStatusChanged(tr("Log compressor: Timestamp %1 not found in dataset, restarting search.").arg(time));
offsetLimit*=2;
}
}
}
if (dataLines > 100) if (index % (dataLines/100) == 0) emit logProcessingStatusChanged(tr("Log compressor: Processed %1% of %2 lines").arg(index/(float)dataLines*100, 0, 'f', 2).arg(dataLines));
if (!failed) {
// When the algorithm reaches here the correct index was found
lastTimeIndex = index;
QString outLine = outLines->at(index);
QStringList outParts = outLine.split(separator);
// Replace measurement placeholder with current value
outParts.replace(fieldIndex+1, value);
outLine = outParts.join(separator);
outLines->replace(index, outLine);
}
}
///////////////////////////
// HOLE FILLING
// If hole filling is enabled, run again through the whole file and replace holes
if (holeFillingEnabled)
{
// Build up the fill values - initialize to NaN
QStringList fillValues;
int fillCount = keys->count();
for (int i = 0; i< fillCount; ++i)
{
fillValues.append("NaN");
}
// Run through all lines and replace with fill values
for (int index = 0; index < outLines->count(); ++index)
{
QString line = outLines->at(index);
//qDebug() << "LINE" << line;
QStringList fields = line.split(separator, QString::SkipEmptyParts);
// The fields line contains the timestamp
// index of the data fields therefore runs from 1 to n-1
int fieldCount = fields.count();
for (int i = 1; i < fillCount+1; ++i)
{
if (fieldCount <= i) fields.append("");
// Allow input data to be screwed up
if (fields.at(i) == "\t" || fields.at(i) == " " || fields.at(i) == "\n")
{
// Remove invalid data
if (fieldCount > fillCount+1)
{
// This field has a seperator value and is too much
//qDebug() << "REMOVED INVALID INPUT DATA";
fields.removeAt(i);
}
// Continue on invalid data
continue;
}
// Check if this is NaN
if (fields.at(i) == 0 || fields.at(i) == "")
{
// Value was empty, replace it
fields.replace(i, fillValues[i-1]);
//qDebug() << "FILL" << fillValues.at(i-1);
}
else
{
// Value was not NaN, use it as
// new fill value
fillValues.replace(i-1, fields[i]);
}
}
outLines->replace(index, fields.join(separator));
}
}
// Add header, write out file
file.close();
if (outFileName == logFileName) {
QFile::remove(file.fileName());
outfile.setFileName(file.fileName());
}
if (!outfile.open(QIODevice::WriteOnly | QIODevice::Text))
return;
outfile.write(QString(QString("timestamp_ms") + separator + header.replace(" ", "_") + QString("\n")).toLatin1());
emit logProcessingStatusChanged(tr("Log Compressor: Writing output to file %1").arg(QFileInfo(outFileName).absoluteFilePath()));
// File output
for (int i = 0; i < outLines->length(); i++) {
//qDebug() << outLines->at(i);
outfile.write(QString(outLines->at(i) + "\n").toLatin1());
}
currentDataLine = 0;
dataLines = 1;
delete keys;
emit logProcessingStatusChanged(tr("Log compressor: Finished processing file: %1").arg(outfile.fileName()));
qDebug() << "Done with logfile processing";
emit finishedFile(outfile.fileName());
running = false;
}
/**
* @param holeFilling If hole filling is enabled, the compressor tries to fill empty data fields with previous
* values from the same variable (or NaN, if no previous value existed)
*/
void LogCompressor::startCompression(bool holeFilling)
{
// Set hole filling
holeFillingEnabled = holeFilling;
start();
}
bool LogCompressor::isFinished()
{
return !running;
}
int LogCompressor::getCurrentLine()
{
return currentDataLine;
}
int LogCompressor::getDataLines()
{
return dataLines;
}
/*===================================================================
QGroundControl Open Source Ground Control Station
(c) 2009, 2010 QGROUNDCONTROL PROJECT <http://www.qgroundcontrol.org>
This file is part of the QGROUNDCONTROL project
QGROUNDCONTROL is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
QGROUNDCONTROL is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with QGROUNDCONTROL. If not, see <http://www.gnu.org/licenses/>.
======================================================================*/
/**
* @file
* @brief Implementation of class LogCompressor. This class reads in a file containing messages and translates it into a tab-delimited CSV file.
* @author Lorenz Meier <mavteam@student.ethz.ch>
*
*/
#include <QFile>
#include <QTemporaryFile>
#include <QTextStream>
#include <QStringList>
#include <QFileInfo>
#include <QList>
#include "LogCompressor.h"
#include <QDebug>
/**
* Initializes all the variables necessary for a compression run. This won't actually happen
* until startCompression(...) is called.
*/
LogCompressor::LogCompressor(QString logFileName, QString outFileName, QString delimiter) :
logFileName(logFileName),
outFileName(outFileName),
running(true),
currentDataLine(0),
holeFillingEnabled(true),
delimiter(delimiter)
{
}
void LogCompressor::run()
{
// Verify that the input file is useable
QFile infile(logFileName);
if (!infile.exists() || !infile.open(QIODevice::ReadOnly | QIODevice::Text)) {
emit logProcessingStatusChanged(tr("Log Compressor: Cannot start/compress log file, since input file %1 is not readable").arg(QFileInfo(infile.fileName()).absoluteFilePath()));
return;
}
// Verify that the output file is useable
QTemporaryFile outTmpFile;
if (!outTmpFile.open()) {
emit logProcessingStatusChanged(tr("Log Compressor: Cannot start/compress log file, since output file %1 is not writable").arg(QFileInfo(outTmpFile.fileName()).absoluteFilePath()));
return;
}
// First we search the input file through keySearchLimit number of lines
// looking for variables. This is neccessary before CSV files require
// the same number of fields for every line.
const unsigned int keySearchLimit = 15000;
unsigned int keyCounter = 0;
QTextStream in(&infile);
QMap<QString, int> messageMap;
while (!in.atEnd() && keyCounter < keySearchLimit) {
QString messageName = in.readLine().split(delimiter).at(2);
messageMap.insert(messageName, 0);
++keyCounter;
}
// Now update each key with its index in the output string. These are
// all offset by one to account for the first field: timestamp_ms.
QMap<QString, int>::iterator i = messageMap.constBegin();
int j;
for (i = messageMap.begin(), j = 1; i != messageMap.end(); ++i, ++j) {
i.value() = j;
}
// Open the output file and write the header line to it
QStringList headerList(messageMap.keys());
QString headerLine = "timestamp_ms" + delimiter + headerList.join(delimiter) + "\n";
outTmpFile.write(headerLine.toLocal8Bit());
emit logProcessingStatusChanged(tr("Log compressor: Dataset contains dimension: ") + headerLine);
// Reset our position in the input file before we start the main processing loop.
infile.reset();
in.reset();
in.resetStatus();
// Template list stores a list for populating with data as it's parsed from messages.
QStringList templateList;
for (int i = 0; i < headerList.size() + 1; ++i) {
templateList << (holeFillingEnabled?"NaN":"");
}
QStringList filledList(templateList);
QStringList currentLine = in.readLine().split(delimiter);
currentDataLine = 1;
while (!in.atEnd()) {
// We only overwrite data from the last time set if we aren't doing a zero-order hold
if (!holeFillingEnabled) {
filledList = templateList;
}
// Populate this time set with the data from this first message
filledList.replace(0, currentLine.at(0));
filledList.replace(messageMap.value(currentLine.at(2)), currentLine.at(3));
// Continue searching for messages in the same time set and adding that data
// to the current time set if appropriate.
while (!in.atEnd()) {
QStringList newLine = in.readLine().split(delimiter);
++currentDataLine;
if (newLine.at(0) == currentLine.at(0)) {
QString currentDataName = newLine.at(2);
QString currentDataValue = newLine.at(3);
filledList.replace(messageMap.value(currentDataName), currentDataValue);
} else {
currentLine = newLine;
break;
}
}
// Write this current time set out to the file
QString output = filledList.join(delimiter) + "\n";
outTmpFile.write(output.toLocal8Bit());
}
// We're now done with the source file
infile.close();
// Make sure we remove the source file before replacing it.
QFile::remove(outFileName);
outTmpFile.copy(outFileName);
outTmpFile.close();
emit logProcessingStatusChanged(tr("Log Compressor: Writing output to file %1").arg(QFileInfo(outFileName).absoluteFilePath()));
// Clean up and update the status before we return.
currentDataLine = 0;
emit logProcessingStatusChanged(tr("Log compressor: Finished processing file: %1").arg(outFileName));
emit finishedFile(outFileName);
qDebug() << "Done with logfile processing";
running = false;
}
/**
* @param holeFilling If hole filling is enabled, the compressor tries to fill empty data fields with previous
* values from the same variable (or NaN, if no previous value existed)
*/
void LogCompressor::startCompression(bool holeFilling)
{
holeFillingEnabled = holeFilling;
start();
}
bool LogCompressor::isFinished()
{
return !running;
}
int LogCompressor::getCurrentLine()
{
return currentDataLine;
}
\ No newline at end of file
src/LogCompressor.h
View file @ 27f2a93b
......@@ -8,29 +8,31 @@ class LogCompressor : public QThread
Q_OBJECT
public:
/** @brief Create the log compressor. It will only get active upon calling startCompression() */
LogCompressor(QString logFileName, QString outFileName="", int uasid = 0);
LogCompressor(QString logFileName, QString outFileName="", QString delimiter="\t");
/** @brief Start the compression of a raw, line-based logfile into a CSV file */
void startCompression(bool holeFilling=false);
bool isFinished();
int getDataLines();
int getCurrentLine();
protected:
void run();
QString logFileName;
QString outFileName;
bool running;
int currentDataLine;
int dataLines;
int uasid;
bool holeFillingEnabled; ///< Enables the filling of holes in the dataset with the previous value (or NaN if none exists)
void run(); ///< This function actually performs the compression. It's an overloaded function from QThread
QString logFileName; ///< The input file name.
QString outFileName; ///< The output file name. If blank defaults to logFileName
bool running; ///< True when the startCompression() function is operating.
int currentDataLine; ///< The current line of data that is being processed. Only relevant when running==true
QString delimiter; ///< Delimiter between fields in the output file. Defaults to tab ('\t')
bool holeFillingEnabled; ///< Enables the filling of holes in the dataset with the previous value (or NaN if none exists)
signals:
/** @brief This signal is emitted when there is a change in the status of the parsing algorithm. For instance if an error is encountered.
* @param status A status message
*/
void logProcessingStatusChanged(QString status);
/** @brief This signal is emitted once a logfile has been finished writing
* @param fileName The name out the output (CSV) file
* @param fileName The name of the output (CSV) file
*/
void logProcessingStatusChanged(QString);
void finishedFile(QString fileName);
};
#endif // LOGCOMPRESSOR_H
#endif // LOGCOMPRESSOR_H
\ No newline at end of file
src/ui/Linechart.ui
View file @ 27f2a93b
......@@ -145,7 +145,7 @@
<string>Display only variable names in curve list</string>
</property>
<property name="text">
<string>Short Names</string>
<string>Short names</string>
</property>
</widget>
</item>
......
src/ui/MAVLinkDecoder.cc
View file @ 27f2a93b
......@@ -12,10 +12,10 @@ MAVLinkDecoder::MAVLinkDecoder(MAVLinkProtocol* protocol, QObject *parent) :
componentID[i] = -1;
componentMulti[i] = false;
onboardTimeOffset[i] = 0;
onboardToGCSUnixTimeOffsetAndDelay[i] = 0;
firstOnboardTime[i] = 0;
}
// Fill filter
messageFilter.insert(MAVLINK_MSG_ID_HEARTBEAT, false);
messageFilter.insert(MAVLINK_MSG_ID_SYS_STATUS, false);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment