Skip to content

Q
qgroundcontrol
Commit 0e044f6c authored by lm's avatar lm
Browse files
Options

Fixed logging of large datasets

parent 835279e9
Hide whitespace changes
Inline Side-by-side
Showing with 108 additions and 60 deletions
src/LogCompressor.cc
View file @ 0e044f6c
/*=====================================================================
/*===================================================================
QGroundControl Open Source Ground Control Station
(c) 2009, 2010 QGROUNDCONTROL PROJECT <http://www.qgroundcontrol.org>
......@@ -10,15 +9,15 @@ This file is part of the QGROUNDCONTROL project
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
QGROUNDCONTROL is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with QGROUNDCONTROL. If not, see <http://www.gnu.org/licenses/>.
======================================================================*/
/**
......@@ -32,6 +31,7 @@ This file is part of the QGROUNDCONTROL project
#include <QTextStream>
#include <QStringList>
#include <QFileInfo>
#include <QList>
#include "LogCompressor.h"
#include <QDebug>
......@@ -56,12 +56,13 @@ void LogCompressor::run()
QFile file(fileName);
QFile outfile(outFileName);
QStringList* keys = new QStringList();
QStringList* times = new QStringList();
QList<quint64> times;// = new QList<quint64>();
QList<quint64> finalTimes;
if (!file.exists()) return;
if (!file.open(QIODevice::ReadOnly | QIODevice::Text))
return;
if (outFileName != "")
{
// Check if file is writeable
......@@ -70,18 +71,27 @@ void LogCompressor::run()
return;
}
}
// Find all keys
QTextStream in(&file);
while (!in.atEnd()) {
// Search only a certain region, assuming that not more
// than N dimensions at H Hertz can be send
const unsigned int keySearchLimit = 15000;
// e.g. 500 Hz * 30 values or
// e.g. 100 Hz * 150 values
unsigned int keyCounter = 0;
while (!in.atEnd() && keyCounter < keySearchLimit) {
QString line = in.readLine();
// Accumulate map of keys
// Data field name is at position 2
QString key = line.split(separator).at(2);
if (!keys->contains(key)) keys->append(key);
keyCounter++;
}
keys->sort();
QString header = "";
QString spacer = "";
for (int i = 0; i < keys->length(); i++)
......@@ -90,42 +100,58 @@ void LogCompressor::run()
spacer += " " + separator;
}
emit logProcessingStatusChanged(tr("Log compressor: Dataset contains dimension: ") + header);
//qDebug() << header;
//qDebug() << "NOW READING TIMES";
// Find all times
//in.reset();
file.reset();
in.reset();
in.resetStatus();
while (!in.atEnd()) {
bool ok;
while (!in.atEnd())
{
QString line = in.readLine();
// Accumulate map of keys
// Data field name is at position 2
QString time = line.split(separator).at(0);
if (!times->contains(time))
// Data field name is at position 2b
quint64 time = static_cast<QString>(line.split(separator).at(0)).toLongLong(&ok);
if (ok)
{
times->append(time);
times.append(time);
}
}
dataLines = times->length();
times->sort();
qSort(times);
qint64 lastTime = -1;
// Create lines
QStringList* outLines = new QStringList();
for (int i = 0; i < times->length(); i++)
for (int i = 0; i < times.length(); i++)
{
outLines->append(times->at(i) + separator + spacer);
if (times.at(i) != lastTime)
{
outLines->append(QString("%1").arg(times.at(i)) + separator + spacer);
lastTime = static_cast<qint64>(times.at(i));
finalTimes.append(times.at(i));
//qDebug() << "ADDED:" << outLines->last();
}
}
dataLines = finalTimes.length();
emit logProcessingStatusChanged(tr("Log compressor: Now processing %1 log lines").arg(finalTimes.length()));
// Fill in the values for all keys
file.reset();
QTextStream data(&file);
int linecounter = 0;
quint64 lastTimeIndex = 0;
bool failed = false;
while (!data.atEnd())
{
linecounter++;
......@@ -133,7 +159,7 @@ void LogCompressor::run()
QString line = data.readLine();
QStringList parts = line.split(separator);
// Get time
QString time = parts.first();
quint64 time = static_cast<QString>(parts.first()).toLongLong(&ok);
QString field = parts.at(2);
QString value = parts.at(3);
// Enforce NaN if no value is present
......@@ -142,15 +168,19 @@ void LogCompressor::run()
value = "NaN";
}
// Get matching output line
// Constraining the search area might result in not finding a key,
// but it significantly reduces the time needed for the search
// setting a window of 1000 entries means that a 1 Hz data point
// can still be located
int offsetLimit = 200;
quint64 offsetLimit = 100;
quint64 offset;
quint64 index = -1;
while (index == -1)
qint64 index = -1;
failed = false;
// Search the index until it is valid (!= -1)
// or the start of the list has been reached (failed)
while (index == -1 && !failed)
{
if (lastTimeIndex > offsetLimit)
{
......@@ -160,14 +190,27 @@ void LogCompressor::run()
{
offset = 0;
}
quint64 index = times->indexOf(time, offset);
index = finalTimes.indexOf(time, offset);
if (index == -1)
{
qDebug() << "INDEX NOT FOUND DURING LOGFILE PROCESSING, RESTARTING SEARCH";
// FIXME Reset and start without offset heuristic again
offsetLimit+=1000;
if (offset == 0)
{
emit logProcessingStatusChanged(tr("Log compressor: Timestamp %1 not found in dataset, ignoring log line %2").arg(time).arg(linecounter));
//continue;
failed = true;
}
else
{
emit logProcessingStatusChanged(tr("Log compressor: Timestamp %1 not found in dataset, restarting search.").arg(time));
offsetLimit*=2;
}
}
}
if (index % (dataLines/10) == 0) emit logProcessingStatusChanged(tr("Log compressor: Processed %1%% of %2 lines").arg(index/(float)dataLines).arg(dataLines));
// When the algorithm reaches here the correct index was found
lastTimeIndex = index;
QString outLine = outLines->at(index);
QStringList outParts = outLine.split(separator);
......@@ -176,12 +219,11 @@ void LogCompressor::run()
outLine = outParts.join(separator);
outLines->replace(index, outLine);
}
// Add header, write out file
file.close();
if (outFileName == "")
{
QFile::remove(file.fileName());
......@@ -191,19 +233,20 @@ void LogCompressor::run()
return;
outfile.write(QString(QString("unix_timestamp") + separator + header.replace(" ", "_") + QString("\n")).toLatin1());
//QString fileHeader = QString("unix_timestamp") + header.replace(" ", "_") + QString("\n");
// File output
for (int i = 0; i < outLines->length(); i++)
{
//qDebug() << outLines->at(i);
outfile.write(QString(outLines->at(i) + "\n").toLatin1());
}
currentDataLine = 0;
dataLines = 1;
delete keys;
qDebug() << "Done with logfile processing";
emit logProcessingStatusChanged(tr("Log compressor: Finished processing file: %1").arg(outfile.fileName()));
//qDebug() << "Done with logfile processing";
emit finishedFile(outfile.fileName());
running = false;
}
......
src/LogCompressor.h
View file @ 0e044f6c
......@@ -27,6 +27,7 @@ signals:
/** @brief This signal is emitted once a logfile has been finished writing
* @param fileName The name out the output (CSV) file
*/
void logProcessingStatusChanged(QString);
void finishedFile(QString fileName);
};
......
src/ui/MainWindow.cc
View file @ 0e044f6c
......@@ -120,6 +120,8 @@ MainWindow::MainWindow(QWidget *parent):
// Load mavlink view as default widget set
//loadMAVLinkView();
statusBar()->setSizeGripEnabled(true);
if (settings.contains("geometry"))
{
// Restore the window geometry
......@@ -144,8 +146,7 @@ MainWindow::MainWindow(QWidget *parent):
MainWindow::~MainWindow()
{
delete statusBar;
statusBar = NULL;
}
void MainWindow::buildCommonWidgets()
......@@ -811,15 +812,6 @@ void MainWindow::configureWindowName()
#endif
}
QStatusBar* MainWindow::createStatusBar()
{
QStatusBar* bar = new QStatusBar();
/* Add status fields and messages */
/* Enable resize grip in the bottom right corner */
bar->setSizeGripEnabled(true);
return bar;
}
void MainWindow::startVideoCapture()
{
QString format = "bmp";
......@@ -890,7 +882,7 @@ void MainWindow::reloadStylesheet()
*/
void MainWindow::showStatusMessage(const QString& status, int timeout)
{
statusBar->showMessage(status, timeout);
statusBar()->showMessage(status, timeout);
}
/**
......@@ -901,7 +893,7 @@ void MainWindow::showStatusMessage(const QString& status, int timeout)
*/
void MainWindow::showStatusMessage(const QString& status)
{
statusBar->showMessage(status, 5);
statusBar()->showMessage(status, 20000);
}
void MainWindow::showCriticalMessage(const QString& title, const QString& message)
......@@ -915,6 +907,17 @@ void MainWindow::showCriticalMessage(const QString& title, const QString& messag
msgBox.exec();
}
void MainWindow::showInfoMessage(const QString& title, const QString& message)
{
QMessageBox msgBox(MainWindow::instance());
msgBox.setIcon(QMessageBox::Information);
msgBox.setText(title);
msgBox.setInformativeText(message);
msgBox.setStandardButtons(QMessageBox::Ok);
msgBox.setDefaultButton(QMessageBox::Ok);
msgBox.exec();
}
/**
* @brief Create all actions associated to the main window
*
......
src/ui/MainWindow.h
View file @ 0e044f6c
......@@ -98,6 +98,8 @@ public slots:
void showStatusMessage(const QString& status);
/** @brief Shows a critical message as popup or as widget */
void showCriticalMessage(const QString& title, const QString& message);
/** @brief Shows an info message as popup or as widget */
void showInfoMessage(const QString& title, const QString& message);
void addLink();
void addLink(LinkInterface* link);
......@@ -296,10 +298,6 @@ protected:
VIEW_SECTIONS currentView;
bool aboutToCloseFlag;
QStatusBar* statusBar;
QStatusBar* createStatusBar();
void clearView();
void buildCommonWidgets();
......
src/ui/linechart/LinechartWidget.cc
View file @ 0e044f6c
......@@ -48,6 +48,7 @@ This file is part of the PIXHAWK project
#include "LinechartWidget.h"
#include "LinechartPlot.h"
#include "LogCompressor.h"
#include "MainWindow.h"
#include "QGC.h"
#include "MG.h"
......@@ -348,7 +349,7 @@ void LinechartWidget::startLogging()
// Let user select the log file name
QDate date(QDate::currentDate());
// QString("./pixhawk-log-" + date.toString("yyyy-MM-dd") + "-" + QString::number(logindex) + ".log")
QString fileName = QFileDialog::getSaveFileName(this, tr("Specify log file name"), QDesktopServices::storageLocation(QDesktopServices::DesktopLocation), tr("Logfile (*.csv, *.txt);;"));
QString fileName = QFileDialog::getSaveFileName(this, tr("Specify log file name"), QDesktopServices::storageLocation(QDesktopServices::DesktopLocation), tr("Logfile (*.csv *.txt);;"));
if (!fileName.contains("."))
{
......@@ -399,6 +400,8 @@ void LinechartWidget::stopLogging()
// Postprocess log file
compressor = new LogCompressor(logFile->fileName());
connect(compressor, SIGNAL(finishedFile(QString)), this, SIGNAL(logfileWritten(QString)));
connect(compressor, SIGNAL(logProcessingStatusChanged(QString)), MainWindow::instance(), SLOT(showStatusMessage(QString)));
MainWindow::instance()->showInfoMessage("Logging ended", "QGroundControl is now compressing the logfile in a consistent CVS file. This may take a while, you can continue to use QGroundControl. Status updates appear at the bottom of the window.");
compressor->startCompression();
}
logButton->setText(tr("Start logging"));
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment