From 0e044f6c63c9981de04a95684a91d1e10d4c9f35 Mon Sep 17 00:00:00 2001 From: lm Date: Mon, 3 Jan 2011 13:45:45 +0100 Subject: [PATCH] Fixed logging of large datasets --- src/LogCompressor.cc | 127 +++++++++++++++++++--------- src/LogCompressor.h | 1 + src/ui/MainWindow.cc | 29 ++++--- src/ui/MainWindow.h | 6 +- src/ui/linechart/LinechartWidget.cc | 5 +- 5 files changed, 108 insertions(+), 60 deletions(-) diff --git a/src/LogCompressor.cc b/src/LogCompressor.cc index fd394d985..6ba3136ec 100644 --- a/src/LogCompressor.cc +++ b/src/LogCompressor.cc @@ -1,5 +1,4 @@ -/*===================================================================== - +/*=================================================================== QGroundControl Open Source Ground Control Station (c) 2009, 2010 QGROUNDCONTROL PROJECT @@ -10,15 +9,15 @@ This file is part of the QGROUNDCONTROL project it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. - + QGROUNDCONTROL is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with QGROUNDCONTROL. If not, see . - + ======================================================================*/ /** @@ -32,6 +31,7 @@ This file is part of the QGROUNDCONTROL project #include #include #include +#include #include "LogCompressor.h" #include @@ -56,12 +56,13 @@ void LogCompressor::run() QFile file(fileName); QFile outfile(outFileName); QStringList* keys = new QStringList(); - QStringList* times = new QStringList(); - + QList times;// = new QList(); + QList finalTimes; + if (!file.exists()) return; if (!file.open(QIODevice::ReadOnly | QIODevice::Text)) return; - + if (outFileName != "") { // Check if file is writeable @@ -70,18 +71,27 @@ void LogCompressor::run() return; } } - + // Find all keys QTextStream in(&file); - while (!in.atEnd()) { + + // Search only a certain region, assuming that not more + // than N dimensions at H Hertz can be send + const unsigned int keySearchLimit = 15000; + // e.g. 500 Hz * 30 values or + // e.g. 100 Hz * 150 values + + unsigned int keyCounter = 0; + while (!in.atEnd() && keyCounter < keySearchLimit) { QString line = in.readLine(); // Accumulate map of keys // Data field name is at position 2 QString key = line.split(separator).at(2); if (!keys->contains(key)) keys->append(key); + keyCounter++; } keys->sort(); - + QString header = ""; QString spacer = ""; for (int i = 0; i < keys->length(); i++) @@ -90,42 +100,58 @@ void LogCompressor::run() spacer += " " + separator; } + emit logProcessingStatusChanged(tr("Log compressor: Dataset contains dimension: ") + header); + //qDebug() << header; - + //qDebug() << "NOW READING TIMES"; - + // Find all times //in.reset(); file.reset(); in.reset(); in.resetStatus(); - while (!in.atEnd()) { + bool ok; + while (!in.atEnd()) + { QString line = in.readLine(); // Accumulate map of keys - // Data field name is at position 2 - QString time = line.split(separator).at(0); - if (!times->contains(time)) + // Data field name is at position 2b + quint64 time = static_cast(line.split(separator).at(0)).toLongLong(&ok); + if (ok) { - times->append(time); + times.append(time); } } - - dataLines = times->length(); - - times->sort(); - + + qSort(times); + + qint64 lastTime = -1; + // Create lines QStringList* outLines = new QStringList(); - for (int i = 0; i < times->length(); i++) + for (int i = 0; i < times.length(); i++) { - outLines->append(times->at(i) + separator + spacer); + if (times.at(i) != lastTime) + { + outLines->append(QString("%1").arg(times.at(i)) + separator + spacer); + lastTime = static_cast(times.at(i)); + finalTimes.append(times.at(i)); + //qDebug() << "ADDED:" << outLines->last(); + } } + dataLines = finalTimes.length(); + + emit logProcessingStatusChanged(tr("Log compressor: Now processing %1 log lines").arg(finalTimes.length())); + // Fill in the values for all keys file.reset(); QTextStream data(&file); int linecounter = 0; quint64 lastTimeIndex = 0; + bool failed = false; + while (!data.atEnd()) { linecounter++; @@ -133,7 +159,7 @@ void LogCompressor::run() QString line = data.readLine(); QStringList parts = line.split(separator); // Get time - QString time = parts.first(); + quint64 time = static_cast(parts.first()).toLongLong(&ok); QString field = parts.at(2); QString value = parts.at(3); // Enforce NaN if no value is present @@ -142,15 +168,19 @@ void LogCompressor::run() value = "NaN"; } // Get matching output line - + // Constraining the search area might result in not finding a key, // but it significantly reduces the time needed for the search // setting a window of 1000 entries means that a 1 Hz data point // can still be located - int offsetLimit = 200; + quint64 offsetLimit = 100; quint64 offset; - quint64 index = -1; - while (index == -1) + qint64 index = -1; + failed = false; + + // Search the index until it is valid (!= -1) + // or the start of the list has been reached (failed) + while (index == -1 && !failed) { if (lastTimeIndex > offsetLimit) { @@ -160,14 +190,27 @@ void LogCompressor::run() { offset = 0; } - quint64 index = times->indexOf(time, offset); + + index = finalTimes.indexOf(time, offset); if (index == -1) { - qDebug() << "INDEX NOT FOUND DURING LOGFILE PROCESSING, RESTARTING SEARCH"; - // FIXME Reset and start without offset heuristic again - offsetLimit+=1000; + if (offset == 0) + { + emit logProcessingStatusChanged(tr("Log compressor: Timestamp %1 not found in dataset, ignoring log line %2").arg(time).arg(linecounter)); + //continue; + failed = true; + } + else + { + emit logProcessingStatusChanged(tr("Log compressor: Timestamp %1 not found in dataset, restarting search.").arg(time)); + offsetLimit*=2; + } } } + + if (index % (dataLines/10) == 0) emit logProcessingStatusChanged(tr("Log compressor: Processed %1%% of %2 lines").arg(index/(float)dataLines).arg(dataLines)); + + // When the algorithm reaches here the correct index was found lastTimeIndex = index; QString outLine = outLines->at(index); QStringList outParts = outLine.split(separator); @@ -176,12 +219,11 @@ void LogCompressor::run() outLine = outParts.join(separator); outLines->replace(index, outLine); } - - - + + // Add header, write out file file.close(); - + if (outFileName == "") { QFile::remove(file.fileName()); @@ -191,19 +233,20 @@ void LogCompressor::run() return; outfile.write(QString(QString("unix_timestamp") + separator + header.replace(" ", "_") + QString("\n")).toLatin1()); //QString fileHeader = QString("unix_timestamp") + header.replace(" ", "_") + QString("\n"); - + // File output for (int i = 0; i < outLines->length(); i++) { //qDebug() << outLines->at(i); outfile.write(QString(outLines->at(i) + "\n").toLatin1()); - + } - + currentDataLine = 0; dataLines = 1; delete keys; - qDebug() << "Done with logfile processing"; + emit logProcessingStatusChanged(tr("Log compressor: Finished processing file: %1").arg(outfile.fileName())); + //qDebug() << "Done with logfile processing"; emit finishedFile(outfile.fileName()); running = false; } diff --git a/src/LogCompressor.h b/src/LogCompressor.h index fff3dee98..5df91ce47 100644 --- a/src/LogCompressor.h +++ b/src/LogCompressor.h @@ -27,6 +27,7 @@ signals: /** @brief This signal is emitted once a logfile has been finished writing * @param fileName The name out the output (CSV) file */ + void logProcessingStatusChanged(QString); void finishedFile(QString fileName); }; diff --git a/src/ui/MainWindow.cc b/src/ui/MainWindow.cc index 56b3cb83e..9a3f66828 100644 --- a/src/ui/MainWindow.cc +++ b/src/ui/MainWindow.cc @@ -120,6 +120,8 @@ MainWindow::MainWindow(QWidget *parent): // Load mavlink view as default widget set //loadMAVLinkView(); + statusBar()->setSizeGripEnabled(true); + if (settings.contains("geometry")) { // Restore the window geometry @@ -144,8 +146,7 @@ MainWindow::MainWindow(QWidget *parent): MainWindow::~MainWindow() { - delete statusBar; - statusBar = NULL; + } void MainWindow::buildCommonWidgets() @@ -811,15 +812,6 @@ void MainWindow::configureWindowName() #endif } -QStatusBar* MainWindow::createStatusBar() -{ - QStatusBar* bar = new QStatusBar(); - /* Add status fields and messages */ - /* Enable resize grip in the bottom right corner */ - bar->setSizeGripEnabled(true); - return bar; -} - void MainWindow::startVideoCapture() { QString format = "bmp"; @@ -890,7 +882,7 @@ void MainWindow::reloadStylesheet() */ void MainWindow::showStatusMessage(const QString& status, int timeout) { - statusBar->showMessage(status, timeout); + statusBar()->showMessage(status, timeout); } /** @@ -901,7 +893,7 @@ void MainWindow::showStatusMessage(const QString& status, int timeout) */ void MainWindow::showStatusMessage(const QString& status) { - statusBar->showMessage(status, 5); + statusBar()->showMessage(status, 20000); } void MainWindow::showCriticalMessage(const QString& title, const QString& message) @@ -915,6 +907,17 @@ void MainWindow::showCriticalMessage(const QString& title, const QString& messag msgBox.exec(); } +void MainWindow::showInfoMessage(const QString& title, const QString& message) +{ + QMessageBox msgBox(MainWindow::instance()); + msgBox.setIcon(QMessageBox::Information); + msgBox.setText(title); + msgBox.setInformativeText(message); + msgBox.setStandardButtons(QMessageBox::Ok); + msgBox.setDefaultButton(QMessageBox::Ok); + msgBox.exec(); +} + /** * @brief Create all actions associated to the main window * diff --git a/src/ui/MainWindow.h b/src/ui/MainWindow.h index 9ff4500de..1d9e61d70 100644 --- a/src/ui/MainWindow.h +++ b/src/ui/MainWindow.h @@ -98,6 +98,8 @@ public slots: void showStatusMessage(const QString& status); /** @brief Shows a critical message as popup or as widget */ void showCriticalMessage(const QString& title, const QString& message); + /** @brief Shows an info message as popup or as widget */ + void showInfoMessage(const QString& title, const QString& message); void addLink(); void addLink(LinkInterface* link); @@ -296,10 +298,6 @@ protected: VIEW_SECTIONS currentView; bool aboutToCloseFlag; - QStatusBar* statusBar; - QStatusBar* createStatusBar(); - - void clearView(); void buildCommonWidgets(); diff --git a/src/ui/linechart/LinechartWidget.cc b/src/ui/linechart/LinechartWidget.cc index d1703573e..09198bdb2 100644 --- a/src/ui/linechart/LinechartWidget.cc +++ b/src/ui/linechart/LinechartWidget.cc @@ -48,6 +48,7 @@ This file is part of the PIXHAWK project #include "LinechartWidget.h" #include "LinechartPlot.h" #include "LogCompressor.h" +#include "MainWindow.h" #include "QGC.h" #include "MG.h" @@ -348,7 +349,7 @@ void LinechartWidget::startLogging() // Let user select the log file name QDate date(QDate::currentDate()); // QString("./pixhawk-log-" + date.toString("yyyy-MM-dd") + "-" + QString::number(logindex) + ".log") - QString fileName = QFileDialog::getSaveFileName(this, tr("Specify log file name"), QDesktopServices::storageLocation(QDesktopServices::DesktopLocation), tr("Logfile (*.csv, *.txt);;")); + QString fileName = QFileDialog::getSaveFileName(this, tr("Specify log file name"), QDesktopServices::storageLocation(QDesktopServices::DesktopLocation), tr("Logfile (*.csv *.txt);;")); if (!fileName.contains(".")) { @@ -399,6 +400,8 @@ void LinechartWidget::stopLogging() // Postprocess log file compressor = new LogCompressor(logFile->fileName()); connect(compressor, SIGNAL(finishedFile(QString)), this, SIGNAL(logfileWritten(QString))); + connect(compressor, SIGNAL(logProcessingStatusChanged(QString)), MainWindow::instance(), SLOT(showStatusMessage(QString))); + MainWindow::instance()->showInfoMessage("Logging ended", "QGroundControl is now compressing the logfile in a consistent CVS file. This may take a while, you can continue to use QGroundControl. Status updates appear at the bottom of the window."); compressor->startCompression(); } logButton->setText(tr("Start logging")); -- 2.22.0