LogCompressor.cc 10.1 KB
Newer Older
lm's avatar
lm committed
1
/*===================================================================
pixhawk's avatar
pixhawk committed
2 3 4 5 6 7 8 9 10 11
QGroundControl Open Source Ground Control Station

(c) 2009, 2010 QGROUNDCONTROL PROJECT <http://www.qgroundcontrol.org>

This file is part of the QGROUNDCONTROL project

    QGROUNDCONTROL is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.
12

pixhawk's avatar
pixhawk committed
13 14 15 16
    QGROUNDCONTROL is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.
17

pixhawk's avatar
pixhawk committed
18 19
    You should have received a copy of the GNU General Public License
    along with QGROUNDCONTROL. If not, see <http://www.gnu.org/licenses/>.
20

pixhawk's avatar
pixhawk committed
21 22 23 24 25 26 27 28 29
======================================================================*/

/**
 * @file
 *   @brief Implementation of class LogCompressor
 *   @author Lorenz Meier <mavteam@student.ethz.ch>
 *
 */

pixhawk's avatar
pixhawk committed
30 31 32
#include <QFile>
#include <QTextStream>
#include <QStringList>
33
#include <QFileInfo>
lm's avatar
lm committed
34
#include <QList>
pixhawk's avatar
pixhawk committed
35 36 37 38
#include "LogCompressor.h"

#include <QDebug>

39 40 41
/**
 * It will only get active upon calling startCompression()
 */
42
LogCompressor::LogCompressor(QString logFileName, QString outFileName, int uasid) :
43 44 45 46 47
    logFileName(logFileName),
    outFileName(outFileName),
    running(true),
    currentDataLine(0),
    dataLines(1),
48 49
    uasid(uasid),
    holeFillingEnabled(true)
pixhawk's avatar
pixhawk committed
50 51 52 53 54 55 56 57
{
}

void LogCompressor::run()
{
    QString separator = "\t";
    QString fileName = logFileName;
    QFile file(fileName);
58
    QFile outfile(outFileName);
pixhawk's avatar
pixhawk committed
59
    QStringList* keys = new QStringList();
lm's avatar
lm committed
60 61
    QList<quint64> times;// = new QList<quint64>();
    QList<quint64> finalTimes;
lm's avatar
lm committed
62

63
    //qDebug() << "LOG COMPRESSOR: Starting" << fileName;
64 65

    if (!file.exists() || !file.open(QIODevice::ReadOnly | QIODevice::Text)) {
66
        //qDebug() << "LOG COMPRESSOR: INPUT FILE DOES NOT EXIST";
lm's avatar
lm committed
67
        emit logProcessingStatusChanged(tr("Log Compressor: Cannot start/compress log file, since input file %1 is not readable").arg(QFileInfo(fileName).absoluteFilePath()));
pixhawk's avatar
pixhawk committed
68
        return;
lm's avatar
lm committed
69
    }
70 71 72 73 74 75 76 77

    // Check if file is writeable
    if (outFileName == ""/* || !QFileInfo(outfile).isWritable()*/) {
        //qDebug() << "LOG COMPRESSOR: OUTPUT FILE DOES NOT EXIST" << outFileName;
        emit logProcessingStatusChanged(tr("Log Compressor: Cannot start/compress log file, since output file %1 is not writable").arg(QFileInfo(outFileName).absoluteFilePath()));
        return;
    }

pixhawk's avatar
pixhawk committed
78 79
    // Find all keys
    QTextStream in(&file);
lm's avatar
lm committed
80 81 82 83 84 85 86 87 88

    // Search only a certain region, assuming that not more
    // than N dimensions at H Hertz can be send
    const unsigned int keySearchLimit = 15000;
    // e.g. 500 Hz * 30 values or
    // e.g. 100 Hz * 150 values

    unsigned int keyCounter = 0;
    while (!in.atEnd() && keyCounter < keySearchLimit) {
pixhawk's avatar
pixhawk committed
89 90 91 92
        QString line = in.readLine();
        // Accumulate map of keys
        // Data field name is at position 2
        QString key = line.split(separator).at(2);
93 94 95 96
        if (!keys->contains(key))
        {
            keys->append(key);
        }
lm's avatar
lm committed
97
        keyCounter++;
pixhawk's avatar
pixhawk committed
98 99
    }
    keys->sort();
100

pixhawk's avatar
pixhawk committed
101 102
    QString header = "";
    QString spacer = "";
103
    for (int i = 0; i < keys->length(); i++) {
pixhawk's avatar
pixhawk committed
104
        header += keys->at(i) + separator;
105
        spacer += separator;
pixhawk's avatar
pixhawk committed
106 107
    }

lm's avatar
lm committed
108
    emit logProcessingStatusChanged(tr("Log compressor: Dataset contains dimension: ") + header);
109

pixhawk's avatar
pixhawk committed
110 111 112 113 114
    // Find all times
    //in.reset();
    file.reset();
    in.reset();
    in.resetStatus();
lm's avatar
lm committed
115
    bool ok;
116
    while (!in.atEnd()) {
pixhawk's avatar
pixhawk committed
117 118
        QString line = in.readLine();
        // Accumulate map of keys
lm's avatar
lm committed
119 120
        // Data field name is at position 2b
        quint64 time = static_cast<QString>(line.split(separator).at(0)).toLongLong(&ok);
121
        if (ok) {
lm's avatar
lm committed
122
            times.append(time);
pixhawk's avatar
pixhawk committed
123 124
        }
    }
125

lm's avatar
lm committed
126
    qSort(times);
127

lm's avatar
lm committed
128
    qint64 lastTime = -1;
129

pixhawk's avatar
pixhawk committed
130 131
    // Create lines
    QStringList* outLines = new QStringList();
132
    for (int i = 0; i < times.length(); i++) {
lm's avatar
lm committed
133
        // Cast to signed on purpose, 64 bit timestamp still long enough
134
        if (static_cast<qint64>(times.at(i)) != lastTime) {
lm's avatar
lm committed
135 136 137 138 139
            outLines->append(QString("%1").arg(times.at(i)) + separator + spacer);
            lastTime = static_cast<qint64>(times.at(i));
            finalTimes.append(times.at(i));
            //qDebug() << "ADDED:" << outLines->last();
        }
pixhawk's avatar
pixhawk committed
140 141
    }

lm's avatar
lm committed
142 143 144
    dataLines = finalTimes.length();

    emit logProcessingStatusChanged(tr("Log compressor: Now processing %1 log lines").arg(finalTimes.length()));
145

pixhawk's avatar
pixhawk committed
146 147 148
    // Fill in the values for all keys
    file.reset();
    QTextStream data(&file);
149
    int linecounter = 0;
150
    quint64 lastTimeIndex = 0;
lm's avatar
lm committed
151
    bool failed = false;
152 153

    while (!data.atEnd()) {
154 155
        linecounter++;
        currentDataLine = linecounter;
pixhawk's avatar
pixhawk committed
156 157 158
        QString line = data.readLine();
        QStringList parts = line.split(separator);
        // Get time
lm's avatar
lm committed
159
        quint64 time = static_cast<QString>(parts.first()).toLongLong(&ok);
pixhawk's avatar
pixhawk committed
160
        QString field = parts.at(2);
161
        int fieldIndex = keys->indexOf(field);
pixhawk's avatar
pixhawk committed
162
        QString value = parts.at(3);
163 164 165 166 167
//        // Enforce NaN if no value is present
//        if (value.length() == 0 || value == "" || value == " " || value == "\t" || value == "\n") {
//            // Hole filling disabled, fill with NaN
//            value = "NaN";
//        }
pixhawk's avatar
pixhawk committed
168
        // Get matching output line
169

170 171
        // Constraining the search area might result in not finding a key,
        // but it significantly reduces the time needed for the search
172
        // setting a window of 100 entries means that a 1 Hz data point
173
        // can still be located
lm's avatar
lm committed
174
        quint64 offsetLimit = 100;
175
        quint64 offset;
lm's avatar
lm committed
176 177 178 179 180
        qint64 index = -1;
        failed = false;

        // Search the index until it is valid (!= -1)
        // or the start of the list has been reached (failed)
181 182
        while (index == -1 && !failed) {
            if (lastTimeIndex > offsetLimit) {
183
                offset = lastTimeIndex - offsetLimit;
184
            } else {
185 186
                offset = 0;
            }
187

lm's avatar
lm committed
188
            index = finalTimes.indexOf(time, offset);
189 190
            if (index == -1) {
                if (offset == 0) {
lm's avatar
lm committed
191
                    emit logProcessingStatusChanged(tr("Log compressor: Timestamp %1 not found in dataset, ignoring log line %2").arg(time).arg(linecounter));
lm's avatar
lm committed
192
                    qDebug() << "Completely failed finding value";
lm's avatar
lm committed
193 194
                    //continue;
                    failed = true;
195
                } else {
lm's avatar
lm committed
196 197 198
                    emit logProcessingStatusChanged(tr("Log compressor: Timestamp %1 not found in dataset, restarting search.").arg(time));
                    offsetLimit*=2;
                }
199 200
            }
        }
lm's avatar
lm committed
201

202
        if (dataLines > 100) if (index % (dataLines/100) == 0) emit logProcessingStatusChanged(tr("Log compressor: Processed %1% of %2 lines").arg(index/(float)dataLines*100, 0, 'f', 2).arg(dataLines));
203 204

        if (!failed) {
lm's avatar
lm committed
205 206 207 208 209
            // When the algorithm reaches here the correct index was found
            lastTimeIndex = index;
            QString outLine = outLines->at(index);
            QStringList outParts = outLine.split(separator);
            // Replace measurement placeholder with current value
210
            outParts.replace(fieldIndex+1, value);
lm's avatar
lm committed
211 212 213
            outLine = outParts.join(separator);
            outLines->replace(index, outLine);
        }
pixhawk's avatar
pixhawk committed
214
    }
215

216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257
    ///////////////////////////
    // HOLE FILLING

    // If hole filling is enabled, run again through the whole file and replace holes
    if (holeFillingEnabled)
    {
        // Build up the fill values - initialize to NaN
        QStringList fillValues;
        int fillCount = keys->count();
        for (int i = 0; i< fillCount; ++i)
        {
            fillValues.append("NaN");
        }

        // Run through all lines and replace with fill values
        for (int index = 0; index < outLines->count(); ++index)
        {
            QString line = outLines->at(index);
            QStringList fields = line.split(separator, QString::SkipEmptyParts);
            // The fields line contains the timestamp
            // index of the data fields therefore runs from 1 to n-1
            int fieldCount = fields.count();
            for (int i = 1; i < fillCount+1; ++i)
            {
                if (fieldCount < fillCount) fields.append("");

                // Check if this is NaN
                if (fields[i] == 0 || fields[i] == "" || fields[i] == "\t" || fields[i] == " " || fields[i] == "\n")
                {
                    // Value was empty, replace it
                    fields.replace(i, fillValues[i-1]);
                }
                else
                {
                    // Value was not NaN, use it as
                    // new fill value
                    fillValues.replace(i-1, fields[i]);
                }
            }
            outLines->replace(index, fields.join(separator));
        }
    }
258

pixhawk's avatar
pixhawk committed
259 260
    // Add header, write out file
    file.close();
261 262

    if (outFileName == logFileName) {
263 264
        QFile::remove(file.fileName());
        outfile.setFileName(file.fileName());
265

266 267
    }
    if (!outfile.open(QIODevice::WriteOnly | QIODevice::Text))
pixhawk's avatar
pixhawk committed
268
        return;
269
    outfile.write(QString(QString("timestamp_ms") + separator + header.replace(" ", "_") + QString("\n")).toLatin1());
270
    emit logProcessingStatusChanged(tr("Log Compressor: Writing output to file %1").arg(QFileInfo(outFileName).absoluteFilePath()));
271

272
    // File output
273
    for (int i = 0; i < outLines->length(); i++) {
pixhawk's avatar
pixhawk committed
274
        //qDebug() << outLines->at(i);
275
        outfile.write(QString(outLines->at(i) + "\n").toLatin1());
276

pixhawk's avatar
pixhawk committed
277
    }
278

279 280
    currentDataLine = 0;
    dataLines = 1;
pixhawk's avatar
pixhawk committed
281
    delete keys;
lm's avatar
lm committed
282
    emit logProcessingStatusChanged(tr("Log compressor: Finished processing file: %1").arg(outfile.fileName()));
lm's avatar
lm committed
283
    qDebug() << "Done with logfile processing";
284
    emit finishedFile(outfile.fileName());
285 286 287
    running = false;
}

288 289 290 291 292
void LogCompressor::startCompression()
{
    start();
}

293 294 295 296 297 298 299 300 301 302 303 304 305
bool LogCompressor::isFinished()
{
    return !running;
}

int LogCompressor::getCurrentLine()
{
    return currentDataLine;
}

int LogCompressor::getDataLines()
{
    return dataLines;
pixhawk's avatar
pixhawk committed
306
}