LogCompressor.cc 6.99 KB
Newer Older
1 2
/****************************************************************************
 *
Gus Grubba's avatar
Gus Grubba committed
3
 * (c) 2009-2020 QGROUNDCONTROL PROJECT <http://www.qgroundcontrol.org>
4 5 6 7 8
 *
 * QGroundControl is licensed according to the terms in the file
 * COPYING.md in the root of the source code directory.
 *
 ****************************************************************************/
9 10 11 12


/**
 * @file
13 14
 *   @brief Implementation of class LogCompressor.
 *          This class reads in a file containing messages and translates it into a tab-delimited CSV file.
15 16 17
 *   @author Lorenz Meier <mavteam@student.ethz.ch>
 */

18 19 20
#include "LogCompressor.h"
#include "QGCApplication.h"

21
#include <QFile>
Lorenz Meier's avatar
Lorenz Meier committed
22 23
#include <QFileInfo>
#include <QDir>
24 25 26 27 28 29 30 31 32 33 34 35 36 37 38
#include <QTextStream>
#include <QStringList>
#include <QFileInfo>
#include <QList>
#include <QDebug>

/**
 * Initializes all the variables necessary for a compression run. This won't actually happen
 * until startCompression(...) is called.
 */
LogCompressor::LogCompressor(QString logFileName, QString outFileName, QString delimiter) :
	logFileName(logFileName),
	outFileName(outFileName),
	running(true),
	currentDataLine(0),
39 40
    delimiter(delimiter),
    holeFillingEnabled(true)
41
{
42
    connect(this, &LogCompressor::logProcessingCriticalError, qgcApp(), &QGCApplication::criticalMessageBoxOnMainThread);
43 44 45 46 47 48 49
}

void LogCompressor::run()
{
	// Verify that the input file is useable
	QFile infile(logFileName);
	if (!infile.exists() || !infile.open(QIODevice::ReadOnly | QIODevice::Text)) {
50
		_signalCriticalError(tr("Log Compressor: Cannot start/compress log file, since input file %1 is not readable").arg(QFileInfo(infile.fileName()).absoluteFilePath()));
51 52 53
		return;
	}

54 55 56 57
//    outFileName = logFileName;

    QString outFileName;

Lorenz Meier's avatar
Lorenz Meier committed
58
    QStringList parts = QFileInfo(infile.fileName()).absoluteFilePath().split(".", QString::SkipEmptyParts);
59

Lorenz Meier's avatar
Lorenz Meier committed
60 61
    parts.replace(0, parts.first() + "_compressed");
    parts.replace(parts.size()-1, "txt");
62 63
    outFileName = parts.join(".");

64
	// Verify that the output file is useable
65
    QFile outTmpFile(outFileName);
Lorenz Meier's avatar
Lorenz Meier committed
66
    if (!outTmpFile.open(QIODevice::WriteOnly | QIODevice::Text | QIODevice::Truncate)) {
67
		_signalCriticalError(tr("Log Compressor: Cannot start/compress log file, since output file %1 is not writable").arg(QFileInfo(outTmpFile.fileName()).absoluteFilePath()));
68 69 70 71 72
		return;
	}


	// First we search the input file through keySearchLimit number of lines
Ricardo de Almeida Gonzaga's avatar
Ricardo de Almeida Gonzaga committed
73
	// looking for variables. This is necessary before CSV files require
74 75 76 77 78
	// the same number of fields for every line.
	const unsigned int keySearchLimit = 15000;
	unsigned int keyCounter = 0;
	QTextStream in(&infile);
	QMap<QString, int> messageMap;
79

80 81 82 83 84 85 86 87
	while (!in.atEnd() && keyCounter < keySearchLimit) {
		QString messageName = in.readLine().split(delimiter).at(2);
		messageMap.insert(messageName, 0);
		++keyCounter;
	}

	// Now update each key with its index in the output string. These are
	// all offset by one to account for the first field: timestamp_ms.
LM's avatar
LM committed
88
    QMap<QString, int>::iterator i = messageMap.begin();
89 90 91 92 93 94 95
	int j;
	for (i = messageMap.begin(), j = 1; i != messageMap.end(); ++i, ++j) {
		i.value() = j;
	}

	// Open the output file and write the header line to it
	QStringList headerList(messageMap.keys());
96

97
	QString headerLine = "timestamp_ms" + delimiter + headerList.join(delimiter) + "\n";
98
    // Clean header names from symbols Matlab considers as Latex syntax
99 100 101 102
    headerLine = headerLine.replace("timestamp", "TIMESTAMP");
    headerLine = headerLine.replace(":", "");
    headerLine = headerLine.replace("_", "");
    headerLine = headerLine.replace(".", "");
103 104
	outTmpFile.write(headerLine.toLocal8Bit());

105
    _signalCriticalError(tr("Log compressor: Dataset contains dimensions: ") + headerLine);
106 107 108 109 110 111 112

    // Template list stores a list for populating with data as it's parsed from messages.
    QStringList templateList;
    for (int i = 0; i < headerList.size() + 1; ++i) {
        templateList << (holeFillingEnabled?"NaN":"");
    }

113

114 115
//	// Reset our position in the input file before we start the main processing loop.
//    in.seek(0);
116

117 118 119 120 121 122
//    // Search through all lines and build a list of unique timestamps
//    QMap<quint64, QStringList> timestampMap;
//    while (!in.atEnd()) {
//        quint64 timestamp = in.readLine().split(delimiter).at(0).toULongLong();
//        timestampMap.insert(timestamp, templateList);
//    }
123

124
    // Jump back to start of file
125 126
    in.seek(0);

127 128 129 130
    // Map of final output lines, key is time
    QMap<quint64, QStringList> timestampMap;

    // Run through the whole file and fill map of timestamps
131 132 133
    while (!in.atEnd()) {
        QStringList newLine = in.readLine().split(delimiter);
        quint64 timestamp = newLine.at(0).toULongLong();
134 135 136 137 138 139

        // Check if timestamp does exist - if not, add it
        if (!timestampMap.contains(timestamp)) {
            timestampMap.insert(timestamp, templateList);
        }

140 141 142 143 144 145 146 147 148 149
        QStringList list = timestampMap.value(timestamp);

        QString currentDataName = newLine.at(2);
        QString currentDataValue = newLine.at(3);
        list.replace(messageMap.value(currentDataName), currentDataValue);
        timestampMap.insert(timestamp, list);
    }

    int lineCounter = 0;

150 151
    QStringList lastList = timestampMap.values().at(1);

152 153 154 155
    foreach (QStringList list, timestampMap.values()) {
        // Write this current time set out to the file
        // only do so from the 2nd line on, since the first
        // line could be incomplete
156
        if (lineCounter > 1) {
157 158
            // Set the timestamp
            list.replace(0,QString("%1").arg(timestampMap.keys().at(lineCounter)));
159 160 161 162

            // Fill holes if necessary
            if (holeFillingEnabled) {
                int index = 0;
163
                foreach (const QString& str, list) {
164 165 166 167 168 169 170 171 172 173
                    if (str == "" || str == "NaN") {
                        list.replace(index, lastList.at(index));
                    }
                    index++;
                }
            }

            // Set last list
            lastList = list;

174 175 176 177 178 179
            // Write data columns
            QString output = list.join(delimiter) + "\n";
            outTmpFile.write(output.toLocal8Bit());
        }
        lineCounter++;
    }
180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207

	// We're now done with the source file
	infile.close();

	// Clean up and update the status before we return.
	currentDataLine = 0;
	emit finishedFile(outFileName);
	running = false;
}

/**
 * @param holeFilling If hole filling is enabled, the compressor tries to fill empty data fields with previous
 * values from the same variable (or NaN, if no previous value existed)
 */
void LogCompressor::startCompression(bool holeFilling)
{
	holeFillingEnabled = holeFilling;
	start();
}

bool LogCompressor::isFinished()
{
	return !running;
}

int LogCompressor::getCurrentLine()
{
	return currentDataLine;
LM's avatar
LM committed
208
}
209 210 211 212 213 214


void LogCompressor::_signalCriticalError(const QString& msg)
{
    emit logProcessingCriticalError(tr("Log Compressor"), msg);
}