LogCompressor.cc 7.16 KB
Newer Older
1 2
/****************************************************************************
 *
Gus Grubba's avatar
Gus Grubba committed
3
 * (c) 2009-2020 QGROUNDCONTROL PROJECT <http://www.qgroundcontrol.org>
4 5 6 7 8
 *
 * QGroundControl is licensed according to the terms in the file
 * COPYING.md in the root of the source code directory.
 *
 ****************************************************************************/
9 10 11 12


/**
 * @file
13 14
 *   @brief Implementation of class LogCompressor.
 *          This class reads in a file containing messages and translates it into a tab-delimited CSV file.
15 16 17
 *   @author Lorenz Meier <mavteam@student.ethz.ch>
 */

18 19 20
#include "LogCompressor.h"
#include "QGCApplication.h"

21
#include <QFile>
Lorenz Meier's avatar
Lorenz Meier committed
22 23
#include <QFileInfo>
#include <QDir>
24 25 26 27 28 29 30 31 32 33 34 35 36 37 38
#include <QTextStream>
#include <QStringList>
#include <QFileInfo>
#include <QList>
#include <QDebug>

/**
 * Initializes all the variables necessary for a compression run. This won't actually happen
 * until startCompression(...) is called.
 */
LogCompressor::LogCompressor(QString logFileName, QString outFileName, QString delimiter) :
	logFileName(logFileName),
	outFileName(outFileName),
	running(true),
	currentDataLine(0),
39 40
    delimiter(delimiter),
    holeFillingEnabled(true)
41
{
42
    connect(this, &LogCompressor::logProcessingCriticalError, qgcApp(), &QGCApplication::criticalMessageBoxOnMainThread);
43 44 45 46 47 48 49
}

void LogCompressor::run()
{
	// Verify that the input file is useable
	QFile infile(logFileName);
	if (!infile.exists() || !infile.open(QIODevice::ReadOnly | QIODevice::Text)) {
50
		_signalCriticalError(tr("Log Compressor: Cannot start/compress log file, since input file %1 is not readable").arg(QFileInfo(infile.fileName()).absoluteFilePath()));
51 52 53
		return;
	}

54 55 56 57
//    outFileName = logFileName;

    QString outFileName;

58
#if QT_VERSION < QT_VERSION_CHECK(5, 15, 0)
Lorenz Meier's avatar
Lorenz Meier committed
59
    QStringList parts = QFileInfo(infile.fileName()).absoluteFilePath().split(".", QString::SkipEmptyParts);
60 61 62
#else
    QStringList parts = QFileInfo(infile.fileName()).absoluteFilePath().split(".", Qt::SkipEmptyParts);
#endif
63

Lorenz Meier's avatar
Lorenz Meier committed
64 65
    parts.replace(0, parts.first() + "_compressed");
    parts.replace(parts.size()-1, "txt");
66 67
    outFileName = parts.join(".");

68
	// Verify that the output file is useable
69
    QFile outTmpFile(outFileName);
Lorenz Meier's avatar
Lorenz Meier committed
70
    if (!outTmpFile.open(QIODevice::WriteOnly | QIODevice::Text | QIODevice::Truncate)) {
71
		_signalCriticalError(tr("Log Compressor: Cannot start/compress log file, since output file %1 is not writable").arg(QFileInfo(outTmpFile.fileName()).absoluteFilePath()));
72 73 74 75 76
		return;
	}


	// First we search the input file through keySearchLimit number of lines
Ricardo de Almeida Gonzaga's avatar
Ricardo de Almeida Gonzaga committed
77
	// looking for variables. This is necessary before CSV files require
78 79 80 81 82
	// the same number of fields for every line.
	const unsigned int keySearchLimit = 15000;
	unsigned int keyCounter = 0;
	QTextStream in(&infile);
	QMap<QString, int> messageMap;
83

84 85 86 87 88 89 90 91
	while (!in.atEnd() && keyCounter < keySearchLimit) {
		QString messageName = in.readLine().split(delimiter).at(2);
		messageMap.insert(messageName, 0);
		++keyCounter;
	}

	// Now update each key with its index in the output string. These are
	// all offset by one to account for the first field: timestamp_ms.
LM's avatar
LM committed
92
    QMap<QString, int>::iterator i = messageMap.begin();
93 94 95 96 97 98 99
	int j;
	for (i = messageMap.begin(), j = 1; i != messageMap.end(); ++i, ++j) {
		i.value() = j;
	}

	// Open the output file and write the header line to it
	QStringList headerList(messageMap.keys());
100

101
	QString headerLine = "timestamp_ms" + delimiter + headerList.join(delimiter) + "\n";
102
    // Clean header names from symbols Matlab considers as Latex syntax
103 104 105 106
    headerLine = headerLine.replace("timestamp", "TIMESTAMP");
    headerLine = headerLine.replace(":", "");
    headerLine = headerLine.replace("_", "");
    headerLine = headerLine.replace(".", "");
107 108
	outTmpFile.write(headerLine.toLocal8Bit());

109
    _signalCriticalError(tr("Log compressor: Dataset contains dimensions: ") + headerLine);
110 111 112 113 114 115 116

    // Template list stores a list for populating with data as it's parsed from messages.
    QStringList templateList;
    for (int i = 0; i < headerList.size() + 1; ++i) {
        templateList << (holeFillingEnabled?"NaN":"");
    }

117

118 119
//	// Reset our position in the input file before we start the main processing loop.
//    in.seek(0);
120

121 122 123 124 125 126
//    // Search through all lines and build a list of unique timestamps
//    QMap<quint64, QStringList> timestampMap;
//    while (!in.atEnd()) {
//        quint64 timestamp = in.readLine().split(delimiter).at(0).toULongLong();
//        timestampMap.insert(timestamp, templateList);
//    }
127

128
    // Jump back to start of file
129 130
    in.seek(0);

131 132 133 134
    // Map of final output lines, key is time
    QMap<quint64, QStringList> timestampMap;

    // Run through the whole file and fill map of timestamps
135 136 137
    while (!in.atEnd()) {
        QStringList newLine = in.readLine().split(delimiter);
        quint64 timestamp = newLine.at(0).toULongLong();
138 139 140 141 142 143

        // Check if timestamp does exist - if not, add it
        if (!timestampMap.contains(timestamp)) {
            timestampMap.insert(timestamp, templateList);
        }

144 145 146 147 148 149 150 151 152 153
        QStringList list = timestampMap.value(timestamp);

        QString currentDataName = newLine.at(2);
        QString currentDataValue = newLine.at(3);
        list.replace(messageMap.value(currentDataName), currentDataValue);
        timestampMap.insert(timestamp, list);
    }

    int lineCounter = 0;

154 155
    QStringList lastList = timestampMap.values().at(1);

156 157 158 159
    foreach (QStringList list, timestampMap.values()) {
        // Write this current time set out to the file
        // only do so from the 2nd line on, since the first
        // line could be incomplete
160
        if (lineCounter > 1) {
161 162
            // Set the timestamp
            list.replace(0,QString("%1").arg(timestampMap.keys().at(lineCounter)));
163 164 165 166

            // Fill holes if necessary
            if (holeFillingEnabled) {
                int index = 0;
167
                foreach (const QString& str, list) {
168 169 170 171 172 173 174 175 176 177
                    if (str == "" || str == "NaN") {
                        list.replace(index, lastList.at(index));
                    }
                    index++;
                }
            }

            // Set last list
            lastList = list;

178 179 180 181 182 183
            // Write data columns
            QString output = list.join(delimiter) + "\n";
            outTmpFile.write(output.toLocal8Bit());
        }
        lineCounter++;
    }
184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211

	// We're now done with the source file
	infile.close();

	// Clean up and update the status before we return.
	currentDataLine = 0;
	emit finishedFile(outFileName);
	running = false;
}

/**
 * @param holeFilling If hole filling is enabled, the compressor tries to fill empty data fields with previous
 * values from the same variable (or NaN, if no previous value existed)
 */
void LogCompressor::startCompression(bool holeFilling)
{
	holeFillingEnabled = holeFilling;
	start();
}

bool LogCompressor::isFinished()
{
	return !running;
}

int LogCompressor::getCurrentLine()
{
	return currentDataLine;
LM's avatar
LM committed
212
}
213 214 215 216 217 218


void LogCompressor::_signalCriticalError(const QString& msg)
{
    emit logProcessingCriticalError(tr("Log Compressor"), msg);
}