LogCompressor.cc 6.99 KB
Newer Older
1
2
3
4
5
6
7
8
/****************************************************************************
 *
 *   (c) 2009-2016 QGROUNDCONTROL PROJECT <http://www.qgroundcontrol.org>
 *
 * QGroundControl is licensed according to the terms in the file
 * COPYING.md in the root of the source code directory.
 *
 ****************************************************************************/
9
10
11
12
13
14
15
16
17


/**
 * @file
 *   @brief Implementation of class LogCompressor. This class reads in a file containing messages and translates it into a tab-delimited CSV file.
 *   @author Lorenz Meier <mavteam@student.ethz.ch>
 *
 */

18
19
20
#include "LogCompressor.h"
#include "QGCApplication.h"

21
#include <QFile>
Lorenz Meier's avatar
Lorenz Meier committed
22
23
#include <QFileInfo>
#include <QDir>
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
#include <QTextStream>
#include <QStringList>
#include <QFileInfo>
#include <QList>
#include <QDebug>

/**
 * Initializes all the variables necessary for a compression run. This won't actually happen
 * until startCompression(...) is called.
 */
LogCompressor::LogCompressor(QString logFileName, QString outFileName, QString delimiter) :
	logFileName(logFileName),
	outFileName(outFileName),
	running(true),
	currentDataLine(0),
39
40
    delimiter(delimiter),
    holeFillingEnabled(true)
41
{
42
    connect(this, &LogCompressor::logProcessingCriticalError, qgcApp(), &QGCApplication::criticalMessageBoxOnMainThread);
43
44
45
46
47
48
49
}

void LogCompressor::run()
{
	// Verify that the input file is useable
	QFile infile(logFileName);
	if (!infile.exists() || !infile.open(QIODevice::ReadOnly | QIODevice::Text)) {
50
		_signalCriticalError(tr("Log Compressor: Cannot start/compress log file, since input file %1 is not readable").arg(QFileInfo(infile.fileName()).absoluteFilePath()));
51
52
53
		return;
	}

54
55
56
57
//    outFileName = logFileName;

    QString outFileName;

Lorenz Meier's avatar
Lorenz Meier committed
58
    QStringList parts = QFileInfo(infile.fileName()).absoluteFilePath().split(".", QString::SkipEmptyParts);
59

Lorenz Meier's avatar
Lorenz Meier committed
60
61
    parts.replace(0, parts.first() + "_compressed");
    parts.replace(parts.size()-1, "txt");
62
63
    outFileName = parts.join(".");

64
	// Verify that the output file is useable
65
    QFile outTmpFile(outFileName);
Lorenz Meier's avatar
Lorenz Meier committed
66
    if (!outTmpFile.open(QIODevice::WriteOnly | QIODevice::Text | QIODevice::Truncate)) {
67
		_signalCriticalError(tr("Log Compressor: Cannot start/compress log file, since output file %1 is not writable").arg(QFileInfo(outTmpFile.fileName()).absoluteFilePath()));
68
69
70
71
72
		return;
	}


	// First we search the input file through keySearchLimit number of lines
Ricardo de Almeida Gonzaga's avatar
Ricardo de Almeida Gonzaga committed
73
	// looking for variables. This is necessary before CSV files require
74
75
76
77
78
	// the same number of fields for every line.
	const unsigned int keySearchLimit = 15000;
	unsigned int keyCounter = 0;
	QTextStream in(&infile);
	QMap<QString, int> messageMap;
79

80
81
82
83
84
85
86
87
	while (!in.atEnd() && keyCounter < keySearchLimit) {
		QString messageName = in.readLine().split(delimiter).at(2);
		messageMap.insert(messageName, 0);
		++keyCounter;
	}

	// Now update each key with its index in the output string. These are
	// all offset by one to account for the first field: timestamp_ms.
LM's avatar
LM committed
88
    QMap<QString, int>::iterator i = messageMap.begin();
89
90
91
92
93
94
95
	int j;
	for (i = messageMap.begin(), j = 1; i != messageMap.end(); ++i, ++j) {
		i.value() = j;
	}

	// Open the output file and write the header line to it
	QStringList headerList(messageMap.keys());
96

97
	QString headerLine = "timestamp_ms" + delimiter + headerList.join(delimiter) + "\n";
98
    // Clean header names from symbols Matlab considers as Latex syntax
99
100
101
102
    headerLine = headerLine.replace("timestamp", "TIMESTAMP");
    headerLine = headerLine.replace(":", "");
    headerLine = headerLine.replace("_", "");
    headerLine = headerLine.replace(".", "");
103
104
	outTmpFile.write(headerLine.toLocal8Bit());

105
    _signalCriticalError(tr("Log compressor: Dataset contains dimensions: ") + headerLine);
106
107
108
109
110
111
112

    // Template list stores a list for populating with data as it's parsed from messages.
    QStringList templateList;
    for (int i = 0; i < headerList.size() + 1; ++i) {
        templateList << (holeFillingEnabled?"NaN":"");
    }

113

114
115
//	// Reset our position in the input file before we start the main processing loop.
//    in.seek(0);
116

117
118
119
120
121
122
//    // Search through all lines and build a list of unique timestamps
//    QMap<quint64, QStringList> timestampMap;
//    while (!in.atEnd()) {
//        quint64 timestamp = in.readLine().split(delimiter).at(0).toULongLong();
//        timestampMap.insert(timestamp, templateList);
//    }
123

124
    // Jump back to start of file
125
126
    in.seek(0);

127
128
129
130
    // Map of final output lines, key is time
    QMap<quint64, QStringList> timestampMap;

    // Run through the whole file and fill map of timestamps
131
132
133
    while (!in.atEnd()) {
        QStringList newLine = in.readLine().split(delimiter);
        quint64 timestamp = newLine.at(0).toULongLong();
134
135
136
137
138
139

        // Check if timestamp does exist - if not, add it
        if (!timestampMap.contains(timestamp)) {
            timestampMap.insert(timestamp, templateList);
        }

140
141
142
143
144
145
146
147
148
149
        QStringList list = timestampMap.value(timestamp);

        QString currentDataName = newLine.at(2);
        QString currentDataValue = newLine.at(3);
        list.replace(messageMap.value(currentDataName), currentDataValue);
        timestampMap.insert(timestamp, list);
    }

    int lineCounter = 0;

150
151
    QStringList lastList = timestampMap.values().at(1);

152
153
154
155
    foreach (QStringList list, timestampMap.values()) {
        // Write this current time set out to the file
        // only do so from the 2nd line on, since the first
        // line could be incomplete
156
        if (lineCounter > 1) {
157
158
            // Set the timestamp
            list.replace(0,QString("%1").arg(timestampMap.keys().at(lineCounter)));
159
160
161
162

            // Fill holes if necessary
            if (holeFillingEnabled) {
                int index = 0;
163
                foreach (const QString& str, list) {
164
165
166
167
168
169
170
171
172
173
                    if (str == "" || str == "NaN") {
                        list.replace(index, lastList.at(index));
                    }
                    index++;
                }
            }

            // Set last list
            lastList = list;

174
175
176
177
178
179
            // Write data columns
            QString output = list.join(delimiter) + "\n";
            outTmpFile.write(output.toLocal8Bit());
        }
        lineCounter++;
    }
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207

	// We're now done with the source file
	infile.close();

	// Clean up and update the status before we return.
	currentDataLine = 0;
	emit finishedFile(outFileName);
	running = false;
}

/**
 * @param holeFilling If hole filling is enabled, the compressor tries to fill empty data fields with previous
 * values from the same variable (or NaN, if no previous value existed)
 */
void LogCompressor::startCompression(bool holeFilling)
{
	holeFillingEnabled = holeFilling;
	start();
}

bool LogCompressor::isFinished()
{
	return !running;
}

int LogCompressor::getCurrentLine()
{
	return currentDataLine;
LM's avatar
LM committed
208
}
209
210
211
212
213
214


void LogCompressor::_signalCriticalError(const QString& msg)
{
    emit logProcessingCriticalError(tr("Log Compressor"), msg);
}