/**************************************************************************** * * (c) 2009-2016 QGROUNDCONTROL PROJECT * * QGroundControl is licensed according to the terms in the file * COPYING.md in the root of the source code directory. * ****************************************************************************/ /** * @file * @brief Map Tile Cache Worker Thread * * @author Gus Grubba * */ #include "QGCMapEngine.h" #include "QGCMapTileSet.h" #include #include #include #include #include #include #include #include "time.h" const char* kDefaultSet = "Default Tile Set"; const QString kSession = QLatin1String("QGeoTileWorkerSession"); QGC_LOGGING_CATEGORY(QGCTileCacheLog, "QGCTileCacheLog") //-- Update intervals #define LONG_TIMEOUT 5 #define SHORT_TIMEOUT 2 //----------------------------------------------------------------------------- QGCCacheWorker::QGCCacheWorker() : _db(NULL) , _valid(false) , _failed(false) , _defaultSet(UINT64_MAX) , _totalSize(0) , _totalCount(0) , _defaultSize(0) , _defaultCount(0) , _lastUpdate(0) , _updateTimeout(SHORT_TIMEOUT) { } //----------------------------------------------------------------------------- QGCCacheWorker::~QGCCacheWorker() { } //----------------------------------------------------------------------------- void QGCCacheWorker::setDatabaseFile(const QString& path) { _databasePath = path; } //----------------------------------------------------------------------------- void QGCCacheWorker::quit() { _mutex.lock(); while(_taskQueue.count()) { QGCMapTask* task = _taskQueue.dequeue(); delete task; } _mutex.unlock(); if(this->isRunning()) { _waitc.wakeAll(); } } //----------------------------------------------------------------------------- bool QGCCacheWorker::enqueueTask(QGCMapTask* task) { //-- If not initialized, the only allowed task is Init if(!_valid && task->type() != QGCMapTask::taskInit) { task->setError("Database Not Initialized"); task->deleteLater(); return false; } if(!_taskQueue.contains(task)) { _mutex.lock(); _taskQueue.enqueue(task); _mutex.unlock(); if(this->isRunning()) { _waitc.wakeAll(); } else { this->start(QThread::NormalPriority); } return true; } //-- Should never happen return false; } //----------------------------------------------------------------------------- void QGCCacheWorker::run() { if(!_valid && !_failed) { _init(); } if(_valid) { _db = new QSqlDatabase(QSqlDatabase::addDatabase("QSQLITE", kSession)); _db->setDatabaseName(_databasePath); _db->setConnectOptions("QSQLITE_ENABLE_SHARED_CACHE"); _valid = _db->open(); } while(true) { QGCMapTask* task; if(_taskQueue.count()) { _mutex.lock(); task = _taskQueue.dequeue(); _mutex.unlock(); switch(task->type()) { case QGCMapTask::taskInit: break; case QGCMapTask::taskCacheTile: _saveTile(task); break; case QGCMapTask::taskFetchTile: _getTile(task); break; case QGCMapTask::taskFetchTileSets: _getTileSets(task); break; case QGCMapTask::taskCreateTileSet: _createTileSet(task); break; case QGCMapTask::taskGetTileDownloadList: _getTileDownloadList(task); break; case QGCMapTask::taskUpdateTileDownloadState: _updateTileDownloadState(task); break; case QGCMapTask::taskDeleteTileSet: _deleteTileSet(task); break; case QGCMapTask::taskPruneCache: _pruneCache(task); break; case QGCMapTask::taskReset: _resetCacheDatabase(task); break; } task->deleteLater(); //-- Check for update timeout size_t count = _taskQueue.count(); if(count > 100) { _updateTimeout = LONG_TIMEOUT; } else if(count < 25) { _updateTimeout = SHORT_TIMEOUT; } if(!count || (time(0) - _lastUpdate > _updateTimeout)) { if(_valid) { _updateTotals(); } } } else { //-- Wait a bit before shutting things down _waitmutex.lock(); int timeout = 5000; if(!_waitc.wait(&_waitmutex, timeout)) { _waitmutex.unlock(); _mutex.lock(); //-- If nothing to do, close db and leave thread if(!_taskQueue.count()) { _mutex.unlock(); break; } _mutex.unlock(); } _waitmutex.unlock(); } } if(_db) { delete _db; _db = NULL; QSqlDatabase::removeDatabase(kSession); } } //----------------------------------------------------------------------------- bool QGCCacheWorker::_findTileSetID(const QString name, quint64& setID) { QSqlQuery query(*_db); QString s = QString("SELECT setID FROM TileSets WHERE name = \"%1\"").arg(name); if(query.exec(s)) { if(query.next()) { setID = query.value(0).toULongLong(); return true; } } return false; } //----------------------------------------------------------------------------- quint64 QGCCacheWorker::_getDefaultTileSet() { if(_defaultSet != UINT64_MAX) return _defaultSet; QSqlQuery query(*_db); QString s = QString("SELECT setID FROM TileSets WHERE defaultSet = 1"); if(query.exec(s)) { if(query.next()) { _defaultSet = query.value(0).toULongLong(); return _defaultSet; } } return 1L; } //----------------------------------------------------------------------------- void QGCCacheWorker::_saveTile(QGCMapTask *mtask) { if(_valid) { QGCSaveTileTask* task = static_cast(mtask); QSqlQuery query(*_db); query.prepare("INSERT INTO Tiles(hash, format, tile, size, type, date) VALUES(?, ?, ?, ?, ?, ?)"); query.addBindValue(task->tile()->hash()); query.addBindValue(task->tile()->format()); query.addBindValue(task->tile()->img()); query.addBindValue(task->tile()->img().size()); query.addBindValue(task->tile()->type()); query.addBindValue(QDateTime::currentDateTime().toTime_t()); if(query.exec()) { quint64 tileID = query.lastInsertId().toULongLong(); quint64 setID = task->tile()->set() == UINT64_MAX ? _getDefaultTileSet() : task->tile()->set(); QString s = QString("INSERT INTO SetTiles(tileID, setID) VALUES(%1, %2)").arg(tileID).arg(setID); query.prepare(s); if(!query.exec()) { qWarning() << "Map Cache SQL error (add tile into SetTiles):" << query.lastError().text(); } qCDebug(QGCTileCacheLog) << "_saveTile() HASH:" << task->tile()->hash(); } else { //-- Tile was already there. // QtLocation some times requests the same tile twice in a row. The first is saved, the second is already there. } } else { qWarning() << "Map Cache SQL error (saveTile() open db):" << _db->lastError(); } } //----------------------------------------------------------------------------- void QGCCacheWorker::_getTile(QGCMapTask* mtask) { if(!_valid) { mtask->setError("No Cache Database"); return; } bool found = false; QGCFetchTileTask* task = static_cast(mtask); QSqlQuery query(*_db); QString s = QString("SELECT tile, format, type FROM Tiles WHERE hash = \"%1\"").arg(task->hash()); if(query.exec(s)) { if(query.next()) { QByteArray ar = query.value(0).toByteArray(); QString format = query.value(1).toString(); UrlFactory::MapType type = (UrlFactory::MapType)query.value(2).toInt(); qCDebug(QGCTileCacheLog) << "_getTile() (Found in DB) HASH:" << task->hash(); QGCCacheTile* tile = new QGCCacheTile(task->hash(), ar, format, type); task->setTileFetched(tile); found = true; } } if(!found) { qCDebug(QGCTileCacheLog) << "_getTile() (NOT in DB) HASH:" << task->hash(); task->setError("Tile not in cache database"); } } //----------------------------------------------------------------------------- void QGCCacheWorker::_getTileSets(QGCMapTask* mtask) { if(!_valid) { mtask->setError("No Cache Database"); return; } QGCFetchTileSetTask* task = static_cast(mtask); QSqlQuery query(*_db); QString s = QString("SELECT * FROM TileSets ORDER BY defaultSet DESC, name ASC"); qCDebug(QGCTileCacheLog) << "_getTileSets(): " << s; if(query.exec(s)) { while(query.next()) { QString name = query.value("name").toString(); QGCCachedTileSet* set = new QGCCachedTileSet(name); set->setId(query.value("setID").toULongLong()); set->setMapTypeStr(query.value("typeStr").toString()); set->setTopleftLat(query.value("topleftLat").toDouble()); set->setTopleftLon(query.value("topleftLon").toDouble()); set->setBottomRightLat(query.value("bottomRightLat").toDouble()); set->setBottomRightLon(query.value("bottomRightLon").toDouble()); set->setMinZoom(query.value("minZoom").toInt()); set->setMaxZoom(query.value("maxZoom").toInt()); set->setType((UrlFactory::MapType)query.value("type").toInt()); set->setTotalTileCount(query.value("numTiles").toUInt()); set->setDefaultSet(query.value("defaultSet").toInt() != 0); set->setCreationDate(QDateTime::fromTime_t(query.value("date").toUInt())); _updateSetTotals(set); //-- Object created here must be moved to app thread to be used there set->moveToThread(QApplication::instance()->thread()); task->tileSetFetched(set); } } else { task->setError("No tile set in database"); } } //----------------------------------------------------------------------------- void QGCCacheWorker::_updateSetTotals(QGCCachedTileSet* set) { if(set->defaultSet()) { _updateTotals(); set->setSavedTileCount(_totalCount); set->setSavedTileSize(_totalSize); set->setTotalTileCount(_defaultCount); set->setTotalTileSize(_defaultSize); return; } QSqlQuery subquery(*_db); QString sq = QString("SELECT COUNT(size), SUM(size) FROM Tiles A INNER JOIN SetTiles B on A.tileID = B.tileID WHERE B.setID = %1").arg(set->id()); qCDebug(QGCTileCacheLog) << "_updateSetTotals(): " << sq; if(subquery.exec(sq)) { if(subquery.next()) { set->setSavedTileCount(subquery.value(0).toUInt()); set->setSavedTileSize(subquery.value(1).toULongLong()); qCDebug(QGCTileCacheLog) << "Set" << set->id() << "Totals:" << set->savedTileCount() << " " << set->savedTileSize() << "Expected: " << set->totalTileCount() << " " << set->totalTilesSize(); //-- Update (estimated) size quint64 avg = UrlFactory::averageSizeForType(set->type()); if(set->totalTileCount() <= set->savedTileCount()) { //-- We're done so the saved size is the total size set->setTotalTileSize(set->savedTileSize()); } else { //-- Otherwise we need to estimate it. if(set->savedTileCount() > 10 && set->savedTileSize()) { avg = set->savedTileSize() / set->savedTileCount(); } set->setTotalTileSize(avg * set->totalTileCount()); } //-- Now figure out the count for tiles unique to this set quint32 ucount = 0; quint64 usize = 0; sq = QString("SELECT COUNT(size), SUM(size) FROM Tiles WHERE tileID IN (SELECT A.tileID FROM SetTiles A join SetTiles B on A.tileID = B.tileID WHERE B.setID = %1 GROUP by A.tileID HAVING COUNT(A.tileID) = 1)").arg(set->id()); if(subquery.exec(sq)) { if(subquery.next()) { //-- This is only accurate when all tiles are downloaded ucount = subquery.value(0).toUInt(); usize = subquery.value(1).toULongLong(); } } //-- If we haven't downloaded it all, estimate size of unique tiles quint32 expectedUcount = set->totalTileCount() - set->savedTileCount(); if(!ucount) { usize = expectedUcount * avg; } else { expectedUcount = ucount; } set->setUniqueTileCount(expectedUcount); set->setUniqueTileSize(usize); } } } //----------------------------------------------------------------------------- void QGCCacheWorker::_updateTotals() { QSqlQuery query(*_db); QString s; s = QString("SELECT COUNT(size), SUM(size) FROM Tiles"); qCDebug(QGCTileCacheLog) << "_updateTotals(): " << s; if(query.exec(s)) { if(query.next()) { _totalCount = query.value(0).toUInt(); _totalSize = query.value(1).toULongLong(); } } s = QString("SELECT COUNT(size), SUM(size) FROM Tiles WHERE tileID IN (SELECT A.tileID FROM SetTiles A join SetTiles B on A.tileID = B.tileID WHERE B.setID = %1 GROUP by A.tileID HAVING COUNT(A.tileID) = 1)").arg(_getDefaultTileSet()); qCDebug(QGCTileCacheLog) << "_updateTotals(): " << s; if(query.exec(s)) { if(query.next()) { _defaultCount = query.value(0).toUInt(); _defaultSize = query.value(1).toULongLong(); } } emit updateTotals(_totalCount, _totalSize, _defaultCount, _defaultSize); _lastUpdate = time(0); } //----------------------------------------------------------------------------- quint64 QGCCacheWorker::_findTile(const QString hash) { quint64 tileID = 0; QSqlQuery query(*_db); QString s = QString("SELECT tileID FROM Tiles WHERE hash = \"%1\"").arg(hash); if(query.exec(s)) { if(query.next()) { tileID = query.value(0).toULongLong(); } } return tileID; } //----------------------------------------------------------------------------- void QGCCacheWorker::_createTileSet(QGCMapTask *mtask) { if(_valid) { //-- Create Tile Set quint32 actual_count = 0; QGCCreateTileSetTask* task = static_cast(mtask); QSqlQuery query(*_db); query.prepare("INSERT INTO TileSets(" "name, typeStr, topleftLat, topleftLon, bottomRightLat, bottomRightLon, minZoom, maxZoom, type, numTiles, date" ") VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"); query.addBindValue(task->tileSet()->name()); query.addBindValue(task->tileSet()->mapTypeStr()); query.addBindValue(task->tileSet()->topleftLat()); query.addBindValue(task->tileSet()->topleftLon()); query.addBindValue(task->tileSet()->bottomRightLat()); query.addBindValue(task->tileSet()->bottomRightLon()); query.addBindValue(task->tileSet()->minZoom()); query.addBindValue(task->tileSet()->maxZoom()); query.addBindValue(task->tileSet()->type()); query.addBindValue(task->tileSet()->totalTileCount()); query.addBindValue(QDateTime::currentDateTime().toTime_t()); if(!query.exec()) { qWarning() << "Map Cache SQL error (add tileSet into TileSets):" << query.lastError().text(); } else { //-- Get just created (auto-incremented) setID quint64 setID = query.lastInsertId().toULongLong(); task->tileSet()->setId(setID); //-- Prepare Download List quint64 tileCount = 0; for(int z = task->tileSet()->minZoom(); z <= task->tileSet()->maxZoom(); z++) { QGCTileSet set = QGCMapEngine::getTileCount(z, task->tileSet()->topleftLon(), task->tileSet()->topleftLat(), task->tileSet()->bottomRightLon(), task->tileSet()->bottomRightLat(), task->tileSet()->type()); tileCount += set.tileCount; UrlFactory::MapType type = task->tileSet()->type(); for(int x = set.tileX0; x <= set.tileX1; x++) { for(int y = set.tileY0; y <= set.tileY1; y++) { //-- See if tile is already downloaded QString hash = QGCMapEngine::getTileHash(type, x, y, z); quint64 tileID = _findTile(hash); if(!tileID) { //-- Set to download query.prepare("INSERT OR IGNORE INTO TilesDownload(setID, hash, type, x, y, z, state) VALUES(?, ?, ?, ?, ? ,? ,?)"); query.addBindValue(setID); query.addBindValue(hash); query.addBindValue(type); query.addBindValue(x); query.addBindValue(y); query.addBindValue(z); query.addBindValue(0); if(!query.exec()) { qWarning() << "Map Cache SQL error (add tile into TilesDownload):" << query.lastError().text(); mtask->setError("Error creating tile set download list"); return; } else actual_count++; } else { //-- Tile already in the database. No need to dowload. QString s = QString("INSERT OR IGNORE INTO SetTiles(tileID, setID) VALUES(%1, %2)").arg(tileID).arg(setID); query.prepare(s); if(!query.exec()) { qWarning() << "Map Cache SQL error (add tile into SetTiles):" << query.lastError().text(); } qCDebug(QGCTileCacheLog) << "_createTileSet() Already Cached HASH:" << hash; } } } } //-- Done _updateSetTotals(task->tileSet()); task->setTileSetSaved(); return; } } mtask->setError("Error saving tile set"); } //----------------------------------------------------------------------------- void QGCCacheWorker::_getTileDownloadList(QGCMapTask* mtask) { if(!_valid) { mtask->setError("No Cache Database"); return; } QList tiles; QGCGetTileDownloadListTask* task = static_cast(mtask); QSqlQuery query(*_db); QString s = QString("SELECT hash, type, x, y, z FROM TilesDownload WHERE setID = %1 AND state = 0 LIMIT %2").arg(task->setID()).arg(task->count()); if(query.exec(s)) { while(query.next()) { QGCTile* tile = new QGCTile; tile->setHash(query.value("hash").toString()); tile->setType((UrlFactory::MapType)query.value("type").toInt()); tile->setX(query.value("x").toInt()); tile->setY(query.value("y").toInt()); tile->setZ(query.value("z").toInt()); tiles.append(tile); } for(int i = 0; i < tiles.size(); i++) { s = QString("UPDATE TilesDownload SET state = %1 WHERE setID = %2 and hash = \"%3\"").arg((int)QGCTile::StateDownloading).arg(task->setID()).arg(tiles[i]->hash()); if(!query.exec(s)) { qWarning() << "Map Cache SQL error (set TilesDownload state):" << query.lastError().text(); } } } task->setTileListFetched(tiles); } //----------------------------------------------------------------------------- void QGCCacheWorker::_updateTileDownloadState(QGCMapTask* mtask) { if(!_valid) { mtask->setError("No Cache Database"); return; } QGCUpdateTileDownloadStateTask* task = static_cast(mtask); QSqlQuery query(*_db); QString s; if(task->state() == QGCTile::StateComplete) { s = QString("DELETE FROM TilesDownload WHERE setID = %1 AND hash = \"%2\"").arg(task->setID()).arg(task->hash()); } else { if(task->hash() == "*") { s = QString("UPDATE TilesDownload SET state = %1 WHERE setID = %2").arg((int)task->state()).arg(task->setID()); } else { s = QString("UPDATE TilesDownload SET state = %1 WHERE setID = %2 AND hash = \"%3\"").arg((int)task->state()).arg(task->setID()).arg(task->hash()); } } if(!query.exec(s)) { qWarning() << "QGCCacheWorker::_updateTileDownloadState() Error:" << query.lastError().text(); } } //----------------------------------------------------------------------------- void QGCCacheWorker::_pruneCache(QGCMapTask* mtask) { if(!_valid) { mtask->setError("No Cache Database"); return; } QGCPruneCacheTask* task = static_cast(mtask); QSqlQuery query(*_db); QString s; //-- Select tiles in default set only, sorted by oldest. s = QString("SELECT tileID, size, hash FROM Tiles WHERE tileID IN (SELECT A.tileID FROM SetTiles A join SetTiles B on A.tileID = B.tileID WHERE B.setID = %1 GROUP by A.tileID HAVING COUNT(A.tileID) = 1) ORDER BY DATE ASC LIMIT 128").arg(_getDefaultTileSet()); qint64 amount = (qint64)task->amount(); QList tlist; if(query.exec(s)) { while(query.next() && amount >= 0) { tlist << query.value(0).toULongLong(); amount -= query.value(1).toULongLong(); qCDebug(QGCTileCacheLog) << "_pruneCache() HASH:" << query.value(2).toString(); } while(tlist.count()) { s = QString("DELETE FROM Tiles WHERE tileID = %1").arg(tlist[0]); tlist.removeFirst(); if(!query.exec(s)) break; } task->setPruned(); } } //----------------------------------------------------------------------------- void QGCCacheWorker::_deleteTileSet(QGCMapTask* mtask) { if(!_valid) { mtask->setError("No Cache Database"); return; } QGCDeleteTileSetTask* task = static_cast(mtask); QSqlQuery query(*_db); QString s; //-- Only delete tiles unique to this set s = QString("DELETE FROM Tiles WHERE tileID IN (SELECT A.tileID FROM SetTiles A JOIN SetTiles B ON A.tileID = B.tileID WHERE B.setID = %1 GROUP BY A.tileID HAVING COUNT(A.tileID) = 1)").arg(task->setID()); query.exec(s); s = QString("DELETE FROM TilesDownload WHERE setID = %1").arg(task->setID()); query.exec(s); s = QString("DELETE FROM TileSets WHERE setID = %1").arg(task->setID()); query.exec(s); s = QString("DELETE FROM SetTiles WHERE setID = %1").arg(task->setID()); query.exec(s); _updateTotals(); task->setTileSetDeleted(); } //----------------------------------------------------------------------------- void QGCCacheWorker::_resetCacheDatabase(QGCMapTask* mtask) { if(!_valid) { mtask->setError("No Cache Database"); return; } QGCResetTask* task = static_cast(mtask); QSqlQuery query(*_db); QString s; s = QString("DROP TABLE Tiles"); query.exec(s); s = QString("DROP TABLE TileSets"); query.exec(s); s = QString("DROP TABLE SetTiles"); query.exec(s); s = QString("DROP TABLE TilesDownload"); query.exec(s); _createDB(); task->setResetCompleted(); } //----------------------------------------------------------------------------- bool QGCCacheWorker::_init() { _failed = false; if(!_databasePath.isEmpty()) { qCDebug(QGCTileCacheLog) << "Mapping cache directory:" << _databasePath; //-- Initialize Database _db = new QSqlDatabase(QSqlDatabase::addDatabase("QSQLITE", kSession)); _db->setDatabaseName(_databasePath); _db->setConnectOptions("QSQLITE_ENABLE_SHARED_CACHE"); if (_db->open()) { _createDB(); } else { qCritical() << "Map Cache SQL error (init() open db):" << _db->lastError(); _failed = true; } delete _db; _db = NULL; QSqlDatabase::removeDatabase(kSession); } else { qCritical() << "Could not find suitable cache directory."; _failed = true; } return _failed; } //----------------------------------------------------------------------------- void QGCCacheWorker::_createDB() { QSqlQuery query(*_db); if(!query.exec( "CREATE TABLE IF NOT EXISTS Tiles (" "tileID INTEGER PRIMARY KEY NOT NULL, " "hash TEXT NOT NULL UNIQUE, " "format TEXT NOT NULL, " "tile BLOB NULL, " "size INTEGER, " "type INTEGER, " "date INTEGER DEFAULT 0)")) { qWarning() << "Map Cache SQL error (create Tiles db):" << query.lastError().text(); } else { if(!query.exec( "CREATE TABLE IF NOT EXISTS TileSets (" "setID INTEGER PRIMARY KEY NOT NULL, " "name TEXT NOT NULL UNIQUE, " "typeStr TEXT, " "topleftLat REAL DEFAULT 0.0, " "topleftLon REAL DEFAULT 0.0, " "bottomRightLat REAL DEFAULT 0.0, " "bottomRightLon REAL DEFAULT 0.0, " "minZoom INTEGER DEFAULT 3, " "maxZoom INTEGER DEFAULT 3, " "type INTEGER DEFAULT -1, " "numTiles INTEGER DEFAULT 0, " "defaultSet INTEGER DEFAULT 0, " "date INTEGER DEFAULT 0)")) { qWarning() << "Map Cache SQL error (create TileSets db):" << query.lastError().text(); } else { if(!query.exec( "CREATE TABLE IF NOT EXISTS SetTiles (" "setID INTEGER, " "tileID INTEGER)")) { qWarning() << "Map Cache SQL error (create SetTiles db):" << query.lastError().text(); } else { if(!query.exec( "CREATE TABLE IF NOT EXISTS TilesDownload (" "setID INTEGER, " "hash TEXT NOT NULL UNIQUE, " "type INTEGER, " "x INTEGER, " "y INTEGER, " "z INTEGER, " "state INTEGER DEFAULT 0)")) { qWarning() << "Map Cache SQL error (create TilesDownload db):" << query.lastError().text(); } else { //-- Database it ready for use _valid = true; } } } } //-- Create default tile set if(_valid) { QString s = QString("SELECT name FROM TileSets WHERE name = \"%1\"").arg(kDefaultSet); if(query.exec(s)) { if(!query.next()) { query.prepare("INSERT INTO TileSets(name, defaultSet, date) VALUES(?, ?, ?)"); query.addBindValue(kDefaultSet); query.addBindValue(1); query.addBindValue(QDateTime::currentDateTime().toTime_t()); if(!query.exec()) { qWarning() << "Map Cache SQL error (Creating default tile set):" << _db->lastError(); _valid = false; } } } else { qWarning() << "Map Cache SQL error (Looking for default tile set):" << _db->lastError(); } } if(!_valid) { QFile file(_databasePath); file.remove(); } _failed = !_valid; }