From ded0f9c6f4b8dcbe85933865ea6ad46d2047c2ba Mon Sep 17 00:00:00 2001 From: Jonathan Young Date: Sun, 18 May 2014 21:41:54 +1000 Subject: [PATCH] Moved tar.gz file extraction to FileExtractWorker. The "patch" wizard page runs it in a background thread instead of the main thread. --- filecopy.cpp | 26 +++++ filecopy.h | 2 + fileextract.cpp | 241 +++++++++++++++++++++++++++++++++++++++ fileextract.h | 58 ++++++++++ installwizard_copy.cpp | 25 +--- installwizard_patch.cpp | 246 +++++++++++++--------------------------- installwizard_patch.h | 19 +++- launch.pro | 6 +- 8 files changed, 429 insertions(+), 194 deletions(-) create mode 100644 fileextract.cpp create mode 100644 fileextract.h diff --git a/filecopy.cpp b/filecopy.cpp index 7925a59..441794c 100644 --- a/filecopy.cpp +++ b/filecopy.cpp @@ -41,6 +41,32 @@ QString FileUtils::uniqueFilename(const QString &filename) return unique; } +QString FileUtils::completeTransaction(const QList &renameOperations) +{ + for (int i = 0; i < renameOperations.size(); i++) + { + const FileOperation &fo = renameOperations.at(i); + const QString randomDest = FileUtils::uniqueFilename(fo.dest); + + // Rename dest to random. + if (!QFile::rename(fo.dest, randomDest)) + { + return QString("Transaction error renaming '%1' to '%2'").arg(fo.dest).arg(randomDest); + } + + // Rename source to dest. + if (!QFile::rename(fo.source, fo.dest)) + { + return QString("Transaction error renaming '%1' to '%2'").arg(fo.source).arg(fo.dest); + } + + // Delete random. + QFile::remove(randomDest); + } + + return QString(); +} + FileCopyWorker::FileCopyWorker(const QList &files) : files(files), isCancelled(false) { } diff --git a/filecopy.h b/filecopy.h index 5846d1e..5ef6975 100644 --- a/filecopy.h +++ b/filecopy.h @@ -37,6 +37,8 @@ class FileUtils { public: static QString uniqueFilename(const QString &filename); + + static QString completeTransaction(const QList &renameOperations); }; /* diff --git a/fileextract.cpp b/fileextract.cpp new file mode 100644 index 0000000..f3ac345 --- /dev/null +++ b/fileextract.cpp @@ -0,0 +1,241 @@ +/* +The MIT License (MIT) + +Copyright (c) 2013 The ioquake Group + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ +#include +#include +#include +#include +#include +#include "fileextract.h" + +struct TarHeader +{ + char name[100]; + char mode[8]; + char uid[8]; + char gid[8]; + char size[12]; + char mtime[12]; + char chksum[8]; + char typeflag; + char linkname[100]; + char magic[6]; + char version[2]; + char uname[32]; + char gname[32]; + char devmajor[8]; + char devminor[8]; + char prefix[155]; +}; + +FileExtractWorker::FileExtractWorker(const QString &archiveFilename, const QList &filesToExtract) : archiveFilename(archiveFilename), filesToExtract(filesToExtract), isCancelled(false) +{ +} + +void FileExtractWorker::extract() +{ + qsrand(time(NULL)); + + // Extract gzip compressed archive. + z_stream zstream; + zstream.zalloc = Z_NULL; + zstream.zfree = Z_NULL; + zstream.opaque = Z_NULL; + zstream.avail_in = 0; + zstream.next_in = Z_NULL; + + int result = inflateInit2(&zstream, 32 | MAX_WBITS); + + if (result != Z_OK) + { + emit errorMessage("zlib inflateInit2 failed"); + return; + } + + QFile archiveFile(archiveFilename); + + if (!archiveFile.open(QIODevice::ReadOnly)) + { + emit errorMessage(QString("Error opening '%1': '%2'").arg(archiveFile.fileName()).arg(archiveFile.errorString())); + return; + } + + tarFile.open(); + + const int bufferSize = 32 * 1024; + static char inputBuffer[bufferSize]; + static char outputBuffer[bufferSize]; + + do + { + qint64 bytesRead = archiveFile.read(inputBuffer, bufferSize); + + if (bytesRead == 0) + break; + + zstream.avail_in = bytesRead; + zstream.next_in = (z_Bytef *)inputBuffer; + + do + { + zstream.avail_out = bufferSize; + zstream.next_out = (z_Bytef *)outputBuffer; + + result = inflate(&zstream, Z_NO_FLUSH); + + if (result != Z_OK && result != Z_STREAM_END) + { + emit errorMessage(QString("zlib error %1").arg(result)); + inflateEnd(&zstream); + return; + } + + tarFile.write(outputBuffer, bufferSize - zstream.avail_out); + } + while (zstream.avail_out == 0); + } + while (result != Z_STREAM_END); + + archiveFile.close(); + inflateEnd(&zstream); + + // Extract the files in the TAR file. + tarFile.seek(0); + + for (;;) + { + // Read TAR file header (padded to 512 bytes). + TarHeader header; + + if (tarFile.read((char *)&header, sizeof(header)) != sizeof(header)) + break; // EOF + + tarFile.seek(tarFile.pos() + 512 - sizeof(header)); + + // Convert size from octal string. + qint64 size = 0; + int exponent = 0; + + for (int i = sizeof(header.size) - 1; i >= 0; i--) + { + int digit = (int)(header.size[i] - '0'); + + if (digit < 0 || digit > 7) + continue; + + size += qint64(digit * pow(8.0f, exponent)); + exponent++; + } + + // See if this is one of the files to be extracted. + int extractFileIndex = -1; + + for (int i = 0; i < filesToExtract.size(); i++) + { + if (filesToExtract.at(i).source == header.name) + { + extractFileIndex = i; + break; + } + } + + // Extract a file. + if (extractFileIndex != -1) + { + // Write to the destination file if it doesn't exist, otherwise write to a uniquely named file. + QString filename(filesToExtract.at(extractFileIndex).dest); + const bool fileExists = QFile::exists(filename); + + if (fileExists) + { + filename = FileUtils::uniqueFilename(filesToExtract.at(extractFileIndex).dest); + } + + QFile file(filename); + + if (!file.open(QIODevice::WriteOnly)) + { + emit errorMessage(QString("'%1': %2").arg(file.fileName()).arg(file.errorString())); + goto cleanup; + } + + // Don't add a rename operation if the destination file doesn't exist - not trying to overwrite, don't need to rename anything to complete the transaction. + if (fileExists) + { + FileOperation fo; + fo.source = filename; + fo.dest = filesToExtract.at(extractFileIndex).dest; + renameOperations.append(fo); + } + + emit fileChanged(QFileInfo(file.fileName()).fileName()); + qint64 totalBytesRead = 0; + + for (;;) + { + qint64 bytesToRead = bufferSize; + + if (totalBytesRead + bytesToRead > size) + { + bytesToRead = size - totalBytesRead; + } + + const qint64 bytesRead = tarFile.read(outputBuffer, bytesToRead); + + if (bytesRead == 0) + break; + + file.write(outputBuffer, bytesRead); + totalBytesRead += bytesRead; + emit progressChanged(totalBytesRead, size); + + QMutexLocker locker(&cancelMutex); + + if (isCancelled) + goto cleanup; + } + } + else + { + tarFile.seek(tarFile.pos() + size); + } + + // TAR file size is padded to 512 byte blocks. + tarFile.seek(tarFile.pos() + ((size % 512) == 0 ? 0 : 512 - (size % 512))); + } + + emit finished(renameOperations); + return; + + // Delete all the destination files. +cleanup: + for (int i = 0; i < renameOperations.size(); i++) + { + QFile::remove(renameOperations.at(i).source); + } +} + +void FileExtractWorker::cancel() +{ + QMutexLocker locker(&cancelMutex); + isCancelled = true; +} diff --git a/fileextract.h b/fileextract.h new file mode 100644 index 0000000..9f765a8 --- /dev/null +++ b/fileextract.h @@ -0,0 +1,58 @@ +/* +The MIT License (MIT) + +Copyright (c) 2013 The ioquake Group + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ +#ifndef FILEEXTRACT_H +#define FILEEXTRACT_H + +#include +#include +#include +#include +#include "filecopy.h" + +class FileExtractWorker : public QObject +{ + Q_OBJECT + +public: + FileExtractWorker(const QString &archiveFilename, const QList &filesToExtract); + void cancel(); + +public slots: + void extract(); + +signals: + void fileChanged(const QString &filename); + void progressChanged(qint64 bytesWritten, qint64 bytesTotal); + void errorMessage(const QString &message); + void finished(QList renameOperations); + +private: + QString archiveFilename; + const QList filesToExtract; + QTemporaryFile tarFile; + bool isCancelled; + QMutex cancelMutex; + QList renameOperations; +}; + +#endif // FILEEXTRACT_H diff --git a/installwizard_copy.cpp b/installwizard_copy.cpp index c3863f3..8017224 100644 --- a/installwizard_copy.cpp +++ b/installwizard_copy.cpp @@ -119,27 +119,12 @@ void InstallWizard_Copy::finishCopy(QList renameOperations) copyThread.wait(); // Complete the transaction. - for (int i = 0; i < renameOperations.size(); i++) + const QString transactionError = FileUtils::completeTransaction(renameOperations); + + if (!transactionError.isEmpty()) { - const FileOperation &fo = renameOperations.at(i); - const QString randomDest = FileUtils::uniqueFilename(fo.dest); - - // Rename dest to random. - if (!QFile::rename(fo.dest, randomDest)) - { - ui->lblStatus->setText(QString("Transaction error renaming '%1' to '%2'").arg(fo.dest).arg(randomDest)); - return; - } - - // Rename source to dest. - if (!QFile::rename(fo.source, fo.dest)) - { - ui->lblStatus->setText(QString("Transaction error renaming '%1' to '%2'").arg(fo.source).arg(fo.dest)); - return; - } - - // Delete random. - QFile::remove(randomDest); + ui->lblStatus->setText(transactionError); + return; } isCopyFinished = true; diff --git a/installwizard_patch.cpp b/installwizard_patch.cpp index 37991c8..42a3b28 100644 --- a/installwizard_patch.cpp +++ b/installwizard_patch.cpp @@ -20,67 +20,49 @@ COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -#include #include #include -#include #include "installwizard_patch.h" #include "ui_installwizard_patch.h" #include "installwizard.h" - -struct TarHeader -{ - char name[100]; - char mode[8]; - char uid[8]; - char gid[8]; - char size[12]; - char mtime[12]; - char chksum[8]; - char typeflag; - char linkname[100]; - char magic[6]; - char version[2]; - char uname[32]; - char gname[32]; - char devmajor[8]; - char devminor[8]; - char prefix[155]; -}; +#include "fileextract.h" InstallWizard_Patch::InstallWizard_Patch(QWidget *parent) : QWizardPage(parent), ui(new Ui::InstallWizard_Patch), patchFile(NULL), - unzippedPatchFile(NULL), networkReply(NULL), isCancelled(false), isDownloadFinished(false), isPatchInstalled(false), - usePatchFileBuffer(true) + usePatchFileBuffer(true), + extractWorker(NULL), + isExtractFinished(false) { ui->setupUi(this); } InstallWizard_Patch::~InstallWizard_Patch() { + extractThread.quit(); + extractThread.wait(); delete patchFile; - delete unzippedPatchFile; delete ui; } void InstallWizard_Patch::initializePage() { - isCancelled = isDownloadFinished = isPatchInstalled = false; + isCancelled = isDownloadFinished = isPatchInstalled = isExtractFinished = false; patchFileBuffer.clear(); usePatchFileBuffer = true; + extractWorker = NULL; patchFile = new QTemporaryFile; patchFile->open(); networkReply = nam.get(QNetworkRequest(QUrl("http://localhost:8080/linuxq3apoint-1.32b-3.x86.run"))); connect(networkReply, &QNetworkReply::readyRead, this, &InstallWizard_Patch::downloadRead); - connect(networkReply, &QNetworkReply::downloadProgress, this, &InstallWizard_Patch::downloadProgress); + connect(networkReply, &QNetworkReply::downloadProgress, this, &InstallWizard_Patch::updateProgress); connect(networkReply, &QNetworkReply::finished, this, &InstallWizard_Patch::downloadFinished); } @@ -91,7 +73,7 @@ void InstallWizard_Patch::cleanupPage() bool InstallWizard_Patch::isComplete() const { - return isDownloadFinished && isPatchInstalled; + return isDownloadFinished && isExtractFinished && isPatchInstalled; } void InstallWizard_Patch::cancel() @@ -101,12 +83,15 @@ void InstallWizard_Patch::cancel() delete patchFile; patchFile = NULL; - delete unzippedPatchFile; - unzippedPatchFile = NULL; if (!isDownloadFinished) networkReply->abort(); + if (!isExtractFinished) + { + extractWorker->cancel(); + } + isCancelled = true; } @@ -132,13 +117,6 @@ void InstallWizard_Patch::downloadRead() } } -void InstallWizard_Patch::downloadProgress(qint64 bytesRead, qint64 bytesTotal) -{ - ui->lblStatus->setText(QString("Downloading %1MB / %2MB").arg(bytesRead / 1024.0 / 1024.0, 0, 'f', 2).arg(bytesTotal / 1024.0 / 1024.0, 0, 'f', 2)); - ui->pbProgress->setMaximum((int)bytesTotal); - ui->pbProgress->setValue((int)bytesRead); -} - void InstallWizard_Patch::downloadFinished() { Q_ASSERT(networkReply); @@ -159,144 +137,72 @@ void InstallWizard_Patch::downloadFinished() isDownloadFinished = true; - // Extract gzip compressed archive. - ui->lblStatus->setText("Installing..."); + // Build a list of pak files to extract. + QList filesToExtract; - z_stream zstream; - zstream.zalloc = Z_NULL; - zstream.zfree = Z_NULL; - zstream.opaque = Z_NULL; - zstream.avail_in = 0; - zstream.next_in = Z_NULL; - - int result = inflateInit2(&zstream, 32 | MAX_WBITS); - - if (result != Z_OK) + for (int i = 1; i <= 8; i++) { - ui->lblStatus->setText("zlib inflateInit2 failed"); - cancel(); + FileOperation fo; + fo.source = QString("baseq3/pak%1.pk3").arg(i); + fo.dest = QString("%1/baseq3/pak%2.pk3").arg(((InstallWizard *)wizard())->getQuakePath()).arg(i); + filesToExtract.append(fo); + } + + // Start extract thread. + qRegisterMetaType >("QList"); + extractWorker = new FileExtractWorker(patchFile->fileName(), filesToExtract); + extractWorker->moveToThread(&extractThread); + connect(&extractThread, &QThread::finished, extractWorker, &QObject::deleteLater); + connect(this, &InstallWizard_Patch::extract, extractWorker, &FileExtractWorker::extract); + connect(extractWorker, &FileExtractWorker::fileChanged, this, &InstallWizard_Patch::setExtractFilename); + connect(extractWorker, &FileExtractWorker::progressChanged, this, &InstallWizard_Patch::updateProgress); + connect(extractWorker, &FileExtractWorker::errorMessage, this, &InstallWizard_Patch::setErrorMessage); + connect(extractWorker, &FileExtractWorker::finished, this, &InstallWizard_Patch::finishExtract); + extractThread.start(); + emit extract(); +} + +void InstallWizard_Patch::updateProgress(qint64 bytesRead, qint64 bytesTotal) +{ + if (!isDownloadFinished) + { + ui->lblStatus->setText(QString("Downloading %1MB / %2MB").arg(bytesRead / 1024.0 / 1024.0, 0, 'f', 2).arg(bytesTotal / 1024.0 / 1024.0, 0, 'f', 2)); + } + else if (!isExtractFinished) + { + ui->lblStatus->setText(QString("Extracting %1 %2MB / %3MB").arg(extractFilename).arg(bytesRead / 1024.0 / 1024.0, 0, 'f', 2).arg(bytesTotal / 1024.0 / 1024.0, 0, 'f', 2)); + } + + ui->pbProgress->setMaximum((int)bytesTotal); + ui->pbProgress->setValue((int)bytesRead); +} + +void InstallWizard_Patch::setExtractFilename(const QString &filename) +{ + extractFilename = filename; +} + +void InstallWizard_Patch::setErrorMessage(const QString &message) +{ + ui->lblStatus->setText(message); +} + +void InstallWizard_Patch::finishExtract(QList renameOperations) +{ + extractThread.quit(); + extractThread.wait(); + isExtractFinished = true; + emit completeChanged(); + + // Complete the transaction. + const QString transactionError = FileUtils::completeTransaction(renameOperations); + + if (!transactionError.isEmpty()) + { + ui->lblStatus->setText(transactionError); return; } - patchFile->seek(0); - unzippedPatchFile = new QTemporaryFile; - unzippedPatchFile->open(); - - const int bufferSize = 32 * 1024; - static char inputBuffer[bufferSize]; - static char outputBuffer[bufferSize]; - - do - { - qint64 bytesRead = patchFile->read(inputBuffer, bufferSize); - - if (bytesRead == 0) - break; - - zstream.avail_in = bytesRead; - zstream.next_in = (z_Bytef *)inputBuffer; - - do - { - zstream.avail_out = bufferSize; - zstream.next_out = (z_Bytef *)outputBuffer; - - result = inflate(&zstream, Z_NO_FLUSH); - - if (result != Z_OK && result != Z_STREAM_END) - { - ui->lblStatus->setText(QString("zlib error %1").arg(result)); - inflateEnd(&zstream); - cancel(); - return; - } - - unzippedPatchFile->write(outputBuffer, bufferSize - zstream.avail_out); - } - while (zstream.avail_out == 0); - } - while (result != Z_STREAM_END); - - inflateEnd(&zstream); - - // Extract all the baseq3 pk3 files in the TAR file. - unzippedPatchFile->seek(0); - int nPaksExtracted = 0; - - for (;;) - { - // Read TAR file header (padded to 512 bytes). - TarHeader header; - unzippedPatchFile->read((char *)&header, sizeof(header)); - unzippedPatchFile->seek(unzippedPatchFile->pos() + 512 - sizeof(header)); - - // Convert size from octal string. - qint64 size = 0; - int exponent = 0; - - for (int i = sizeof(header.size) - 1; i >= 0; i--) - { - int digit = (int)(header.size[i] - '0'); - - if (digit < 0 || digit > 7) - continue; - - size += qint64(digit * pow(8.0f, exponent)); - exponent++; - } - - // Extract a pk3 file. - const char *pakPrefix = "baseq3/pak"; - - if (strncmp(header.name, pakPrefix, strlen(pakPrefix)) == 0) - { - QString filename(((InstallWizard *)wizard())->getQuakePath()); - filename.append('/'); - filename.append(header.name); - - QFile file(filename); - - if (!file.open(QIODevice::WriteOnly)) - { - ui->lblStatus->setText(QString("Error opening '%1' for writing").arg(filename)); - cancel(); - return; - } - - qint64 totalBytesRead = 0; - - for (;;) - { - qint64 bytesToRead = bufferSize; - - if (totalBytesRead + bytesToRead > size) - { - bytesToRead = size - totalBytesRead; - } - - const qint64 bytesRead = unzippedPatchFile->read(outputBuffer, bytesToRead); - - if (bytesRead == 0) - break; - - file.write(outputBuffer, bytesRead); - totalBytesRead += bytesRead; - } - - nPaksExtracted++; - } - else - { - unzippedPatchFile->seek(unzippedPatchFile->pos() + size); - } - - // TAR file size is padded to 512 byte blocks. - unzippedPatchFile->seek(unzippedPatchFile->pos() + ((size % 512) == 0 ? 0 : 512 - (size % 512))); - - if (nPaksExtracted == 8) - break; - } - isPatchInstalled = true; emit completeChanged(); wizard()->next(); diff --git a/installwizard_patch.h b/installwizard_patch.h index f0c86ad..d69b419 100644 --- a/installwizard_patch.h +++ b/installwizard_patch.h @@ -25,12 +25,16 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. #include #include +#include #include +#include "filecopy.h" namespace Ui { class InstallWizard_Patch; } +class FileExtractWorker; + class InstallWizard_Patch : public QWizardPage { Q_OBJECT @@ -45,12 +49,18 @@ public: private slots: void downloadRead(); - void downloadProgress(qint64 bytesRead, qint64 bytesTotal); void downloadFinished(); + void updateProgress(qint64 bytesRead, qint64 bytesTotal); + void setExtractFilename(const QString &filename); + void setErrorMessage(const QString &message); + void finishExtract(QList renameOperations); + +signals: + void extract(); private: Ui::InstallWizard_Patch *ui; - QTemporaryFile *patchFile, *unzippedPatchFile; + QTemporaryFile *patchFile; QNetworkAccessManager nam; QNetworkReply *networkReply; bool isCancelled; @@ -58,6 +68,11 @@ private: bool isPatchInstalled; bool usePatchFileBuffer; QByteArray patchFileBuffer; + + QString extractFilename; + FileExtractWorker *extractWorker; + QThread extractThread; + bool isExtractFinished; }; #endif // INSTALLWIZARD_PATCH_H diff --git a/launch.pro b/launch.pro index 5952940..f6227d3 100644 --- a/launch.pro +++ b/launch.pro @@ -22,7 +22,8 @@ SOURCES += main.cpp\ installwizard_eula.cpp \ installwizard_copy.cpp \ filecopy.cpp \ - quakeutils.cpp + quakeutils.cpp \ + fileextract.cpp HEADERS += mainwindow.h \ settings.h \ @@ -33,7 +34,8 @@ HEADERS += mainwindow.h \ installwizard_eula.h \ installwizard_copy.h \ filecopy.h \ - quakeutils.h + quakeutils.h \ + fileextract.h FORMS += mainwindow.ui \ installwizard.ui \