qt 6.5.1 original

tests/auto/corelib/io/largefile/CMakeLists.txt

@@ -0,0 +1,13 @@
+# Copyright (C) 2022 The Qt Company Ltd.
+# SPDX-License-Identifier: BSD-3-Clause
+
+#####################################################################
+## tst_largefile Test:
+#####################################################################
+
+qt_internal_add_test(tst_largefile
+    SOURCES
+        tst_largefile.cpp
+    LIBRARIES
+        Qt::TestPrivate
+)

tests/auto/corelib/io/largefile/tst_largefile.cpp

@@ -0,0 +1,530 @@
+// Copyright (C) 2016 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
+
+#include <QTest>
+
+#include <QFile>
+#include <QFileInfo>
+#include <QRandomGenerator>
+#include <qplatformdefs.h>
+
+#include <QDebug>
+
+#include <algorithm>
+#include <cstdlib>
+#include <cstdio>
+
+#ifdef Q_OS_WIN
+#  include <qt_windows.h>
+#  include <io.h>
+#  ifndef FSCTL_SET_SPARSE
+// MinGW doesn't define this.
+#    define FSCTL_SET_SPARSE (0x900C4)
+#  endif
+#endif // Q_OS_WIN
+
+#include <QtTest/private/qemulationdetector_p.h>
+
+class tst_LargeFile
+    : public QObject
+{
+    Q_OBJECT
+
+public:
+    tst_LargeFile()
+        : blockSize(1 << 12)
+        , maxSizeBits()
+        , fd_(-1)
+        , stream_(0)
+    {
+    #if defined(QT_LARGEFILE_SUPPORT) && !defined(Q_OS_MAC) && !defined(Q_OS_QNX)
+        maxSizeBits = 36; // 64 GiB
+    #elif defined(Q_OS_MAC)
+        // HFS+ does not support sparse files, so we limit file size for the test
+        // on Mac OS.
+        maxSizeBits = 24; // 16 MiB
+    #elif defined(Q_OS_QNX)
+        // Many of the filesystems that QNX supports use a 32-bit format.
+        // This means that files are limited to 2 GB − 1 bytes.
+        // Limit max size to 256MB
+        maxSizeBits = 28; // 256 MiB
+    #else
+        maxSizeBits = 24; // 16 MiB
+    #endif
+
+        // QEMU only supports < 4GB files
+        if (QTestPrivate::isRunningArmOnX86())
+            maxSizeBits = qMin(maxSizeBits, 28);
+    }
+
+private:
+    void sparseFileData();
+    QByteArray const &getDataBlock(int index, qint64 position);
+
+private slots:
+    // The LargeFile test case was designed to be run in order as a single unit
+
+    void initTestCase();
+    void cleanupTestCase();
+
+    void init();
+    void cleanup();
+
+    // Create and fill large file
+    void createSparseFile();
+    void fillFileSparsely();
+    void closeSparseFile();
+
+    // Verify file was created
+    void fileCreated();
+
+    // Positioning in large files
+    void filePositioning();
+    void fdPositioning();
+    void streamPositioning();
+
+    // Read data from file
+    void openFileForReading();
+    void readFile();
+
+    // Map/unmap large file
+    void mapFile();
+    void mapOffsetOverflow();
+
+    void closeFile() { largeFile.close(); }
+
+    // Test data
+    void fillFileSparsely_data() { sparseFileData(); }
+    void filePositioning_data() { sparseFileData(); }
+    void fdPositioning_data() { sparseFileData(); }
+    void streamPositioning_data() { sparseFileData(); }
+    void readFile_data() { sparseFileData(); }
+    void mapFile_data() { sparseFileData(); }
+
+private:
+    const int blockSize;
+    int maxSizeBits;
+
+    QFile largeFile;
+
+    QByteArrayList generatedBlocks;
+
+    int fd_;
+    FILE *stream_;
+
+    QSharedPointer<QTemporaryDir> m_tempDir;
+    QString m_previousCurrent;
+};
+
+/*
+    Convenience function to hide reinterpret_cast when copying a POD directly
+    into a QByteArray.
+ */
+template <class T>
+static inline void appendRaw(QByteArray &array, T data)
+{
+    array.append(reinterpret_cast<char *>(&data), sizeof(T));
+}
+
+/*
+    Pad array with filler up to size. On return, array.size() returns size.
+ */
+static inline void topUpWith(QByteArray &array, QByteArray filler, int size)
+{
+    for (int i = (size - array.size()) / filler.size(); i > 0; --i)
+        array.append(filler);
+
+    if (array.size() < size) {
+        array.append(filler.left(size - array.size()));
+    }
+}
+
+/*
+   Generate a unique data block containing identifiable data. Unaligned,
+   overlapping and partial blocks should not compare equal.
+ */
+static inline QByteArray generateDataBlock(int blockSize, QString text, qint64 userBits = -1)
+{
+    QByteArray block;
+    block.reserve(blockSize);
+
+    // Use of counter and randomBits means content of block will be dependent
+    // on the generation order. For (file-)systems that do not support sparse
+    // files, these can be removed so the test file can be reused and doesn't
+    // have to be generated for every run.
+
+    static qint64 counter = 0;
+
+    qint64 randomBits = QRandomGenerator::global()->generate64();
+
+    appendRaw(block, randomBits);
+    appendRaw(block, userBits);
+    appendRaw(block, counter);
+    appendRaw(block, (qint32)0xdeadbeef);
+    appendRaw(block, blockSize);
+
+    QByteArray userContent = text.toUtf8();
+    appendRaw(block, userContent.size());
+    block.append(userContent);
+    appendRaw(block, (qint64)0);
+
+    // size, so far
+    appendRaw(block, block.size());
+
+    QByteArray filler("0123456789");
+    block.append(filler.right(10 - block.size() % 10));
+    topUpWith(block, filler, blockSize - 3 * sizeof(qint64));
+
+    appendRaw(block, counter);
+    appendRaw(block, userBits);
+    appendRaw(block, randomBits);
+
+    ++counter;
+    return block;
+}
+
+/*
+   Generates data blocks the first time they are requested. Keeps copies for reuse.
+ */
+QByteArray const &tst_LargeFile::getDataBlock(int index, qint64 position)
+{
+    if (index >= generatedBlocks.size())
+        generatedBlocks.resize(index + 1);
+
+    if (generatedBlocks[index].isNull()) {
+        QString text = QString("Current %1-byte block (index = %2) "
+            "starts %3 bytes into the file '%4'.")
+                .arg(blockSize)
+                .arg(index)
+                .arg(position)
+                .arg("qt_largefile.tmp");
+
+        generatedBlocks[index] = generateDataBlock(blockSize, text, (qint64)1 << index);
+    }
+
+    return generatedBlocks[index];
+}
+
+void tst_LargeFile::initTestCase()
+{
+    m_previousCurrent = QDir::currentPath();
+    m_tempDir = QSharedPointer<QTemporaryDir>::create();
+    QVERIFY2(!m_tempDir.isNull(), qPrintable("Could not create temporary directory."));
+    QVERIFY2(QDir::setCurrent(m_tempDir->path()), qPrintable("Could not switch current directory"));
+
+    QFile file("qt_largefile.tmp");
+    QVERIFY( !file.exists() || file.remove() );
+}
+
+void tst_LargeFile::cleanupTestCase()
+{
+    if (largeFile.isOpen())
+        largeFile.close();
+
+    QFile file("qt_largefile.tmp");
+    QVERIFY( !file.exists() || file.remove() );
+
+    QDir::setCurrent(m_previousCurrent);
+}
+
+void tst_LargeFile::init()
+{
+    fd_ = -1;
+    stream_ = 0;
+}
+
+void tst_LargeFile::cleanup()
+{
+    if (-1 != fd_)
+        QT_CLOSE(fd_);
+    if (stream_)
+        ::fclose(stream_);
+}
+
+void tst_LargeFile::sparseFileData()
+{
+    QTest::addColumn<int>("index");
+    QTest::addColumn<qint64>("position");
+    QTest::addColumn<QByteArray>("block");
+
+    QTest::newRow(QString("block[%1] @%2)")
+            .arg(0).arg(0)
+            .toLocal8Bit().constData())
+        << 0 << (qint64)0 << getDataBlock(0, 0);
+
+    // While on Linux sparse files scale well, on Windows, testing at every
+    // power of 2 leads to very large files. i += 4 gives us a good coverage
+    // without taxing too much on resources.
+    for (int index = 12; index <= maxSizeBits; index += 4) {
+        qint64 position = (qint64)1 << index;
+        QByteArray block = getDataBlock(index, position);
+
+        QTest::newRow(
+            QString("block[%1] @%2)")
+                .arg(index).arg(position)
+                .toLocal8Bit().constData())
+            << index << position << block;
+    }
+}
+
+void tst_LargeFile::createSparseFile()
+{
+#if defined(Q_OS_WIN32)
+    // On Windows platforms, we must explicitly set the file to be sparse,
+    // so disk space is not allocated for the full file when writing to it.
+    HANDLE handle = ::CreateFileA("qt_largefile.tmp",
+        GENERIC_WRITE, 0, 0, CREATE_ALWAYS, 0, 0);
+    QVERIFY( INVALID_HANDLE_VALUE != handle );
+
+    DWORD bytes;
+    if (!::DeviceIoControl(handle, FSCTL_SET_SPARSE, NULL, 0, NULL, 0,
+            &bytes, NULL)) {
+        qWarning("Unable to set test file as sparse. "
+            "Limiting test file to 16MiB.");
+        maxSizeBits = 24;
+    }
+
+    int fd = ::_open_osfhandle((intptr_t)handle, 0);
+    QVERIFY( -1 != fd );
+    QVERIFY( largeFile.open(fd, QIODevice::WriteOnly | QIODevice::Unbuffered) );
+#else // !Q_OS_WIN32
+    largeFile.setFileName("qt_largefile.tmp");
+    QVERIFY( largeFile.open(QIODevice::WriteOnly | QIODevice::Unbuffered) );
+#endif
+}
+
+void tst_LargeFile::closeSparseFile()
+{
+#if defined(Q_OS_WIN32)
+    int fd = largeFile.handle();
+#endif
+
+    largeFile.close();
+
+#if defined(Q_OS_WIN32)
+    if (-1 != fd)
+        ::_close(fd);
+#endif
+}
+
+void tst_LargeFile::fillFileSparsely()
+{
+    QFETCH( qint64, position );
+    QFETCH( QByteArray, block );
+    QCOMPARE( block.size(), blockSize );
+
+    static int lastKnownGoodIndex = 0;
+    struct ScopeGuard {
+        ScopeGuard(tst_LargeFile* test)
+            : this_(test)
+            , failed(true)
+        {
+            QFETCH( int, index );
+            index_ = index;
+        }
+
+        ~ScopeGuard()
+        {
+            if (failed) {
+                this_->maxSizeBits = lastKnownGoodIndex;
+                qWarning("QFile::error %d: '%s'. Maximum size bits reset to %d.",
+                        this_->largeFile.error(),
+                        qPrintable(this_->largeFile.errorString()),
+                        this_->maxSizeBits);
+            } else
+                lastKnownGoodIndex = qMax<int>(index_, lastKnownGoodIndex);
+        }
+
+    private:
+        tst_LargeFile * const this_;
+        int index_;
+
+    public:
+        bool failed;
+    };
+
+    ScopeGuard resetMaxSizeBitsOnFailure(this);
+
+    QVERIFY( largeFile.seek(position) );
+    QCOMPARE( largeFile.pos(), position );
+
+    QCOMPARE( largeFile.write(block), (qint64)blockSize );
+    QCOMPARE( largeFile.pos(), position + blockSize );
+    QVERIFY( largeFile.flush() );
+
+    resetMaxSizeBitsOnFailure.failed = false;
+}
+
+void tst_LargeFile::fileCreated()
+{
+    QFileInfo info("qt_largefile.tmp");
+
+    QVERIFY( info.exists() );
+    QVERIFY( info.isFile() );
+    QVERIFY( info.size() >= ((qint64)1 << maxSizeBits) + blockSize );
+}
+
+void tst_LargeFile::filePositioning()
+{
+    QFETCH( qint64, position );
+
+    QFile file("qt_largefile.tmp");
+    QVERIFY( file.open(QIODevice::ReadOnly) );
+
+    QVERIFY( file.seek(position) );
+    QCOMPARE( file.pos(), position );
+}
+
+void tst_LargeFile::fdPositioning()
+{
+    QFETCH( qint64, position );
+
+    fd_ = QT_OPEN("qt_largefile.tmp",
+            QT_OPEN_RDONLY | QT_OPEN_LARGEFILE);
+    QVERIFY( -1 != fd_ );
+
+    QFile file;
+    QVERIFY( file.open(fd_, QIODevice::ReadOnly) );
+    QCOMPARE( file.pos(), (qint64)0 );
+    QVERIFY( file.seek(position) );
+    QCOMPARE( file.pos(), position );
+
+    file.close();
+
+    QCOMPARE( QT_OFF_T(QT_LSEEK(fd_, QT_OFF_T(0), SEEK_SET)), QT_OFF_T(0) );
+    QCOMPARE( QT_OFF_T(QT_LSEEK(fd_, QT_OFF_T(position), SEEK_SET)), QT_OFF_T(position) );
+
+    QVERIFY( file.open(fd_, QIODevice::ReadOnly) );
+    QCOMPARE( QT_OFF_T(QT_LSEEK(fd_, QT_OFF_T(0), SEEK_CUR)), QT_OFF_T(position) );
+    QCOMPARE( file.pos(), position );
+    QVERIFY( file.seek(0) );
+    QCOMPARE( file.pos(), (qint64)0 );
+
+    file.close();
+
+    QVERIFY( !QT_CLOSE(fd_) );
+    fd_ = -1;
+}
+
+void tst_LargeFile::streamPositioning()
+{
+    QFETCH( qint64, position );
+
+    stream_ = QT_FOPEN("qt_largefile.tmp", "rb");
+    QVERIFY( 0 != stream_ );
+
+    QFile file;
+    QVERIFY( file.open(stream_, QIODevice::ReadOnly) );
+    QCOMPARE( file.pos(), (qint64)0 );
+    QVERIFY( file.seek(position) );
+    QCOMPARE( file.pos(), position );
+
+    file.close();
+
+    QVERIFY( !QT_FSEEK(stream_, QT_OFF_T(0), SEEK_SET) );
+    QCOMPARE( QT_OFF_T(QT_FTELL(stream_)), QT_OFF_T(0) );
+    QVERIFY( !QT_FSEEK(stream_, QT_OFF_T(position), SEEK_SET) );
+    QCOMPARE( QT_OFF_T(QT_FTELL(stream_)), QT_OFF_T(position) );
+
+    QVERIFY( file.open(stream_, QIODevice::ReadOnly) );
+    QCOMPARE( QT_OFF_T(QT_FTELL(stream_)), QT_OFF_T(position) );
+    QCOMPARE( file.pos(), position );
+    QVERIFY( file.seek(0) );
+    QCOMPARE( file.pos(), (qint64)0 );
+
+    file.close();
+
+    QVERIFY( !::fclose(stream_) );
+    stream_ = 0;
+}
+
+void tst_LargeFile::openFileForReading()
+{
+    largeFile.setFileName("qt_largefile.tmp");
+    QVERIFY( largeFile.open(QIODevice::ReadOnly) );
+}
+
+void tst_LargeFile::readFile()
+{
+    QFETCH( qint64, position );
+    QFETCH( QByteArray, block );
+    QCOMPARE( block.size(), blockSize );
+
+    QVERIFY( largeFile.size() >= position + blockSize );
+
+    QVERIFY( largeFile.seek(position) );
+    QCOMPARE( largeFile.pos(), position );
+
+    QCOMPARE( largeFile.read(blockSize), block );
+    QCOMPARE( largeFile.pos(), position + blockSize );
+}
+
+void tst_LargeFile::mapFile()
+{
+    QFETCH( qint64, position );
+    QFETCH( QByteArray, block );
+    QCOMPARE( block.size(), blockSize );
+
+    // Keep full block mapped to facilitate OS and/or internal reuse by Qt.
+    uchar *baseAddress = largeFile.map(position, blockSize);
+    QVERIFY( baseAddress );
+    QVERIFY( std::equal(block.begin(), block.end(), reinterpret_cast<char*>(baseAddress)) );
+
+    for (int offset = 1; offset <  blockSize; ++offset) {
+        uchar *address = largeFile.map(position + offset, blockSize - offset);
+
+        QVERIFY( address );
+        if ( !std::equal(block.begin() + offset, block.end(), reinterpret_cast<char*>(address)) ) {
+            qDebug() << "Expected:" << block.toHex();
+            qDebug() << "Actual  :" << QByteArray(reinterpret_cast<char*>(address), blockSize).toHex();
+            QVERIFY(false);
+        }
+
+        QVERIFY( largeFile.unmap( address ) );
+    }
+
+    QVERIFY( largeFile.unmap( baseAddress ) );
+}
+
+//Mac: memory-mapping beyond EOF may succeed but it could generate bus error on access
+//FreeBSD: same
+//Linux: memory-mapping beyond EOF usually succeeds, but depends on the filesystem
+//  32-bit: limited to 44-bit offsets (when sizeof(off_t) == 8)
+//Windows: memory-mapping beyond EOF is not allowed
+void tst_LargeFile::mapOffsetOverflow()
+{
+    enum {
+#ifdef Q_OS_WIN
+        Succeeds = false,
+        MaxOffset = 63
+#else
+        Succeeds = true,
+#  if (defined(Q_OS_LINUX) || defined(Q_OS_ANDROID)) && Q_PROCESSOR_WORDSIZE == 4
+        MaxOffset = sizeof(QT_OFF_T) > 4 ? 43 : 30
+#  else
+        MaxOffset = 8 * sizeof(QT_OFF_T) - 1
+#  endif
+#endif
+    };
+
+    QByteArray zeroPage(blockSize, '\0');
+    for (int i = maxSizeBits + 1; i < 63; ++i) {
+        bool succeeds = Succeeds && (i <= MaxOffset);
+        uchar *address = 0;
+        qint64 offset = Q_INT64_C(1) << i;
+
+        if (succeeds)
+            QTest::ignoreMessage(QtWarningMsg, "QFSFileEngine::map: Mapping a file beyond its size is not portable");
+        address = largeFile.map(offset, blockSize);
+        QCOMPARE(!!address, succeeds);
+
+        if (succeeds)
+            QTest::ignoreMessage(QtWarningMsg, "QFSFileEngine::map: Mapping a file beyond its size is not portable");
+        address = largeFile.map(offset + blockSize, blockSize);
+        QCOMPARE(!!address, succeeds);
+    }
+}
+
+QTEST_APPLESS_MAIN(tst_LargeFile)
+#include "tst_largefile.moc"
+