Changeset 794

Timestamp:

Oct 14, 2010, 6:12:21 PM (15 years ago)

Author:

Dmitry A. Kuminov

Message:

Significantly improve the performance of directory iterators walking through directories with thousands of files on slow file systems with inefficient cache policies (closes #176) [based on patch by rudi].

Location:

trunk/src/corelib/io

Files:

• qfsfileengine_iterator.cpp (modified) (1 diff)
• qfsfileengine_iterator_os2.cpp (modified) (9 diffs)
• qfsfileengine_os2.cpp (modified) (2 diffs)

trunk/src/corelib/io/qfsfileengine_iterator.cpp

@@ -72 +72 @@
 }
 
+#ifndef Q_OS_OS2
 QFileInfo QFSFileEngineIterator::currentFileInfo() const
 {
     return QAbstractFileEngineIterator::currentFileInfo();
 }
+#endif
 
 QT_END_NAMESPACE

trunk/src/corelib/io/qfsfileengine_iterator_os2.cpp

@@ -42 +42 @@
 ****************************************************************************/
 
+// temporary, until struct dirent in kLIBC gets creation and access time fields
+#define QT_OS2_USE_DOSFINDFIRST
+
 #include "qplatformdefs.h"
 #include "qfsfileengine_iterator_p.h"
@@ -49 +52 @@
 #include <QtCore/qvariant.h>
 
+#ifndef QT_OS2_USE_DOSFINDFIRST
 #include <dirent.h>
 #include <unistd.h>
+#else
+#include <QtCore/qdatetime.h>
+#include "qfsfileengine.h"
+#include "qfileinfo_p.h"
+#endif
 
 #ifndef QT_NO_FSFILEENGINE
 
 QT_BEGIN_NAMESPACE
+
+#ifndef QT_OS2_USE_DOSFINDFIRST
 
 class QFSFileEngineIteratorPlatformSpecificData
@@ -76 +87 @@
 };
 
+#else // !QT_OS2_USE_DOSFINDFIRST
+
+class QFSFileEngineIteratorPlatformSpecificData : public QFileInfo
+{
+public:
+    inline QFSFileEngineIteratorPlatformSpecificData()
+        : done(false), hdir(HDIR_CREATE)
+    {
+        memset(&findBuf, 0, sizeof(findBuf));
+    }
+
+    static QDateTime convertFileDateTime(FDATE fdate, FTIME ftime);
+    void updateFileInfo(const QString &path, FILEFINDBUF3L *ffb);
+    void resetFileInfo() { d_ptr->reset(); }
+
+    bool done;
+    HDIR hdir;
+    FILEFINDBUF3L findBuf;
+};
+
+#endif // !QT_OS2_USE_DOSFINDFIRST
+
 void QFSFileEngineIterator::advance()
 {
+#ifndef QT_OS2_USE_DOSFINDFIRST
+
     currentEntry = platform->dirEntry ? QFile::decodeName(QByteArray(platform->dirEntry->d_name)) : QString();
 
@@ -99 +134 @@
 #endif
     }
+
+#else // !QT_OS2_USE_DOSFINDFIRST
+
+    if (platform->hdir == HDIR_CREATE) {
+        platform->resetFileInfo();
+        currentEntry.clear();
+        return;
+    }
+
+    // set QFileInfo from find buffer
+    platform->updateFileInfo(path(), &platform->findBuf);
+    currentEntry = platform->fileName();
+
+    ULONG count = 1;
+    if (DosFindNext(platform->hdir, &platform->findBuf,
+                    sizeof(platform->findBuf), &count) != NO_ERROR) {
+        DosFindClose(platform->hdir);
+        platform->hdir = HDIR_CREATE;
+        platform->done = true;
+    }
+
+#endif // !QT_OS2_USE_DOSFINDFIRST
 }
 
@@ -108 +165 @@
 void QFSFileEngineIterator::deletePlatformSpecifics()
 {
+#ifndef QT_OS2_USE_DOSFINDFIRST
+
     if (platform->dir) {
         ::closedir(platform->dir);
@@ -115 +174 @@
 #endif
     }
+
+#else // !QT_OS2_USE_DOSFINDFIRST
+
+    if (platform->hdir != HDIR_CREATE)
+        DosFindClose(platform->hdir);
+
+#endif // !QT_OS2_USE_DOSFINDFIRST
+
     delete platform;
     platform = 0;
@@ -121 +188 @@
 bool QFSFileEngineIterator::hasNext() const
 {
+#ifndef QT_OS2_USE_DOSFINDFIRST
     if (!platform->done && !platform->dir) {
+#else
+    if (!platform->done && platform->hdir == HDIR_CREATE) {
+#endif
         QFSFileEngineIterator *that = const_cast<QFSFileEngineIterator *>(this);
         QString path = that->path();
@@ -149 +220 @@
             }
         }
+
         if (arc == ERROR_NOT_READY || arc == ERROR_DISK_CHANGE ||
             arc == ERROR_SECTOR_NOT_FOUND || arc == ERROR_INVALID_DRIVE) {
-            that->platform->dir = 0;
             that->platform->done = true;
+#ifndef QT_OS2_USE_DOSFINDFIRST
         } else if ((that->platform->dir = ::opendir(QFile::encodeName(path).data())) == 0) {
             that->platform->done = true;
@@ -163 +235 @@
             that->platform->mt_file = (dirent *)new char[maxPathName];
 #endif
-
             that->advance();
+#else // !QT_OS2_USE_DOSFINDFIRST
+        } else {
+            ULONG count = 1;
+            if (DosFindFirst(QFile::encodeName(path).append('*').constData(),
+                             &that->platform->hdir,
+                             FILE_NORMAL | FILE_READONLY | FILE_HIDDEN |
+                             FILE_SYSTEM | FILE_DIRECTORY | FILE_ARCHIVED,
+                             &that->platform->findBuf, sizeof(that->platform->findBuf),
+                             &count, FIL_STANDARDL) != NO_ERROR) {
+                that->platform->done = true;
+            } else {
+                that->platform->resetFileInfo();
+                that->currentEntry.clear();
+            }
+#endif // !QT_OS2_USE_DOSFINDFIRST
         }
     }
+
     return !platform->done;
 }
 
+QFileInfo QFSFileEngineIterator::currentFileInfo() const
+{
+#ifndef QT_OS2_USE_DOSFINDFIRST
+    return QAbstractFileEngineIterator::currentFileInfo();
+#else
+    // return a copy of our cached file info
+    return *platform;
+#endif
+}
+
+#ifdef QT_OS2_USE_DOSFINDFIRST
+
+//static
+QDateTime QFSFileEngineIteratorPlatformSpecificData::convertFileDateTime(FDATE fdate,
+                                                                         FTIME ftime)
+{
+    QDateTime dt;
+
+    dt.setDate(QDate(fdate.year + 1980, fdate.month, fdate.day));
+    dt.setTime(QTime(ftime.hours, ftime.minutes, ftime.twosecs * 2));
+
+    return dt;
+}
+
+void QFSFileEngineIteratorPlatformSpecificData::updateFileInfo(const QString &path,
+                                                               FILEFINDBUF3L *ffb)
+{
+    QString fileName = path;
+    if (!fileName.endsWith(QLatin1Char('/')))
+        fileName.append(QLatin1Char('/'));
+    fileName.append(QFile::decodeName(QByteArray(ffb->achName)));
+    d_ptr->initFileEngine(fileName);
+
+    d_ptr->data->fileFlags = QAbstractFileEngine::ExistsFlag    |
+                             QAbstractFileEngine::LocalDiskFlag |
+                             QAbstractFileEngine::ReadOwnerPerm |
+                             QAbstractFileEngine::ReadUserPerm  |
+                             QAbstractFileEngine::ReadGroupPerm |
+                             QAbstractFileEngine::ReadOtherPerm;
+
+    if ((ffb->attrFile & FILE_READONLY) == 0) {
+        d_ptr->data->fileFlags |= QAbstractFileEngine::WriteOwnerPerm |
+                                  QAbstractFileEngine::WriteUserPerm  |
+                                  QAbstractFileEngine::WriteGroupPerm |
+                                  QAbstractFileEngine::WriteOtherPerm;
+    }
+
+    if (ffb->attrFile & FILE_DIRECTORY)
+        d_ptr->data->fileFlags |= QAbstractFileEngine::DirectoryType;
+    else
+        d_ptr->data->fileFlags |= QAbstractFileEngine::FileType;
+
+    if (ffb->attrFile & FILE_HIDDEN)
+        d_ptr->data->fileFlags |= QAbstractFileEngine::HiddenFlag;
+
+    d_ptr->data->fileTimes[QAbstractFileEngine::CreationTime] =
+            convertFileDateTime(ffb->fdateCreation, ffb->ftimeCreation);
+
+    d_ptr->data->fileTimes[QAbstractFileEngine::ModificationTime] =
+            convertFileDateTime(ffb->fdateLastWrite, ffb->ftimeLastWrite);
+
+    d_ptr->data->fileTimes[QAbstractFileEngine::AccessTime] =
+            convertFileDateTime(ffb->fdateLastAccess, ffb->ftimeLastAccess);
+
+    d_ptr->data->fileSize = ffb->cbFile;
+
+    // mark fields as "cached" and "present"
+    d_ptr->data->cachedFlags  = QFileInfoPrivate::CachedSize |
+                                QFileInfoPrivate::CachedFileFlags |
+                                QFileInfoPrivate::CachedLinkTypeFlag |
+                                QFileInfoPrivate::CachedBundleTypeFlag |
+                                QFileInfoPrivate::CachedMTime |
+                                QFileInfoPrivate::CachedCTime |
+                                QFileInfoPrivate::CachedATime;
+    d_ptr->data->cache_enabled = 1;
+}
+
+#endif // QT_OS2_USE_DOSFINDFIRST
+
 QT_END_NAMESPACE
 

trunk/src/corelib/io/qfsfileengine_os2.cpp

@@ -42 +42 @@
 ****************************************************************************/
 
+// temporary, until struct dirent in kLIBC gets creation and access time fields
+#define QT_OS2_USE_DOSFINDFIRST
+
 #include "qplatformdefs.h"
 #include "qabstractfileengine.h"
@@ -661 +664 @@
             is_link = false;        // drive/share names are never symlinks
         } else {
+#ifdef QT_OS2_USE_DOSFINDFIRST
+            is_link = false;
+#else
             QT_STATBUF st;          // don't clobber our main one
             that->is_link = (QT_LSTAT(nativeFilePath.constData(), &st) == 0) ?
                             S_ISLNK(st.st_mode) : false;
+#endif
         }
     }