Piccolo 番外 - 文件系统

GAMES 编程游戏制作

新年前最后一篇~

在之前，我写了一篇文章关于自定义的虚拟文件系统，但是这篇文章介绍的文件系统过于复杂，且在实际测试时效率不够高，关键问题在于需要从vblock一层层查找vnode来确定文件位置，而并没有全局的文件路径缓存。因此在研究学习Piccolo过程中，又重新整理了一下这部分的代码。

我先对文件系统需求进行分析。我希望可以从普通文件目录与压缩文件中读取文件内容，且该文件系统的配置可以利用Piccolo的反射系统来读取数据。还有就是可以通过类似追加的方式来给文件系统打补丁，比如对main.zip添加patch.zip的补丁，使文件系统可以读取patch中数据。最后就是希望文件系统的速度比上一个版本快。

最后设计的代码如下，完整代码在这里：

#pragma once
#include "runtime/core/meta/reflection/reflection.h"

#include <vector>
#include <string>

namespace Piccolo
{
    REFLECTION_TYPE(FSConfig)
    CLASS(FSConfig, Fields)
    {
        REFLECTION_BODY(FSConfig)
    public:
        std::string m_vpath;
        std::string m_rpath;
        std::string m_type;
    };


    // stroe mount points
    REFLECTION_TYPE(VFSConfig)
    CLASS(VFSConfig, Fields)
    {
        REFLECTION_BODY(VFSConfig)
    public:
        std::vector<FSConfig> m_configs;
    };
}

#pragma once
#include "runtime/platform/file_system/basic/file.h"
#include "runtime/platform/file_system/basic/file_system.h"
#include "runtime/platform/file_system/vfs_config.h"

#include <unordered_map>

namespace Piccolo
{
    class VFS
    {
    public:
        void mount(const VFSConfig& config);
        void unmountAll();

        FilePtr open(const std::string& vpath, uint32_t mode);
        bool    close(FilePtr file);

        size_t read(FilePtr file, std::vector<std::byte>& buffer);
        size_t write(FilePtr file, const std::vector<std::byte>& buffer);

        std::future<size_t> readAsync(std::shared_ptr<thread_pool> tp, FilePtr file, std::vector<std::byte>& buffer);
        std::future<size_t> writeAsync(std::shared_ptr<thread_pool> tp, FilePtr file, const std::vector<std::byte>& buffer);

    private:
        void mountFS(const FSConfig& fs);
        void unmountFS(const FSConfig& fs);

        void buildVFSCache();

    private:
        std::vector<FileSystemPtr>                     m_fs;
        std::unordered_map<std::string, FileSystemPtr> m_fileCache;
        std::unordered_map<std::string, FileSystemPtr> m_dirCache;
    };

    extern VFS g_vfs;
} // namespace Piccolo

#include "runtime/platform/file_system/vfs.h"

#include "runtime/platform/file_system/basic/file_utils.h"
#include "runtime/platform/file_system/memory_file/memory_file.h"
#include "runtime/platform/file_system/memory_file/memory_file_system.h"
#include "runtime/platform/file_system/native_file/native_file.h"
#include "runtime/platform/file_system/native_file/native_file_system.h"
#include "runtime/platform/file_system/zip_file/zip_file.h"
#include "runtime/platform/file_system/zip_file/zip_file_system.h"

#include "runtime/function/global/global_context.h"
#include "runtime/resource/config_manager/config_manager.h"

#include <algorithm>

namespace Piccolo
{
    VFS g_vfs;

    void VFS::mount(const VFSConfig& config)
    {
        for (auto& fs : config.m_configs)
        {
            mountFS(fs);
        }
        buildVFSCache();
    }

    void VFS::mountFS(const FSConfig& fs)
    {
        if (fs.m_type == "native")
        {
            auto root  = g_runtime_global_context.m_config_manager->getRootFolder().string();
            auto rpath = combine_path(root, fs.m_rpath);
            m_fs.emplace_back(std::make_shared<NativeFileSystem>(fs.m_vpath, rpath));
        }
        else if (fs.m_type == "memory")
        {
            m_fs.emplace_back(std::make_shared<MemoryFileSystem>(fs.m_vpath, fs.m_rpath));
        }
        else if (fs.m_type == "compress-zip")
        {
            auto root  = g_runtime_global_context.m_config_manager->getRootFolder().string();
            auto rpath = combine_path(root, fs.m_rpath);
            m_fs.emplace_back(std::make_shared<ZipFileSystem>(fs.m_vpath, rpath));
        }
    }

    void VFS::unmountFS(const FSConfig& config)
    {
        auto iter =
            std::find_if(m_fs.begin(), m_fs.end(), [config](FileSystemPtr fs) { return fs->m_rpath == config.m_rpath && fs->m_vpath == config.m_vpath; });
        if (iter != m_fs.end())
        {
            m_fs.erase(iter);
        }
    }

    void VFS::unmountAll()
    {
        m_fs.clear();
        m_fileCache.clear();
        m_dirCache.clear();
    }

    void VFS::buildVFSCache()
    {
        for (auto fs : m_fs)
        {
            fs->buildFSCache();

            for (auto file : fs->m_vfiles)
            {
                m_fileCache[file] = fs;
            }
            for (auto dir : fs->m_vdirs)
            {
                m_dirCache[dir] = fs;
            }
        }
    }

    FilePtr VFS::open(const std::string& vpath, uint32_t mode)
    {
        auto fs = m_fileCache.find(vpath);
        if (fs == m_fileCache.end())
        {
            return nullptr;
        }
        else
        {
            return fs->second->open(vpath, mode);
        }
    }

    bool VFS::close(FilePtr file) { return file->close(); }

    size_t VFS::read(FilePtr file, std::vector<std::byte>& buffer) { return file->read(buffer); }

    size_t VFS::write(FilePtr file, const std::vector<std::byte>& buffer) { return file->write(buffer); }

    std::future<size_t> VFS::readAsync(std::shared_ptr<thread_pool> tp, FilePtr file, std::vector<std::byte>& buffer)
    {
        return tp->enqueue_task(&VFS::read, this, file, buffer);
    }

    std::future<size_t> VFS::writeAsync(std::shared_ptr<thread_pool> tp, FilePtr file, const std::vector<std::byte>& buffer)
    {
        return tp->enqueue_task(&VFS::write, this, file, buffer);
    }
} // namespace Piccolo

#pragma once

#include <cstddef>
#include <memory>
#include <string>
#include <vector>

namespace Piccolo
{
    using FileBuffer = std::vector<std::byte>;

    class FileSystem;

    class File
    {
    public:
        enum Mode : uint32_t
        {
            read_bin       = 1 << 0,
            write_bin      = 1 << 1,
            readwrite_bin  = read_bin | write_bin,
            read_text      = 1 << 2,
            write_text     = 1 << 3,
            readwrite_text = read_text | write_text,
            append         = 1 << 4,
            truncate       = 1 << 5,
        };

        enum Origin : uint32_t
        {
            beg = 0,
            end = 1,
            set = 2,
        };

    public:
        File(const std::string& vpath, const std::string& rpath) : m_vpath {vpath}, m_rpath {rpath} {}
        virtual ~File() = default;

        virtual bool open(uint32_t mode) = 0;
        virtual bool close()             = 0;

        virtual bool   isOpened() const                   = 0;
        virtual bool   isReadOnly() const                 = 0;
        virtual size_t size() const                       = 0;
        virtual size_t seek(size_t offset, Origin origin) = 0;
        virtual size_t tell()                             = 0;
        // If Read Text, will read [size - 1] bytes, because last will be ['\0'] for string
        virtual size_t read(std::vector<std::byte>& data)        = 0;
        virtual size_t write(const std::vector<std::byte>& data) = 0;
        virtual size_t read(std::string& data)                   = 0;
        virtual size_t write(const std::string& data)            = 0;

    public:
        uint32_t m_mode;

        std::string m_vpath;
        std::string m_rpath;

        std::shared_ptr<FileSystem> m_fs;
    };

    using FilePtr = std::shared_ptr<File>;
} // namespace Piccolo

#pragma once
#include "runtime/platform/file_system/basic/file.h"

#include "runtime/core/thread/thread_pool.h"

#include <algorithm>
#include <future>

namespace Piccolo
{
    class FileSystem
    {
    public:
        FileSystem(const std::string& vpath, const std::string& rpath) : m_vpath {vpath}, m_rpath {rpath} {}
        virtual ~FileSystem() = default;

        bool isFileExist(const std::string& file_name) const
        {
            auto iter = std::find_if(m_vfiles.begin(), m_vfiles.end(), [file_name](const std::string& file) { return file_name == file; });
            return iter != m_vfiles.end();
        }

        bool isDirExist(const std::string& dir_name) const
        {
            auto iter = std::find_if(m_vdirs.begin(), m_vdirs.end(), [dir_name](const std::string& file) { return dir_name == file; });
            return iter != m_vdirs.end();
        }

        size_t read(FilePtr file, std::vector<std::byte>& buffer) { return file->read(buffer); }
        size_t write(FilePtr file, const std::vector<std::byte>& buffer) { return file->write(buffer); }
        size_t read(FilePtr file, std::string& buffer) { return file->read(buffer); }
        size_t write(FilePtr file, const std::string& buffer) { return file->write(buffer); }

        std::future<size_t> readAsync(std::shared_ptr<thread_pool> tp, FilePtr file, std::vector<std::byte>& buffer)
        {
            return tp->enqueue_task([file, &buffer]() { return file->read(buffer); });
        }

        std::future<size_t> writeAsync(std::shared_ptr<thread_pool> tp, FilePtr file, const std::vector<std::byte>& buffer)
        {
            return tp->enqueue_task([file, &buffer]() { return file->write(buffer); });
        }

        std::future<size_t> readAsync(std::shared_ptr<thread_pool> tp, FilePtr file, std::string& buffer)
        {
            return tp->enqueue_task([file, &buffer]() { return file->read(buffer); });
        }

        std::future<size_t> writeAsync(std::shared_ptr<thread_pool> tp, FilePtr file, const std::string& buffer)
        {
            return tp->enqueue_task([file, &buffer]() { return file->write(buffer); });
        }

        // TODO
        // virtual bool createFile(const std::string& file_path) = 0;
        // virtual bool deleteFile(const std::string& file_path) = 0;
        // virtual bool moveFile(const std::string& src, const std::string& dst) = 0;
        // virtual bool copyFile(const std::string& src, const std::string& dst) = 0;

        // TODO
        // virtual bool createDir(const std::string& dir_path) = 0;
        // virtual bool deleteDir(const std::string& dir_path) = 0;
        // virtual bool moveDir(const std::string& src, const std::string& dst) = 0;
        // virtual bool copyDir(const std::string& src, const std::string& dst) = 0;

        // -------------------------------------------------------------------
        // -------------------------------------------------------------------
        // -------------------------------------------------------------------

        virtual void    buildFSCache()                                = 0;
        virtual FilePtr open(const std::string& vpath, uint32_t mode) = 0;
        virtual bool    close(FilePtr file)                           = 0;

        // -------------------------------------------------------------------
        // -------------------------------------------------------------------
        // -------------------------------------------------------------------

    public:
        std::string m_vpath;
        std::string m_rpath;

        std::vector<std::string> m_vfiles;
        std::vector<std::string> m_vdirs;

        std::vector<std::string> m_rfiles;
        std::vector<std::string> m_rdirs;
    };

    using FileSystemPtr = std::shared_ptr<FileSystem>;
} // namespace Piccolo

在这里vfs使用的是全局变量，根据需求也可以改成单例模式。除此之外，vfs当前只支持读取功能，对写入的支持比较有限，只支持普通文件目录已经存在的文件写入。后续可以考虑升级这个部分的功能，不过由于运行的游戏无需修改游戏资源文件，对于现在的场景来说，基本够用了。

如果觉得文章对你有用，请随意赞赏

Piccolo 番外 - 文件系统

https://rocketsky.top/archives/piccolo%E7%95%AA%E5%A4%96-%E6%96%87%E4%BB%B6%E7%B3%BB%E7%BB%9F

作者

Rocket Sky

发布于

2023-01-17

更新于

2023-06-13

许可协议

CC BY 4.0