FileSystem.cpp

//////////////////////////////////////////////////////////////////////////////////
// Copyright (C) Inspiration Byte
// 2009-2020
//////////////////////////////////////////////////////////////////////////////////
// Description: Equilibrium Filesystem
//////////////////////////////////////////////////////////////////////////////////

#include "core/core_common.h"

#ifdef _WIN32
#define WIN32_LEAN_AND_MEAN
#include <Windows.h>
#include <direct.h>	// _mkdir()
#else
#include <unistd.h> // rmdir(), getcwd()
#define USE_PATH_TO_FILE_MAPPING
#endif

#include <sys/types.h>
#include <sys/stat.h>

#include "core/IDkCore.h"
#include "core/ICommandLine.h"
#include "core/platform/OSFindData.h"
#include "utils/CRC32.h"
#include "utils/KeyValues.h"

#include "FileSystem.h"
#include "dpk/BasePackageFileReader.h"

static int FSStringId(const char *str)
{
	constexpr int MULTIPLIER = 37;

	ASSERT(str);
	int hash = 0;
	for (const char* p = str; *p; p++)
		hash = MULTIPLIER * hash + (int)CType::LowerChar(*p);

	return hash;
}

using namespace Threading;
static CEqMutex	s_FSMutex;

EXPORTED_INTERFACE(IFileSystem, CFileSystem);

struct FSSearchPathInfo
{
	// name hash to file path mapping (case-insensitive support)
	Map<int, EqString> pathToFileMapping{ PP_SL };
	EqString	id;
	EqString	path;
	bool		writePath{ false };

	static void MapFiles(EqStringRef basePath, FSSearchPathInfo& pathInfo);
};

void FSSearchPathInfo::MapFiles(EqStringRef basePath, FSSearchPathInfo& pathInfo)
{
	Array<EqString> openSet(PP_SL);
	openSet.reserve(5000);

	openSet.append(fnmPathCombine(basePath, pathInfo.path));

	while (openSet.numElem())
	{
		EqString path = openSet.fastPopFront();

		OSFindData findData;
		if (!findData.Init(path + _CORRECT_PATH_SEPARATOR_STR "*"))
			break;

		do
		{
			EqStringRef name = findData.GetCurrentPath();
			if (name[0] == '.')
				continue;

			EqStringRef entryName = fnmPathCombine(path.TrimChar(CORRECT_PATH_SEPARATOR), name);
			if (findData.IsDirectory())
			{
				openSet.append(entryName);
			}
			else
			{
				// add a mapping
				const int nameHash = FSStringId(entryName);
				pathInfo.pathToFileMapping.insert(nameHash, entryName);
			}
		} while (findData.GetNext());
	}

	if (pathInfo.pathToFileMapping.size())
		DevMsg(DEVMSG_FS, "Mapped %d files for '%s'\n", pathInfo.pathToFileMapping.size(), pathInfo.path.ToCString());
}

//------------------------------------------------------------------------------
// File stream
//------------------------------------------------------------------------------

COSFileStream::COSFileStream(const char* fileName, COSFile&& file)
	: m_name(fileName)
	, m_osFile(std::move(file))
{
}

VSSize COSFileStream::Seek(int64 pos, EFileStreamSeek seekType )
{
	return m_osFile.Seek(pos, static_cast<COSFile::ESeekPos>(seekType));
}

VSSize COSFileStream::Tell() const
{
	return m_osFile.Tell();
}

VSSize COSFileStream::Read( void *dest, VSSize count, VSSize size)
{
	VSSize numBytes = count * size;
	if (numBytes <= 0)
		return 0;
	return m_osFile.Read(dest, numBytes) / size;
}

VSSize COSFileStream::Write( const void *src, VSSize count, VSSize size)
{
	VSSize numBytes = count * size;
	if (numBytes <= 0)
		return 0;
	return m_osFile.Write(src, numBytes) / size;
}

bool COSFileStream::Flush()
{
	return m_osFile.Flush();
}

VSSize COSFileStream::GetSize() const
{
	const VSSize currentPos = Tell();
	m_osFile.Seek(0, COSFile::ESeekPos::END);

	const VSSize size = Tell();
	m_osFile.Seek(currentPos, COSFile::ESeekPos::SET);

	return size;
}

uint32 COSFileStream::GetCRC32() const
{
	const VSSize pos = Tell();
	const VSSize fileSize = GetSize();

	ubyte* fileData = (ubyte*)PPAlloc(fileSize + 16);
	m_osFile.Read(fileData, fileSize);
	m_osFile.Seek(pos, COSFile::ESeekPos::SET);

	const uint32 crc = CRC32_BlockChecksum(fileData, fileSize);
	PPFree(fileData);

	return crc;
}

//------------------------------------------------------------------------------

class CFlatFileReader : public CBasePackageReader
{
public:
	EPackageType		GetType() const { return PACKAGE_READER_FLAT; }

	bool				InitPackage(const char* filename, const char* name /*= nullptr*/, const char* mountPath /*= nullptr*/);
	IFileStreamPtr		Open(const char* filename, int modeFlags);
	bool				FileExist(const char* filename) const;

	// stubs
	bool				OpenEmbeddedPackage(CBasePackageReader* target, const char* filename) { return false; }
	IFileStreamPtr		Open(int fileIndex, int modeFlags) { return nullptr; }
	int					FindFileIndex(const char* filename) const { return -1; }
	int					GetFileCount() const { return -1; }

private:
	FSSearchPathInfo	m_pathInfo;
};

bool CFlatFileReader::InitPackage(const char* filename, const char* name /*= nullptr*/, const char* mountPath /*= nullptr*/)
{
	m_name = name ? name : filename;
	m_packagePath = filename;

	m_pathInfo.path = m_packagePath;
	FSSearchPathInfo::MapFiles("", m_pathInfo);

	return true;
}

IFileStreamPtr CFlatFileReader::Open(const char* filename, int modeFlags)
{
	if (modeFlags != FS_OPEN_READ)
		return nullptr;

	EqString filePath = fnmPathCombine(m_packagePath, filename);
	const int nameHash = FSStringId(filePath);
	auto it = m_pathInfo.pathToFileMapping.find(nameHash);
	if (!it)
		return nullptr;

	COSFile osFile;
	if (osFile.Open(filePath, COSFile::READ))
		return IFileStreamPtr(CRefPtr_new(COSFileStream, *it, std::move(osFile)));

	return nullptr;
}

bool CFlatFileReader::FileExist(const char* filename) const
{
	EqString filePath = fnmPathCombine(m_packagePath, filename);
	const int nameHash = FSStringId(filePath);
	auto it = m_pathInfo.pathToFileMapping.find(nameHash);
	return !it.atEnd();
}

//------------------------------------------------------------------------------
// Main filesystem code
//------------------------------------------------------------------------------

struct FSFindData
{
	OSFindData	osFind;
	EqString	wildcard;
	int			searchPaths{ -1 };
	int			dirIndex{ 0 };
	bool		singleDir{ false };
};

//--------------------------------------------------

static ESearchPath GetSearchPathByName(EqStringRef str)
{
	if (!str.CompareCaseIns("SP_MOD"))
		return SP_MOD;
	else if (!str.CompareCaseIns("SP_DATA"))
		return SP_DATA;

	return SP_ROOT;
}

static bool mkdirRecursive(const char* path, bool includeDotPath)
{
	char folder[265];
	const char* end;
	const char* curend;

	end = path;

	do
	{
		// skip any separators in the beginning
		while (*end == CORRECT_PATH_SEPARATOR)
			end++;

		// get next string part
		curend = (char*)strchr(end, CORRECT_PATH_SEPARATOR);

		if (curend)
			end = curend;
		else
			end = path + strlen(path);

		strncpy(folder, path, end - path);
		folder[end - path] = 0;

		// stop on file extension if needed
		if (!includeDotPath && strchr(folder, '.'))
			break;

		int result;
#ifdef _WIN32
		result = _mkdir(folder);
#else
		result = mkdir(folder, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
#endif

		if (result > 0 && result != EEXIST)
			return false;

	} while (curend);

	return true;
}

static int fast_stat(const char* path, struct stat* st)
{
#ifdef _WIN32
	if (!path)
		return -1;

	WIN32_FILE_ATTRIBUTE_DATA attrData;
	if (!GetFileAttributesExA(path, GetFileExInfoStandard, &attrData))
		return -1;

	if (!st)
		return 0;

	ZeroMemory(st, sizeof(*st));
	LARGE_INTEGER fileSize;
	fileSize.LowPart = attrData.nFileSizeLow;
	fileSize.HighPart = attrData.nFileSizeHigh;
	st->st_size = fileSize.QuadPart;

	auto FileTimeToTime64 = [](const FILETIME& ft) {
		ULARGE_INTEGER uli;
		uli.LowPart = ft.dwLowDateTime;
		uli.HighPart = ft.dwHighDateTime;
		return static_cast<__time64_t>(uli.QuadPart / 10000000ULL - 11644473600ULL);
	};

	st->st_atime = FileTimeToTime64(attrData.ftLastAccessTime);
	st->st_mtime = FileTimeToTime64(attrData.ftLastWriteTime);
	st->st_ctime = FileTimeToTime64(attrData.ftCreationTime);

	ushort mode = _S_IREAD;
	if (attrData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)
		mode |= _S_IFDIR | _S_IEXEC;
	else
		mode |= _S_IFREG;

	if (!(attrData.dwFileAttributes & FILE_ATTRIBUTE_READONLY))
		mode |= _S_IWRITE;

	st->st_mode = mode | (mode >> 3) | (mode >> 6);
	st->st_nlink = 1;	// default dummy values for unused Win32 attributes

	return 0;
#else
	struct stat dummy{};
	if (!st) st = &dummy;
	return stat(path, st);
#endif
}

//--------------------------------------------------

CFileSystem::CFileSystem()
{
	g_eqCore->RegisterInterface(this);
}

CFileSystem::~CFileSystem()
{
	g_eqCore->UnregisterInterface<CFileSystem>();
}

bool CFileSystem::Init(bool editorMode)
{
	Msg("\n-------- Filesystem Init --------\n");

	const KVSection* fsConfig = g_eqCore->GetConfig().FindSection("FileSystem", KV_FLAG_SECTION);
	if (!fsConfig)
	{
		Msg("E2.CONFIG missing FileSystemDirectories section!\n");
		return false;
	}

	EqStringRef workDir;
	if (fsConfig->Get("WorkDir").GetValues(workDir))
	{
#ifdef _WIN32
		SetCurrentDirectoryA(workDir);
#else
		chdir(workDir);
#endif // _WIN32
	}

	EqStringRef basePath;
	if(fsConfig->Get("BasePath").GetValues(basePath))
	{
		SetBasePath(basePath);
		if (m_basePath.Length() > 0)
			MsgInfo("* Base directory: %s\n", m_basePath.GetData());
	}

	m_dataDir = "EngineBase";
	fsConfig->Get("EngineDataDir").GetValues(m_dataDir);
	MsgInfo("* Engine data directory: %s\n", m_dataDir.GetData());

	if(!editorMode)
	{
		EqStringRef gamePath = g_eqCore->GetApplicationName();
		fsConfig->Get("DefaultGameDir").GetValues(gamePath);

		const int gamePathArg = g_cmdLine->Find("-game");
		if(gamePathArg != -1)
			gamePath = g_cmdLine->GetArgumentsOf(gamePathArg);

		AddSearchPath("$GAME$", gamePath);

		MsgInfo("* Content data directory: %s\n", GetCurrentGameDirectory());

		// FS dev addon for game tools
		const int iDevAddonPathArg = g_cmdLine->Find("-devAddon");
		const char* devAddonPath = g_cmdLine->GetArgumentsOf(iDevAddonPathArg);
		if (devAddonPath)
		{
			AddSearchPath("$MOD$_$WRITE$", devAddonPath);
			MsgInfo("* Dev addon path: %s\n", devAddonPath);
		}
	}

	for(const KVSection& pkgSec : fsConfig->Keys("AddPackage"))
	{
		EqStringRef packageName;
		EqStringRef pathType = "SP_MOD";
		EqStringRef mountPath;
		if (pkgSec.GetValues(packageName, pathType, mountPath) < 1)
			continue;

		const ESearchPath type = GetSearchPathByName(pathType);
		AddPackage(packageName, type, mountPath);
	}

	m_isInit = true;
    return true;
}

void CFileSystem::Shutdown()
{
	m_isInit = false;
	m_fsPackages.clear(true);
	m_findDatas.clear(true);
	m_directories.clear(true);
}

void CFileSystem::SetBasePath(const char* path) 
{ 
	m_basePath = path;
	fnmPathFixSeparators(m_basePath);

	if(m_basePath[m_basePath.Length()-1] != CORRECT_PATH_SEPARATOR)
		m_basePath.Append(CORRECT_PATH_SEPARATOR);
}

EqString CFileSystem::FindFilePath(const char* filename, int searchFlags /*= -1*/) const
{
	EqString existingFilePath;

	auto walkFileFunc = [&](EqStringRef filePath, ESearchPath searchPath, const FSSearchPathInfo& spInfo, int spFlags) -> bool
	{
		if (fast_stat(filePath, nullptr) == 0)
		{
			existingFilePath = filePath;
			return true;
		}

		return false;
	};

	WalkOverSearchPaths(searchFlags, filename, walkFileFunc);

	return existingFilePath;
}

IFileStreamPtr CFileSystem::Open(const char* filename, int openFlags, int searchFlags/* = -1*/ )
{
	ASSERT_MSG(filename, "Open - Must specify 'filename'");
	ASSERT_MSG(openFlags, "Open - Must specify 'mode'");

	if (!filename || !openFlags)
		return nullptr;

	int osModeFlags = 0;
	if(openFlags & FS_OPEN_READ)
		osModeFlags |= COSFile::READ;

	if (openFlags & FS_OPEN_WRITE)
		osModeFlags |= COSFile::WRITE;

	if (openFlags & FS_OPEN_APPEND)
		osModeFlags |= COSFile::APPEND;

	const bool isWrite = osModeFlags & (COSFile::APPEND | COSFile::WRITE);

	IFileStreamPtr fileHandle;
	auto walkFileFunc = [&](EqStringRef filePath, ESearchPath searchPath, const FSSearchPathInfo& spInfo, int spFlags) -> bool
	{
		if (isWrite && !spInfo.writePath)
			return false;

		COSFile osFile;
		if (osFile.Open(filePath, osModeFlags))
		{
			fileHandle = IFileStreamPtr(CRefPtr_new(COSFileStream, filename, std::move(osFile)));
			return true;
		}

		if (isWrite || (spFlags & SP_IGNORE_PACKAGE))
			return false;

		const EqStringRef filePathNoBase = filePath.ToCString() + m_basePath.Length();

		// If failed to load directly, load it from package, in backward order
		for (int j = m_fsPackages.numElem() - 1; j >= 0; j--)
		{
			CBasePackageReader* fsPacakage = static_cast<CBasePackageReader*>(m_fsPackages[j].Ptr());

			if (!(spFlags & fsPacakage->GetSearchPath()))
				continue;

			EqString pkgFileName;
			if (!fsPacakage->GetInternalFileName(pkgFileName, filePathNoBase))
				continue;

			// package readers do not support base path, get rid of it
			fileHandle = fsPacakage->Open(pkgFileName, osModeFlags);
			if (fileHandle)
				return true;
		}

		return false;
	};

	WalkOverSearchPaths(searchFlags, filename, walkFileFunc);
	return fileHandle;
}

ubyte* CFileSystem::GetFileBuffer(const char* filename, VSSize* filesize/* = 0*/, int searchFlags/* = -1*/)
{
	IFileStreamPtr pFile = Open(filename, FS_OPEN_READ, searchFlags);

    if (!pFile)
        return nullptr;

    const VSSize length = pFile->GetSize();
    ubyte *buffer = (ubyte*)PPAlloc(length + 1);

    if (!buffer)
        return nullptr;

	memset(buffer, 0, length+1);

	pFile->Read(buffer, 1, length);
    buffer[length] = 0;

    if (filesize)
        *filesize = length;

    return buffer;
}

VSSize CFileSystem::GetFileSize(const char* filename, int searchFlags/* = -1*/)
{
    IFileStreamPtr file = Open(filename, FS_OPEN_READ, searchFlags);
    if (!file)
        return 0;

	return file->GetSize();
}

uint32 CFileSystem::GetFileCRC32(const char* filename, int searchFlags)
{
    IFileStreamPtr file = Open(filename, FS_OPEN_READ, searchFlags);
    if (!file)
        return 0;

    return file->GetCRC32();
}


bool CFileSystem::FileCopy(const char* filename, const char* dest_file, bool overWrite, ESearchPath search)
{
	char buf[4096];

	if( FileExist(filename, search) && (overWrite || FileExist(dest_file, search) == false))
	{
		IFileStreamPtr fpWrite = Open(dest_file, FS_OPEN_WRITE, search);
		IFileStreamPtr fpRead = Open(filename, FS_OPEN_READ, search);

		if (!fpRead || !fpWrite)
			return false;

		VSSize nread;
		while (nread = fpRead->Read(buf, sizeof(buf), 1), nread > 0)
		{
			char* outPtr = buf;
			VSSize nwritten;

			do {
				nwritten = fpWrite->Write(outPtr, nread, 1);

				if (nwritten >= 0)
				{
					nread -= nwritten;
					outPtr += nwritten;
				}
			} while (nread > 0);
		}

		ASSERT(nread == 0);
	}
	else
		return false;

	return true;
}

bool CFileSystem::FileExist(const char* filename, int searchFlags) const
{
	auto walkFileFunc = [&](EqStringRef filePath, ESearchPath searchPath, const FSSearchPathInfo& spInfo, int spFlags) -> bool
	{
		if (fast_stat(filePath, nullptr) == 0)
			return true;

		if (spFlags & SP_IGNORE_PACKAGE)
			return false;

		const EqStringRef filePathNoBase = filePath.ToCString() + m_basePath.Length();

		// If failed to load directly, load it from package, in backward order
		for (int j = m_fsPackages.numElem() - 1; j >= 0; j--)
		{
			CBasePackageReader* fsPacakage = static_cast<CBasePackageReader*>(m_fsPackages[j].Ptr());

			if (!(spFlags & fsPacakage->GetSearchPath()))
				continue;

			EqString pkgFileName;
			if (!fsPacakage->GetInternalFileName(pkgFileName, filePathNoBase))
				continue;

			// package readers do not support base path, get rid of it
			if (fsPacakage->FileExist(pkgFileName))
				return true;
		}

		return false;
	};

	return WalkOverSearchPaths(searchFlags, filename, walkFileFunc);
}

EqStringRef CFileSystem::GetSearchPath(ESearchPath search, int directoryId) const
{
	EqStringRef searchPath;
	switch (search)
	{
		case SP_DATA:
			searchPath = fnmPathCombine(m_basePath, m_dataDir);
			break;
		case SP_MOD:
			if(directoryId == -1) // default write path
				searchPath = fnmPathCombine(m_basePath, GetCurrentGameDirectory());
			else
				searchPath = fnmPathCombine(m_basePath, m_directories[directoryId].path);
			break;
		case SP_ROOT:
			searchPath = m_basePath;
			break;
		default:
			searchPath = ".";
			break;
	}

	return searchPath;
}

static bool UTIL_IsAbsolutePath(const char* dirOrFileName)
{
	return (dirOrFileName[0] == CORRECT_PATH_SEPARATOR || CType::IsAlphabetic(dirOrFileName[0]) && dirOrFileName[1] == ':');
}

EqStringRef CFileSystem::GetAbsolutePath(ESearchPath search, const char* dirOrFileName) const
{
	EqStringRef fullPath;
	if (search == SP_DIRECT || search == SP_ROOT && UTIL_IsAbsolutePath(dirOrFileName))
	{
		fullPath = EqStringRef::GetTempString(dirOrFileName);
		fnmPathFixSeparators(fullPath);
	}
	else
	{
		fullPath = fnmPathCombine(GetSearchPath(search), dirOrFileName);
	}

	return fullPath;
}

EqStringRef CFileSystem::GetAbsoluteFullPath(ESearchPath search, const char* dirOrFileName) const
{
	constexpr int MAX_PATH_LEN = 1024;

	EqString fullPath = GetAbsolutePath(search, dirOrFileName);

#ifdef _WIN32

	EqWString fullWPath;
	AnsiUnicodeConverter(fullWPath, fullPath);

	wchar_t fullAbsWPath[MAX_PATH_LEN]{ 0 };
	GetFullPathNameW(fullWPath, MAX_PATH_LEN, fullAbsWPath, nullptr);

	AnsiUnicodeConverter(fullPath, fullAbsWPath);
#else
	// like GetFullPathName: lexical normalization, path does not have to exist
	if (fullPath.Length() && fullPath[0] != CORRECT_PATH_SEPARATOR)
	{
		char cwdPath[MAX_PATH_LEN];
		if (getcwd(cwdPath, sizeof(cwdPath)))
			fullPath = fnmPathCombine(cwdPath, fullPath);
	}

	Array<EqStringRef> pathParts(PP_SL);
	int partStart = 0;
	for (int i = 0; i <= fullPath.Length(); ++i)
	{
		if (i < fullPath.Length() && fullPath[i] != CORRECT_PATH_SEPARATOR)
			continue;

		const EqStringRef part(fullPath.ToCString() + partStart, i - partStart);
		partStart = i + 1;

		if (part.Length() == 0 || part == ".")
			continue;

		if (part == "..")
		{
			if (pathParts.numElem())
				pathParts.popBack();
			continue;
		}

		pathParts.append(part);
	}

	EqString normalizedPath;
	for (const EqStringRef part : pathParts)
	{
		normalizedPath.Append(CORRECT_PATH_SEPARATOR);
		normalizedPath.Append(part);
	}

	if (!normalizedPath.Length())
		normalizedPath.Append(CORRECT_PATH_SEPARATOR);

	fullPath = normalizedPath;
#endif

	// fullPath is local, return it from temp string storage
	return EqStringRef::GetTempString(fullPath.ToCString(), fullPath.Length());
}

bool CFileSystem::FileRemove(const char* filename, ESearchPath search ) const
{
	const EqStringRef path = GetAbsolutePath(search, filename);

	return remove(path) == 0;
}

bool CFileSystem::DirExist(const char* dirname, ESearchPath search) const
{
	const EqStringRef path = GetAbsolutePath(search, dirname);

	struct stat statDir;
	return fast_stat(path, &statDir) == 0 && (statDir.st_mode & S_IFDIR);
}

//Directory operations
bool CFileSystem::MakeDir(const char* dirname, ESearchPath search ) const
{
	const EqStringRef path = GetAbsolutePath(search, dirname);

	return mkdirRecursive(path, true);
}

bool CFileSystem::RemoveDir(const char* dirname, ESearchPath search ) const
{
	const EqStringRef path = GetAbsolutePath(search, dirname);

    return rmdir(path) == 0;
}

bool CFileSystem::Rename(const char* oldNameOrPath, const char* newNameOrPath, ESearchPath search) const
{
	const EqStringRef oldPath = GetAbsolutePath(search, oldNameOrPath);
	const EqStringRef newPath = GetAbsolutePath(search, newNameOrPath);

	return rename(oldPath, newPath) == 0;
}

bool CFileSystem::WalkOverSearchPaths(int searchFlags, const char* fileName, const SPWalkFunc& func) const
{
	int flags = searchFlags;
	if (flags == -1)
		flags = SP_MOD | SP_DATA | SP_ROOT;

	const bool isAbsolutePath = UTIL_IsAbsolutePath(fileName);
	if (isAbsolutePath)
		flags = SP_ROOT;

	EqStringRef filePath;

	// First we checking mod directory
	if (flags & SP_MOD)
	{
#ifdef USE_PATH_TO_FILE_MAPPING
		const FSSearchPathInfo* foundSpInfo = nullptr;
		for (const FSSearchPathInfo& spInfo : m_directories)
		{
			if (spInfo.pathToFileMapping.isEmpty())
				continue;

			filePath = fnmPathCombine(m_basePath, spInfo.path, fileName);
			const int nameHash = FSStringId(filePath);

			const auto it = spInfo.pathToFileMapping.find(nameHash);
			if (it)
			{
				// fill will be open directly
				filePath = *it;
				flags |= SP_IGNORE_PACKAGE;
				foundSpInfo = &spInfo;
				break;
			}
		}

		if (foundSpInfo && func(filePath, SP_MOD, *foundSpInfo, flags))
			return true;
#endif

		for (const FSSearchPathInfo& spInfo : m_directories)
		{
			filePath = fnmPathCombine(m_basePath, spInfo.path, fileName);
			if (func(filePath, SP_MOD, spInfo, flags))
				return true;
		}
	}

	//Then we checking data directory
	if (flags & SP_DATA)
	{
		FSSearchPathInfo dataSpInfo;
		filePath = fnmPathCombine(m_basePath, m_dataDir, fileName);
		if (func(filePath, SP_DATA, dataSpInfo, flags))
			return true;
	}

	// And checking root.
	// not adding basepath to this
	if (flags & SP_ROOT)
	{
		FSSearchPathInfo rootSpInfo;
		rootSpInfo.writePath = true;

		if(isAbsolutePath)
		{
			filePath = fileName;
			fnmPathFixSeparators(filePath);
		}
		else
			filePath = fnmPathCombine(m_basePath, fileName);
		
		// TODO: write path detection if it's same as ones from m_directories or m_dataDir
		if (func(filePath, SP_ROOT, rootSpInfo, flags))
			return true;
	}

	return false;
}

bool CFileSystem::SetAccessKey(const char* accessKey)
{
	m_accessKey = accessKey;
	return m_accessKey.Length() > 0;
}

bool CFileSystem::AddPackage(const char* packageName, ESearchPath type, const char* mountPath /*= nullptr*/)
{
	const EqStringRef packagePath = GetAbsolutePath(SP_ROOT, packageName);

	for(IPackFileReaderPtr package : m_fsPackages)
	{
		if(!packagePath.CompareCaseIns(package->GetName()))
		{
			ASSERT_FAIL("Package %s was already added to file system", packageName);
			return false;
		}
	}

	CBasePackageReaderPtr reader = CBasePackageReader::CreateReaderByExtension(packageName);
	if (!reader->InitPackage(packagePath, packageName, mountPath))
	{
		MsgError("Cannot open package '%s'\n", packagePath.ToCString());
		return false;
	}

	if(mountPath)
		DevMsg(DEVMSG_FS, "Adding package file '%s' force mount at '%s'\n", packageName, mountPath);
	else
		DevMsg(DEVMSG_FS, "Adding package file '%s'\n", packageName);

    reader->SetSearchPath(type);
	reader->SetKey(m_accessKey);

    m_fsPackages.append(IPackFileReaderPtr(reader));
    return true;
}

void CFileSystem::RemovePackage(const char* packageName)
{
	const EqStringRef packagePath = GetAbsolutePath(SP_ROOT, packageName);

	for (int i = 0; i < m_fsPackages.numElem(); i++)
	{
		IPackFileReader* reader = m_fsPackages[i];
		if (!packagePath.CompareCaseIns(reader->GetName()))
		{
			m_fsPackages.fastRemoveIndex(i);
			return;
		}
	}
}

// opens package for further reading. Does not add package as FS layer
IPackFileReaderPtr CFileSystem::OpenPackage(const char* packageName, int searchFlags)
{
	CRefPtr<CBasePackageReader> reader = CBasePackageReader::CreateReaderByExtension(packageName);

	auto walkFileFunc = [&](EqStringRef filePath, ESearchPath searchPath, const FSSearchPathInfo& spInfo, int spFlags) -> bool
	{
		if (reader->InitPackage(filePath, packageName, nullptr))
			return true;

		if (DirExist(filePath, SP_DIRECT))
		{
			// open flat reader
			reader = CBasePackageReaderPtr(CRefPtr_new(CFlatFileReader));
			reader->InitPackage(filePath, packageName, nullptr);
			return true;
		}

		if (searchFlags & SP_IGNORE_PACKAGE)
			return false;

		const EqStringRef filePathNoBase = filePath.ToCString() + m_basePath.Length();

		// If failed to load directly, load it from package, in backward order
		for (int j = m_fsPackages.numElem() - 1; j >= 0; j--)
		{
			CBasePackageReader* fsPacakage = static_cast<CBasePackageReader*>(m_fsPackages[j].Ptr());
			if (!(spFlags & fsPacakage->GetSearchPath()))
				continue;

			EqString pkgFileName;
			if (!fsPacakage->GetInternalFileName(pkgFileName, filePathNoBase))
				continue;

			// trying to open package file inside package (embedded DPK)
			if (fsPacakage->OpenEmbeddedPackage(reader, pkgFileName))
				return true;
		}

		return false;
	};

	if (!WalkOverSearchPaths(searchFlags, packageName, walkFileFunc))
		return nullptr;

	if (!reader->GetFileCount())
		return nullptr;

	reader->SetSearchPath(SP_ROOT);
	reader->SetKey(m_accessKey);

	return IPackFileReaderPtr(reader);
}

// sets fallback directory for mod
void CFileSystem::AddSearchPath(const char* pathId, const char* pszDir)
{
	const int idx = arrayFindIndexF(m_directories, [=](const FSSearchPathInfo& spInfo) {
		return spInfo.id == pathId;
	});
	if (idx != -1)
	{
		ErrorMsg("AddSearchPath Error: pathId %s already added", pathId);
		return;
	}

	DevMsg(DEVMSG_FS, "Adding search path '%s' at '%s'\n", pathId, pszDir);

	const bool isReadPriorityPath = CString::SubString(pathId, "$MOD$") || CString::SubString(pathId, "$LOCALIZE$");
	const bool isWriteablePath = CString::SubString(pathId, "$WRITE$");
	const bool isWritePath = !isReadPriorityPath || isWriteablePath;

	int pathInfoIdx = -1;
	{
		FSSearchPathInfo pathInfo;
		pathInfo.id = pathId;
		pathInfo.path = pszDir;
		pathInfo.writePath = isWritePath;
		if(isReadPriorityPath)
			pathInfoIdx = m_directories.insert(pathInfo, 0);
		else
			pathInfoIdx = m_directories.append(pathInfo);
	}

#ifdef USE_PATH_TO_FILE_MAPPING
	if(!isWritePath)
		FSSearchPathInfo::MapFiles(m_basePath, m_directories[pathInfoIdx]);
#endif
}

void CFileSystem::RemoveSearchPath(const char* pathId)
{
	for(int i = 0; i < m_directories.numElem(); i++)
	{
		if(m_directories[i].id == EqStringRef(pathId))
		{
			DevMsg(DEVMSG_FS, "Removing search path '%s'\n", pathId);
			m_directories.removeIndex(i);
			return;
		}
	}
}

// Returns current game path
const char* CFileSystem::GetCurrentGameDirectory() const
{
	// return first directory with 'writePath' attribute set
	for (const FSSearchPathInfo& spInfo : m_directories)
	{
		if (spInfo.writePath)
			return spInfo.path;
	}

    return m_dataDir.GetData();
}

// Returns current engine data path
const char* CFileSystem::GetCurrentDataDirectory() const
{
    return m_dataDir.GetData();
}

bool CFileSystem::InitNextPath(FSFindData* findData) const
{
	EqStringRef fsBaseDir;
	EqStringRef searchWildcard;

	// Try Game paths
	if (findData->searchPaths & SP_MOD)
	{
		while (findData->dirIndex < m_directories.numElem())
		{
			fsBaseDir = GetSearchPath(SP_MOD, findData->dirIndex++);
			searchWildcard = fnmPathCombine(fsBaseDir, findData->wildcard);
			if (findData->osFind.Init(searchWildcard))
				return true;

			if(findData->singleDir)
				break;
		}
		findData->searchPaths &= ~SP_MOD;
	}
	
	// Try Data
	if (findData->searchPaths & SP_DATA)
	{
		findData->searchPaths &= ~SP_DATA;
		fsBaseDir = GetSearchPath(SP_DATA, -1);
		searchWildcard = fnmPathCombine(fsBaseDir, findData->wildcard);
		if (findData->osFind.Init(searchWildcard))
			return true;
	}

	// Try Root
	if (findData->searchPaths & SP_ROOT)
	{
		findData->searchPaths &= ~SP_ROOT;
		fsBaseDir = GetSearchPath(SP_ROOT, -1);
		searchWildcard = fnmPathCombine(fsBaseDir, findData->wildcard);
		if (findData->osFind.Init(searchWildcard))
			return true;
	}

	return false;
}

// opens directory for search props
const char* CFileSystem::FindFirst(const char* wildcard, FSFindData** findData, int searchPath, int dirIndex)
{
	ASSERT(findData != nullptr);

	if(findData == nullptr)
		return nullptr;

	ASSERT_MSG(searchPath, "searchPath flags must be specified");

	FSFindData* newFind = PPNew FSFindData;
	newFind->searchPaths = searchPath;
	newFind->wildcard = wildcard;
	fnmPathFixSeparators(newFind->wildcard);
	newFind->dirIndex = max(0, dirIndex);
	newFind->singleDir = (dirIndex >= 0);

	if (!InitNextPath(newFind))
	{
		delete newFind;
		return nullptr;
	}

	m_findDatas.append(newFind);
	*findData = newFind;

	return newFind->osFind.GetCurrentPath();
}

const char* CFileSystem::FindNext(FSFindData* findData) const
{
	if(!findData)
		return nullptr;

	if (findData->osFind.GetNext())
		return findData->osFind.GetCurrentPath();

	// if GetNext failed, reinit in new search path
	if (!InitNextPath(findData))
		return nullptr;

	return findData->osFind.GetCurrentPath();
}

void CFileSystem::FindClose(FSFindData* findData)
{
	if(!findData)
		return;
	if(m_findDatas.fastRemove(findData))
		delete findData;
}

bool CFileSystem::FindIsDirectory(FSFindData* findData) const
{
	if(!findData)
		return false;
	return findData->osFind.IsDirectory();
}

int	CFileSystem::FindGetDirIndex(FSFindData* findData) const
{
	if(!findData)
		return -1;
	return findData->dirIndex - 1;
}