IOFinder.cs

using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;

namespace Chizl.SystemSearch
{
    public class IOFinder : IDisposable
    {
        private const int _maxFindingsCount = 10000;
        private static List<FileSystemWatcher> _watchers = new List<FileSystemWatcher>();

        // since there will be a lot of threading going on, these status need to be seen by all threads.
        private static bool disposedValue;

        protected virtual void Dispose(bool disposing)
        {
            if (!disposedValue)
            {
                if (disposing)
                    StopScan();

                Scanner.Dispose();
                disposedValue = true;
            }
        }
        public void Dispose()
        {
            Dispose(true);
            GC.SuppressFinalize(this);
        }
        public IOFinder()
        {
            SearchMessage.EventMessaging += SearchMessage_EventMessaging;
        }

        #region Private
        // we want a new instance of the scanner each time, to ensure we have a clean
        // cache, and to prevent any threading issues with multiple searches going on at once.
        internal SystemScan Scanner => new SystemScan();
        public ScanProperties Criteria => GlobalSettings.ScanSettings;
        // we want to run the scan async, but we want the search to
        // wait for the scan to complete before it starts searching the cache.
        private Task Scan(string[] drives, bool sendMsg, bool isRescan) => Scanner.ScanDrives(drives, sendMsg, isRescan);
        // we want to run the search async, but we want it to wait for
        // the scan to complete before it starts searching the cache.
        private Task Search(string[] drives, string searchCriteria, bool isRescan)
        {
            var searchTask = Task.Run(async () =>
            {
                if (!GlobalSettings.FullScanCompleted)
                {
                    SearchMessage.SendMsg(SearchMessageType.UpdateInProgress, $"Full scan hasn't been completed. Please wait...");
                    await Scan(drives, false, isRescan);
                }
            })
            .ContinueWith(previousTask =>
            {
                // synchronous
                SearchCache(drives, searchCriteria);
            });

            return searchTask;
        }
        private void StopWatchers()
        {
            if (_watchers.Count > 0)
            {
                foreach (var watcher in _watchers)
                {
                    watcher.Created -= OnCreated;
                    watcher.Deleted -= OnDeleted;
                    watcher.Renamed -= OnRenamed;
                }
                _watchers.Clear();
            }
        }
        /// <summary>
        /// Searches the cache for the given search criteria.  If the cache is not fully built, it will wait for the 
        /// scan to complete before searching.  Event messages will be sent to subscribers with results and status updates.
        /// </summary>
        /// <param name="drives"></param>
        /// <param name="searchCriteria"></param>
        /// <param name="sendMsg"></param>
        /// <returns></returns>
        private bool SearchCache(string[] drives, string searchCriteria, bool sendMsg = true)
        {
            var retVal = false;

            if (string.IsNullOrWhiteSpace(searchCriteria))
            {
                SearchMessage.SendMsg(SearchMessageType.ScanComplete, $"Cached: [{SystemScan.ScannedFolders.FormatByComma()}] Folders, [{SystemScan.ScannedFiles.FormatByComma()}] Files.");
                return retVal;
            }

            // build the search criteria commands, this will help with the
            // search, and prevent us from having to parse the criteria multiple times.
            var buildSearchCriteria = new BuildSearchCmd(ref searchCriteria);

            if (sendMsg)
            {
                SearchMessage.SendMsg(SearchMessageType.UpdateInProgress, $"Search cache for '{searchCriteria}'.");
                SearchMessage.SendMsg(SearchMessageType.SearchQueryUsed, searchCriteria);
            }

            // search the cache for the criteria, this will return a list of files that match the
            // criteria, but we still need to verify them against the criteria, as some of the
            // criteria is not able to be pre-filtered in the cache.
            retVal = DeepDive(drives, buildSearchCriteria);

            // send total count message, to ensure accuratness
            SearchMessage.SendMsg(SearchMessageType.ScanComplete, $"Cached: [{SystemScan.ScannedFolders.FormatByComma()}] Folders, [{SystemScan.ScannedFiles.FormatByComma()}] Files.");

            return retVal;
        }
        private bool DeepDive(string[] drives, BuildSearchCmd searchCriteria)
        {
            var retVal = false;
            var fullFileList = Scanner.FileDictionary;
            var fullFileListLen = fullFileList.Count();
            // Moved from List<string> to ConcurrentDictionary to prevent Duplication.
            // TryAdd() is faster than List.Contains() + List.Add().
            ConcurrentDictionary<string, bool> findingsDic = new ConcurrentDictionary<string, bool>();
            var filters = new List<(string Path, bool HasExt)>();

            // we want to loop through the search criteria, and filter down the list of files based on the criteria.
            var pathList = searchCriteria.Commands.Where(w => w.CommandType == CommandType.Includes).ToList();
            // we can pre-filter the list based on the extensions, this will help with the search, and prevent us from having to parse the criteria multiple times.
            var extList = searchCriteria.Commands.Where(w => w.CommandType == CommandType.Extensions).ToList();
            // we can pre-filter the list based on the excludes, this will help with the search, and prevent us from having to parse the criteria multiple times.
            var filterList = searchCriteria.Commands.Where(w => w.CommandType == CommandType.Excludes).ToList();

            // we want to loop through the search criteria, and filter down the list of files based on the criteria.
            for (int i = 0; i < searchCriteria.SearchCriteria.Length; i++)
            {
                if (GlobalSettings.HasShutdown)
                    break;

                var wc = searchCriteria.SearchCriteria[i];
                var prevDicCount = findingsDic.Count();

                // if the criteria is a path include, we want to filter the list based on the path, this
                // will help with the search, and prevent us from having to parse the criteria multiple times.
                if (wc.Length == 1 && wc.Equals(Seps.cIncludesPos.ToString()) && pathList.Count() > 0)
                {
                    searchCriteria.SearchCriteria[i] = "";
                    filters.Clear();
                    
                    var hasData = findingsDic.Count > 0;
                    // if we have content, we now need to filter down for each criteria.
                    foreach (var p in pathList)
                    {
                        // if data doesn't exists at the start, then we want to only add
                        // extensions, not filter out after the first extension is found.
                        if (hasData)
                        {
                            if (p.Search.ToLower() == Seps.sNOEXT.ToLower())
                                filters.AddRange(findingsDic.Where(w => !w.Value).Select(s => (s.Key, s.Value)));
                            else
                                filters.AddRange(findingsDic.Where(w => w.Key.ToLower().Contains(p.Search.ToLower())).Select(s => (s.Key, s.Value)));
                        }
                        else
                        {
                            if (p.Search.ToLower() == Seps.sNOEXT.ToLower())
                                foreach (var item in fullFileList.Where(w => !w.Value))
                                    findingsDic.TryAdd(item.Key, item.Value);
                            else
                                foreach (var item in fullFileList.Where(w => w.Key.ToLower().Contains(p.Search.ToLower())))
                                    findingsDic.TryAdd(item.Key, item.Value);
                        }
                    }

                    // if we had content before, we need to clear it out, and add the new filtered content
                    if (prevDicCount != 0)
                        findingsDic.Clear();

                    // if we have filters, we need to add them to the findings, this will be the new list of
                    // findings for the next criteria.
                    if (filters.Count() > 0)
                    {
                        foreach (var item in filters.ToList())
                            findingsDic.TryAdd(item.Path, item.HasExt);
                    }
                }
                // if the criteria is an extension, we want to filter the list based on the extension, this
                // will help with the search, and prevent us from having to parse the criteria multiple times.
                else if (wc.Length == 1 && wc.Equals(Seps.cExtPos.ToString()) && extList.Count() > 0)
                {
                    searchCriteria.SearchCriteria[i] = "";
                    filters.Clear();
                    var hasData = findingsDic.Count > 0;
                    // if we have content, we now need to filter down for each extension.
                    foreach (var e in extList)
                    {
                        // if data doesn't exists at the start, then we want to only add
                        // extensions, not filter out after the first extension is found.
                        if (hasData)
                        {
                            if(e.Search==Seps.cNOEXT.ToString())
                                filters.AddRange(findingsDic.Where(w => !w.Value).Select(s => (s.Key, s.Value)));
                            else
                                filters.AddRange(findingsDic.Where(w => w.Key.ToLower().EndsWith(e.Search.ToLower())).Select(s => (s.Key, s.Value)));
                        }
                        else
                        {
                            if (e.Search == Seps.cNOEXT.ToString())
                                foreach (var item in fullFileList.Where(w => !w.Value))
                                    findingsDic.TryAdd(item.Key, item.Value);
                            else
                                foreach (var item in fullFileList.Where(w => w.Key.ToLower().EndsWith(e.Search.ToLower())))
                                    findingsDic.TryAdd(item.Key, item.Value);
                        }
                    }

                    // if we had content before, we need to clear it out, and add the new filtered content
                    if (prevDicCount != 0)
                        findingsDic.Clear();

                    if (filters.Count() > 0)
                    {
                        foreach (var item in filters.ToList())
                            findingsDic.TryAdd(item.Path, item.HasExt);
                    }
                }
                // if the criteria is a path exclude, we want to filter the list based on the path, this
                // will help with the search, and prevent us from having to parse the criteria multiple times.
                else if (wc.Length == 1 && wc.Equals(Seps.cFilterPos.ToString()) && filterList.Count() > 0)
                {
                    searchCriteria.SearchCriteria[i] = "";
                    filters.Clear();

                    foreach (var f in filterList)
                    {
                        // if we have content, we now need to filter down for each exclusion.
                        // If we don't have content, we have nothing to exclude, so we skip.
                        if (findingsDic.Count > 0)
                        {
                            if (f.Search.ToLower() == Seps.sNOEXT.ToLower())
                                filters.AddRange(findingsDic.Where(w => w.Value).Select(s => (s.Key, s.Value)));
                            else
                                filters.AddRange(findingsDic.Where(w => !w.Key.ToLower().Contains(f.Search.ToLower())).Select(s => (s.Key, s.Value)));

                            if (filters.Count() > 0)
                            {
                                findingsDic.Clear();
                                foreach (var item in filters.ToList())
                                    findingsDic.TryAdd(item.Path, item.HasExt);
                                filters.Clear();
                            }
                        }
                        else
                        {
                            if (f.Search.ToLower() == Seps.sNOEXT.ToLower())
                            {
                                foreach (var item in fullFileList.Where(w => w.Value).Select(s => (s.Key, s.Value)).ToList())
                                    findingsDic.TryAdd(item.Key, item.Value);
                            }
                            else
                            {
                                foreach (var item in fullFileList.Where(w => !w.Key.ToLower().Contains(f.Search.ToLower())).ToList())
                                    findingsDic.TryAdd(item.Key, item.Value);
                            }
                        }
                    }
                }
                // if anything is earched, this is the general search, we want to filter the list
                // based on the search, this will help with the search, and prevent us from having
                // to parse the criteria multiple times.
                else
                {
                    filters.Clear();

                    // if we have content, we now need to filter down for each criteria.
                    if (findingsDic.Count > 0)
                        filters.AddRange(findingsDic.Where(w => w.Key.ToLower().Contains(wc.ToLower())).Select(s => (s.Key, s.Value)));
                    else
                    {
                        foreach (var item in fullFileList.Where(w => w.Key.ToLower().Contains(wc.ToLower())).ToList())
                            findingsDic.TryAdd(item.Key, item.Value);
                    }

                    if (filters.Count() > 0)
                    {
                        foreach (var item in filters.ToList())
                            findingsDic.TryAdd(item.Path, item.HasExt);
                    }
                }

                // at any time, we haven't found anything,
                // then we haven't met the criteria
                if (findingsDic.Count.Equals(0))
                    break;
            }

            var verifiedFiles = 0;
            var fileList = new List<string>();
            if (findingsDic != null && findingsDic.Count > 0)
            {
                retVal = true;
                foreach (var file in findingsDic.Keys)
                {
                    if (GlobalSettings.HasShutdown)
                        break;

                    if (VerifyCriteria(drives, file, searchCriteria))
                    {
                        if (verifiedFiles < _maxFindingsCount)
                            fileList.Add(file);
                        verifiedFiles++;
                    }
                }

                // Bulk send of all findings by split of '\n', instant...  Balances Windows with Linux strings.
                var arrData = string.Join("\n", fileList);
                SearchMessage.SendMsg(SearchMessageType.SearchResults, arrData);
                // send found message
                SearchMessage.SendMsg(SearchMessageType.SearchStatus, $"Filtered: {fileList.Count().FormatByComma()}, Total Found: {verifiedFiles.FormatByComma()}");
            }

            return retVal;
        }
        private bool VerifyCriteria(string[] drives, string file, BuildSearchCmd searchCriteria)
        {
            var fileDrive = file.Substring(0, 2).ToLower();
            var fileName = Path.GetFileName(file).ToLower();
            var folderName = Path.GetDirectoryName(file).ToLower();
            var fileExt = Path.GetExtension(file).ToLower();
            var findCount = 0;
            var loc = 0;
            var prevLoc = 0;

            if (drives.Where(w => w.Substring(0, 2).Equals(file.Substring(0, 2), StringComparison.CurrentCultureIgnoreCase)).Count().Equals(0))
                return false;

            if (Criteria.SearchDirectory)
            {
                foreach (var sArr in searchCriteria.SearchCriteria)
                {
                    if (string.IsNullOrWhiteSpace(sArr))
                        continue;

                    // met criteria, get out.
                    if (findCount >= searchCriteria.SearchCriteria.Length)
                        break;
                    var clnUp = sArr.EndsWith("\\") ? sArr.Substring(0, sArr.Length - 1) : sArr;
                    loc = folderName.IndexOf(sArr, prevLoc, StringComparison.CurrentCultureIgnoreCase);
                    if (loc.Equals(-1) && folderName.EndsWith(clnUp))
                        loc = folderName.IndexOf(clnUp, prevLoc, StringComparison.CurrentCultureIgnoreCase);

                    if (loc < prevLoc)
                    {
                        if (Criteria.SearchFilename)
                            break;

                        return false;
                    }
                    else
                    {
                        prevLoc = loc + 1;
                        findCount++;
                    }
                }
            }
            if (Criteria.SearchFilename && findCount < searchCriteria.SearchCriteria.Length)
            {
                prevLoc = 0;
                foreach (var sArr in searchCriteria.SearchCriteria.Skip(findCount))
                {
                    if (string.IsNullOrWhiteSpace(sArr))
                        continue;

                    // met criteria, get out.
                    if (findCount >= searchCriteria.SearchCriteria.Length)
                        break;

                    loc = fileName.IndexOf(sArr, prevLoc, StringComparison.CurrentCultureIgnoreCase);
                    if (loc < prevLoc)
                        return false;
                    else
                    {
                        prevLoc = loc + 1;
                        findCount++;
                    }
                }
            }

            return true;
        }
        private void SearchMessage_EventMessaging(object sender, SearchEventArgs e) => EventMessaging?.Invoke(this, e);
        private void OnCreated(object sender, FileSystemEventArgs e)
        {
            if (e.ChangeType != WatcherChangeTypes.Created)
                return;

            if (!GlobalSettings.ScanSettings.AllowDir(e.FullPath))
                return;

            List<Task> createList = new List<Task>();
            // we found if a folder is being added, files are missed, so lets sleep a sec and see if that helps.
            Tools.Sleep(100, SleepType.Milliseconds);

            if (File.Exists(e.FullPath))
            {
                createList.Add(Task.Run(() => { Scanner.AddFile(e.FullPath); return Task.CompletedTask; }));
                SearchMessage.SendMsg(SearchMessageType.Info, $"Added: [{e.FullPath}] to cache.");
            }
            else if (Directory.Exists(e.FullPath))
            {
                createList.AddRange(Scanner.ScanSubFolders(new string[] { e.FullPath }, false));
                if (createList.Count > 0)
                    SearchMessage.SendMsg(SearchMessageType.Info, $"Adding: [{createList.Count}] files to cache.");
            }

            Task.WaitAll(createList.ToArray());

            SearchMessage.SendMsg(SearchMessageType.FileScanStatus, $"Cached: [{SystemScan.ScannedFolders.FormatByComma()}] Folders, [{SystemScan.ScannedFiles.FormatByComma()}] Files.");
        }
        private void OnDeleted(object sender, FileSystemEventArgs e)
        {
            if (e.ChangeType != WatcherChangeTypes.Deleted)
                return;

            if (!GlobalSettings.ScanSettings.AllowDir(e.FullPath))
                return;

            List<Task> removeList = new List<Task>();
            // we found if a folder is being added, files are missed, so lets sleep a sec and see if that helps.
            Tools.Sleep(100, SleepType.Milliseconds);

            var dirWithSlash = e.FullPath.EndsWith("\\") ? e.FullPath : $"{e.FullPath}\\";
            var isDir = false;
            var isFile = false;
            if (Scanner.IsDirectory(dirWithSlash))
                isDir = true;
            else
                isFile = true;

            if (isDir)
                removeList.AddRange(Scanner.RemoveRootFolder(dirWithSlash));
            else if (isFile)
                removeList.Add(Task.Run(() => { Scanner.RemoveFile(e.FullPath); return Task.CompletedTask; }));

            Task.WaitAll(removeList.ToArray());

            SearchMessage.SendMsg(SearchMessageType.FileScanStatus, $"Cached: [{SystemScan.ScannedFolders.FormatByComma()}] Folders, [{SystemScan.ScannedFiles.FormatByComma()}] Files.");
        }
        private void OnRenamed(object sender, RenamedEventArgs e)
        {
            if (e.ChangeType != WatcherChangeTypes.Renamed)
                return;

            if (!GlobalSettings.ScanSettings.AllowDir(e.OldFullPath))
                return;

            var fileGoodToGo = false;
            var removedFolders = 0;
            var addFolders = 0;

            Tools.Sleep(100, SleepType.Milliseconds);
            var isDir = Scanner.IsDirectory(e.OldFullPath);
            List<Task> renameList = new List<Task>();

            if (isDir)
                renameList.AddRange(Scanner.RemoveRootFolder(e.OldFullPath, false));
            else
                renameList.Add(Task.Run(() => { Scanner.RemoveFile(e.OldFullPath, false); return Task.CompletedTask; }));

            removedFolders = renameList.Count;
            fileGoodToGo = removedFolders > 0;

            if (isDir)
                renameList.AddRange(Scanner.ScanSubFolders(new string[] { e.FullPath }, false));
            else
                renameList.Add(Task.Run(() => { Scanner.AddFile(e.FullPath); return Task.CompletedTask; }));

            addFolders = renameList.Count - removedFolders;
            fileGoodToGo = fileGoodToGo || addFolders > 0;

            if (fileGoodToGo)
            {
                var info = $"[{e.OldFullPath}]  ->  [{e.FullPath}]";
                var vInfo = info.SplitByStr("->");
                if (e.FullPath.Length > 100 && vInfo.Length == 2)
                {
                    SearchMessage.SendMsg(SearchMessageType.Info, $"Renamed: {vInfo[0].Trim()}");
                    SearchMessage.SendMsg(SearchMessageType.Info, $"To   ->: {vInfo[1].Trim()}");
                }
                else
                    SearchMessage.SendMsg(SearchMessageType.Info, $"Renamed: {info}");

                Tools.Sleep(1);
            }

            Task.WaitAll(renameList.ToArray());
            SearchMessage.SendMsg(SearchMessageType.FileScanStatus, $"Cached: [{SystemScan.ScannedFolders.FormatByComma()}] Folders, [{SystemScan.ScannedFiles.FormatByComma()}] Files");
        }
        public void SetupWatcher(DriveInfo[] drives)
        {
            StopWatchers();
            foreach (var drive in drives.Select(s => (s.Name.EndsWith("\\") ? s.Name : $"{s.Name}\\")))
            {
                var watcher = new FileSystemWatcher(drive)
                {
                    NotifyFilter = NotifyFilters.DirectoryName |
                                   NotifyFilters.FileName |
                                   NotifyFilters.Attributes
                };

                watcher.Created += OnCreated;
                watcher.Deleted += OnDeleted;
                watcher.Renamed += OnRenamed;

                watcher.Filter = "*.*";
                watcher.IncludeSubdirectories = true;
                watcher.EnableRaisingEvents = true;

                _watchers.Add(watcher);
            }
        }
        #endregion

        #region Public Properties
        public event SearchEventHandler EventMessaging;
        public LookupStatus CurrentStatus => GlobalSettings.CurrentStatus;
        public bool FullScanCompleted => GlobalSettings.FullScanCompleted;
        #endregion

        #region Public Methods
        /// <summary>
        /// Scans all drives based on allow filters set in options.  Event messages will be sent to subscribers.
        /// </summary>
        /// <param name="reScan">true: Will clear the cache and start over.  false: will leave, but rescan for any new files to add to the cache.</param>
        public Task ScanToCache(bool isRescan = false) => Scan(GlobalSettings.DriveList, true, isRescan);
        public Task ScanToCache(DriveInfo drive, bool isRescan = false) => Scan(new string[] { drive.Name }, true, isRescan);
        public Task ScanToCache(DriveInfo[] drives, bool isRescan = false) => Scan(drives.Select(w => w.Name).ToArray(), true, isRescan);

        //public Task AddDrive(DriveInfo drive) => Task.Run(() => GlobalSettings.AddRemove($"{drive.Name}{(drive.Name.EndsWith("\\") ? "" : "\\")}", true));
        //public Task RemoveDrive(DriveInfo drive) => Task.Run(() => GlobalSettings.AddRemove($"{drive.Name}{(drive.Name.EndsWith("\\") ? "" : "\\")}", false));
        public Task AddDrive(DriveInfo drive) => ScanToCache(drive, false);
        public Task RemoveDrive(DriveInfo drive) => Task.Run(() => GlobalSettings.AddRemove($"{drive.Name}{(drive.Name.EndsWith("\\") ? "" : "\\")}", false));
        public Task<bool> AddScanExclusion(string pathOrFileContains) => Task.Run(() => {
                GlobalSettings.AddRemove($"{pathOrFileContains}", false); 
                return GlobalSettings.CustomExclusions.TryAdd(pathOrFileContains, true); 
            });
        public Task<string[]> GetScanExclusions() => Task.Run(() => { return GlobalSettings.CustomExclusions.Select(s=>s.Key).ToArray(); });
        public Task<bool> RemoveScanExclusion(string pathOrFileContains) => Task.Run(() => { return GlobalSettings.CustomExclusions.TryRemove(pathOrFileContains, out _); });

        /// <summary>
        /// Starts search in cache if exists.
        /// </summary>
        /// <param name="drive">1 Drive to search</param>
        /// <param name="searchCriteria">Query with search extensions accepted</param>
        /// <returns>Task for wait results, all results are sent as Events</returns>
        public Task Search(DriveInfo drive, string searchCriteria) => Search(new string[] { drive.Name }, searchCriteria, false);
        /// <summary>
        /// Starts search in cache if exists.
        /// </summary>
        /// <param name="drives">All drives to search through</param>
        /// <param name="searchCriteria">Query with search extensions accepted</param>
        /// <returns>Task for wait results, all results are sent as Events</returns>
        public Task Search(DriveInfo[] drives, string searchCriteria) => Search(drives.Select(w => w.Name).ToArray(), searchCriteria, false);
        public void ResetCache() => Scanner.ResetCache();
        public void StopScan()
        {
            StopWatchers();
            GlobalSettings.Shutdown();
        }
        #endregion
    }
}