FileManager DataSource

Registration

// Handles CSV, TSV, JSON and Text out of the box.
// Plugin readers extend this list without changing the registration.
[AddinAttribute(
    Category       = DatasourceCategory.FILE,
    DatasourceType = DataSourceType.CSV | DataSourceType.TSV
                   | DataSourceType.Json | DataSourceType.Text,
    FileType       = "csv,tsv,json,txt")]
public partial class FileDataSource : IDataSource, IFileDataSourceReaderHost { }

Add a connection and open it

dmeEditor.ConfigEditor.AddDataConnection(new ConnectionProperties
{
    ConnectionName = "SalesData",
    FilePath       = @"C:\data",
    FileName       = "sales.csv",
    DatabaseType   = DataSourceType.CSV,
    DriverName     = "FileDataSource"   // classHandler value
});

var fs = dmeEditor.GetDataSource("SalesData") as FileDataSource;
fs.Openconnection();
// Openconnection() sequence:
//   1. FileReaderFactory.RegisterDefaults()        — ensures built-ins are available
//   2. activeReader = FileReaderFactory.GetReader(DatasourceType)
//   3. activeReader.Configure(connectionProperties)
//   4. HasHeader / ParseMode applied from connection props

Standard IDataSource CRUD

// All calls delegate to the active IFileFormatReader internally
EntityStructure entity  = fs.GetEntityStructure("sales.csv", refresh: false);
object          rows    = fs.GetEntity("sales.csv", filters);
IErrorsInfo     ins     = fs.InsertEntity("sales.csv", newRow);
IErrorsInfo     upd     = fs.UpdateEntities("sales.csv", fs.ErrorObject, updatedRow);
IErrorsInfo     del     = fs.DeleteEntity("sales.csv", keyFilter);

fs.Closeconnection();

public interface IFileFormatReader
{
    // Identity
    DataSourceType SupportedType { get; }
    string         GetDefaultExtension();

    // Configuration
    bool      HasHeader { get; set; }
    ParseMode ParseMode { get; set; }  // Lenient | Strict
    void      Configure(IConnectionProperties props);

    // Schema
    string[]        ReadHeaders(string filePath);
    EntityStructure GetEntityStructure(string filePath);

    // Read
    IEnumerable<string[]> ReadRows(string filePath);
    string InferFieldType(string current, string rawValue);

    // Write
    bool CreateFile  (string filePath, IReadOnlyList<string> headers);
    bool AppendRow   (string filePath, IReadOnlyList<string> headers, IReadOnlyList<string> values);
    bool RewriteFile (string filePath, IReadOnlyList<string> headers, IEnumerable<IReadOnlyList<string>> rows);

    // Diagnostics
    IReadOnlyList<RowDiagnostic> LastDiagnostics { get; }
    void ClearDiagnostics();
}

public enum ParseMode { Lenient, Strict }

public record RowDiagnostic(int RowIndex, DiagnosticSeverity Severity, string Message, string? RawLine = null);
public enum DiagnosticSeverity { Info, Warning, Error }

// Attribute properties
DataSourceType FormatType       // routes to this reader
string         DisplayName      // shown in connection manager UI
string         DefaultExtension // e.g. "csv"
string         Description
string         Author
string         Version
string         IconPath         // defaults to "{ext}.svg" if empty

Core API

// Register a reader (replaces any existing entry for the same DataSourceType)
FileReaderFactory.Register(IFileFormatReader reader);

// Scan the core assembly for [FileReaderAttribute] classes — safe to call many times
FileReaderFactory.RegisterDefaults();

// Resolve; falls back to CSV reader for unknown types
IFileFormatReader r = FileReaderFactory.GetReader(DataSourceType.CSV);

// Introspection
bool supported = FileReaderFactory.IsSupported(DataSourceType.Json);
IEnumerable<DataSourceType> all = FileReaderFactory.SupportedTypes;

RegisterWithConfig — also add the format to the connection manager

// Registers in the factory AND upserts a ConnectionDriversConfig into
// configEditor.DataDriversClasses so the format appears in the connection manager.
// classHandler is always set to "FileDataSource".
// Idempotent: duplicate DataSourceType entries are skipped.
FileReaderFactory.RegisterWithConfig(reader, dmeEditor.ConfigEditor);

// Example — third-party Parquet reader registered at startup:
var parquet = new ThirdParty.ParquetFileReader();
FileReaderFactory.RegisterWithConfig(parquet, dmeEditor.ConfigEditor);

BuildDriverConfig — inspect the generated ConnectionDriversConfig

ConnectionDriversConfig cfg = FileReaderFactory.BuildDriverConfig(reader);
// cfg.classHandler       = "FileDataSource"
// cfg.DatasourceCategory = DatasourceCategory.FILE
// cfg.DatasourceType     = reader.SupportedType
// cfg.extensionstoHandle = attr.DefaultExtension ?? reader.GetDefaultExtension()
// cfg.GuidID             = deterministic MD5 GUID from reader type name
// cfg.NeedDrivers        = false

Discover — the startup call

// Scans editor.assemblyHandler.ConfigEditor.FileReaderClasses,
// instantiates each reader, and calls RegisterWithConfig so the format
// also surfaces as a connection type in the connection manager.
var registry = new FileReaderRegistry(dmeEditor);
registry.Discover();   // call once after assemblies are loaded

Other registry API

IFileFormatReader              fresh   = registry.Create(DataSourceType.CSV);
IReadOnlyList<FileReaderDescriptor> all = registry.GetAll();
bool                           hasCsv  = registry.Contains(DataSourceType.CSV);

// Static — scan ONE assembly without ConfigEditor access (used by RegisterDefaults)
FileReaderRegistry.RegisterAttributedReadersFromAssembly(assembly);

IFileDataSourceReaderHost interface

public interface IFileDataSourceReaderHost
{
    // Enumerate
    IReadOnlyList<FileReaderDescriptorInfo> GetAvailableReaderDescriptors(bool discover = false);
    IReadOnlyList<DataSourceType>           GetAvailableReaderTypes();
    FileReaderDescriptorInfo?               GetCurrentReaderDescriptor();
    object?                                 GetCurrentReader();

    // Switch — all overloads return false + reason on failure
    bool TrySwitchReader(DataSourceType        targetType,            bool reconfigure, out string? reason);
    bool TrySwitchReader(string                implementationTypeName, bool reconfigure, out string? reason);
    bool TrySwitchReader(Guid                  readerId,              bool reconfigure, out string? reason);
    bool TrySwitchReaderByExtension(string     extension,             bool reconfigure, out string? reason);

    void ResetReaderSelection();
}

How reader selection works during Openconnection()

// Priority order:
//   1. Override set by TrySwitchReader*() — if one was called since last open
//   2. DatasourceType default from ConnectionProperties

// After a successful switch, DatasourceType is updated automatically.
// Switch is blocked while an active transaction / ingestion is in progress.

Switch by DataSourceType

var host = dmeEditor.GetDataSource("SalesData") as IFileDataSourceReaderHost;

if (host.TrySwitchReader(DataSourceType.TSV, reconfigure: true, out var reason))
    (host as FileDataSource).Openconnection();
else
    Console.WriteLine(reason);

Switch by implementation name (deterministic when multiple readers share a type)

host.TrySwitchReader(
    "DuckDBDataSourceCore.FileReaders.DuckDbCsvFileReader",
    reconfigure: true,
    out var reason);

Switch by ReaderId

var csvReaders = host.GetAvailableReaderDescriptors(discover: true)
    .Where(r => r.FormatType == DataSourceType.CSV)
    .ToList();

// e.g. present list to user, then switch to selected
host.TrySwitchReader(csvReaders[1].ReaderId, reconfigure: true, out _);

1) Implement & decorate

using TheTechIdea.Beep.FileManager.Attributes;
using TheTechIdea.Beep.FileManager.Readers;

[FileReader(DataSourceType.CSV, "My Fast CSV Reader", "csv",
    Description = "High-performance CSV parser",
    Version     = "1.0.0")]
public sealed class MyFastCsvReader : IFileFormatReader
{
    public DataSourceType SupportedType => DataSourceType.CSV;
    public bool HasHeader   { get; set; } = true;
    public ParseMode ParseMode { get; set; } = ParseMode.Lenient;
    public IReadOnlyList<RowDiagnostic> LastDiagnostics { get; } = new List<RowDiagnostic>();

    public string   GetDefaultExtension() => "csv";
    public void     Configure(IConnectionProperties props) { /* delimiter / encoding */ }
    public void     ClearDiagnostics() { }
    public string[] ReadHeaders(string filePath)  => new[] { "Id", "Name" };
    public EntityStructure GetEntityStructure(string filePath) => null;
    public IEnumerable<string[]> ReadRows(string filePath) { yield break; }
    public string InferFieldType(string current, string rawValue) => typeof(string).FullName;
    public bool CreateFile(string fp, IReadOnlyList<string> h)  => true;
    public bool AppendRow (string fp, IReadOnlyList<string> h, IReadOnlyList<string> v) => true;
    public bool RewriteFile(string fp, IReadOnlyList<string> h, IEnumerable<IReadOnlyList<string>> r) => true;
}

2) Register at startup

// Option A — manual (no assembly scanning needed)
FileReaderFactory.RegisterWithConfig(new MyFastCsvReader(), dmeEditor.ConfigEditor);

// Option B — automatic via registry (preferred for plugin assemblies)
var registry = new FileReaderRegistry(dmeEditor);
registry.Discover();   // picks up any [FileReader]-decorated class in loaded assemblies

3) Use it — switch to the new reader on an existing connection

var host = dmeEditor.GetDataSource("SalesData") as IFileDataSourceReaderHost;

var descriptor = host.GetAvailableReaderDescriptors(discover: false)
    .First(r => r.ImplementationTypeName == typeof(MyFastCsvReader).FullName);

host.TrySwitchReader(descriptor.ReaderId, reconfigure: true, out var reason);
(host as FileDataSource).Openconnection();

Full end-to-end with a plugin Parquet reader

// After assemblies are loaded
var registry = new FileReaderRegistry(dmeEditor);
registry.Discover();  // Parquet reader found, RegisterWithConfig called

// Format is now visible in the connection manager
var cfg = dmeEditor.ConfigEditor.DataDriversClasses
    .First(d => d.DatasourceType == DataSourceType.Parquet);
// cfg.classHandler == "FileDataSource"

dmeEditor.ConfigEditor.AddDataConnection(new ConnectionProperties
{
    ConnectionName = "warehouse",
    DriverName     = cfg.PackageName,
    FilePath       = @"C:\data\warehouse.parquet",
    DatabaseType   = DataSourceType.Parquet
});

var ds = dmeEditor.GetDataSource("warehouse");
ds.Openconnection();  // FileDataSource routes to ParquetFileReader

Open a CSV file and read rows

dmeEditor.ConfigEditor.AddDataConnection(new ConnectionProperties
{
    ConnectionName = "sales",
    FilePath       = @"C:\data",
    FileName       = "sales.csv",
    DatabaseType   = DataSourceType.CSV,
    DriverName     = "FileDataSource"
});

var fs = dmeEditor.GetDataSource("sales") as FileDataSource;
fs.Openconnection();

var entity = fs.GetEntityStructure("sales.csv", false);
var rows   = (IEnumerable<object>)fs.GetEntity("sales.csv", null);

fs.Closeconnection();

Inspect all registered readers

var host    = fs as IFileDataSourceReaderHost;
var readers = host.GetAvailableReaderDescriptors(discover: true);

foreach (var r in readers)
    Console.WriteLine($"{r.DisplayName} | {r.FormatType} | {r.DefaultExtension}");

Switch format without recreating the connection

// Currently CSV — switch to TSV reader
if (host.TrySwitchReaderByExtension("tsv", reconfigure: true, out var reason))
    fs.Openconnection();
else
    Console.WriteLine(reason);

Insert and update rows

fs.InsertEntity("sales.csv", new { OrderId = "ORD-001", Amount = 150.00 });
fs.UpdateEntities("sales.csv", fs.ErrorObject, new { OrderId = "ORD-001", Amount = 175.00 });
fs.DeleteEntity("sales.csv", new AppFilter { FieldName = "OrderId", Operator = "=", FilterValue = "ORD-001" });
fs.Closeconnection();