🌐 WebAPIDataSource

// Circuit breaker automatically handles failures
// - Closed: Normal operation
// - Open: Failing fast after repeated failures
// - Half-Open: Testing recovery

// The circuit breaker is automatically managed by WebAPIErrorHelper
// and integrated into all HTTP requests via WebAPIRequestHelper

// High-performance configuration for heavy workloads
var performanceConnection = new WebAPIConnectionProperties
{
    ConnectionName = "High Performance API",
    ConnectionString = "https://api.high-volume.com/v1",
    AuthType = AuthTypeEnum.ApiKey,
    ApiKey = "your-high-perf-key",
    
    // Aggressive performance settings
    TimeoutMs = 10000, // Quick timeout for fast APIs
    MaxRetries = 1, // Minimal retries for speed
    RetryIntervalMs = 500, // Fast retry
    MaxConcurrentRequests = 50, // High concurrency
    
    // Optimized caching
    EnableCaching = true,
    CacheExpiryMinutes = 5, // Short cache for fresh data
    
    // Aggressive rate limiting
    EnableRateLimit = true,
    RateLimitRequestsPerMinute = 1000,
    
    // Compression for bandwidth
    EnableCompression = true,
    
    // Minimal response processing
    ResponseFormat = "json",
    DataPath = "$", // Direct response, no nesting
    
    // Large page sizes
    DefaultPageSize = 500,
    MaxPageSize = 1000,
    
    UserAgent = "HighPerf-Client/1.0"
};

// Minimal headers for performance
performanceConnection.Headers.Clear();
performanceConnection.Headers.Add(new WebApiHeader { Headername = "Accept", Headervalue = "application/json" });
performanceConnection.Headers.Add(new WebApiHeader { Headername = "Accept-Encoding", Headervalue = "gzip, deflate" });

// Comprehensive validation example
public static WebAPIConnectionProperties CreateValidatedConnection(string configName)
{
    var connection = new WebAPIConnectionProperties();
    
    try
    {
        // Load configuration from various sources
        connection.ConnectionName = configName;
        connection.ConnectionString = Configuration.GetConnectionString(configName);
        connection.AuthType = Enum.Parse<AuthTypeEnum>(Configuration[$"{configName}:AuthType"]);
        
        // Set auth-specific properties
        switch (connection.AuthType)
        {
            case AuthTypeEnum.ApiKey:
                connection.ApiKey = Configuration[$"{configName}:ApiKey"];
                connection.ApiKeyHeader = Configuration[$"{configName}:ApiKeyHeader"] ?? "X-API-Key";
                break;
                
            case AuthTypeEnum.Basic:
                connection.UserID = Configuration[$"{configName}:Username"];
                connection.Password = Configuration[$"{configName}:Password"];
                break;
                
            case AuthTypeEnum.OAuth2:
                connection.ClientId = Configuration[$"{configName}:ClientId"];
                connection.ClientSecret = Configuration[$"{configName}:ClientSecret"];
                connection.TokenUrl = Configuration[$"{configName}:TokenUrl"];
                connection.Scope = Configuration[$"{configName}:Scope"];
                break;
        }
        
        // Apply environment-specific settings
        var environment = Configuration["Environment"];
        if (environment == "Production")
        {
            connection.TimeoutMs = 60000;
            connection.MaxRetries = 3;
            connection.ValidateServerCertificate = true;
        }
        else if (environment == "Development")
        {
            connection.TimeoutMs = 30000;
            connection.MaxRetries = 1;
            connection.IgnoreSSLErrors = true; // Only for dev!
        }
        
        // Validate the configuration
        if (!connection.ValidateConfiguration())
        {
            var errors = connection.GetValidationErrors();
            throw new ConfigurationException($"Invalid {configName} configuration: {errors}");
        }
        
        Logger.Information($"Successfully configured {configName} connection");
        return connection;
    }
    catch (Exception ex)
    {
        Logger.Error($"Failed to create {configName} connection: {ex.Message}");
        throw;
    }
}

// Usage
var githubConnection = CreateValidatedConnection("GitHub");
var stripeConnection = CreateValidatedConnection("Stripe");

// Implement custom authentication by extending WebAPIAuthenticationHelper
public class CustomAuthHelper : WebAPIAuthenticationHelper
{
    public override async Task<bool> EnsureAuthenticatedAsync()
    {
        // Custom authentication logic
        var token = await GetCustomToken();
        // Set authentication headers
        return true;
    }
}

// Always check error status after operations
var result = webApiDS.InsertEntity("Users", userData);
if (result.Flag == Errors.Failed)
{
    Logger.WriteLog($"Operation failed: {result.Message}");
    // Handle error appropriately
}

// Use try-catch for comprehensive error handling
try
{
    var data = webApiDS.GetEntity("Users", filters);
}
catch (HttpRequestException ex)
{
    Logger.WriteLog($"HTTP Error: {ex.Message}");
    // Handle network errors
}
catch (Exception ex)
{
    Logger.WriteLog($"Unexpected error: {ex.Message}");
    // Handle other errors
}

// Always open connection before use
if (webApiDS.ConnectionStatus != ConnectionState.Open)
{
    var result = webApiDS.Openconnection();
    if (result == ConnectionState.Open)
    {
        // Connection successful
    }
}

// Close connection when done
webApiDS.Closeconnection();

// Use using statement for automatic disposal
using (var ds = new WebAPIDataSource(name, logger, editor, type, errors))
{
    // Use data source
} // Automatically disposed

// Comprehensive troubleshooting helper
public static class WebAPITroubleshooter
{
    public static void DiagnoseConnection(WebAPIConnectionProperties connection)
    {
        Console.WriteLine("=== WebAPI Connection Diagnostics ===");
        
        // Basic validation
        Console.WriteLine($"Connection Name: {connection.ConnectionName ?? "Not Set"}");
        Console.WriteLine($"Connection String: {connection.ConnectionString ?? "Not Set"}");
        Console.WriteLine($"Auth Type: {connection.AuthType}");
        
        // Validate configuration
        if (connection.ValidateConfiguration())
        {
            Console.WriteLine("✓ Configuration validation passed");
        }
        else
        {
            Console.WriteLine($"✗ Configuration errors: {connection.GetValidationErrors()}");
        }
        
        // Check authentication setup
        CheckAuthentication(connection);
        
        // Check network settings
        CheckNetworkSettings(connection);
        
        // Check performance settings
        CheckPerformanceSettings(connection);
    }
    
    private static void CheckAuthentication(WebAPIConnectionProperties connection)
    {
        Console.WriteLine("\n--- Authentication Check ---");
        
        switch (connection.AuthType)
        {
            case AuthTypeEnum.None:
                Console.WriteLine("✓ No authentication required");
                break;
                
            case AuthTypeEnum.ApiKey:
                Console.WriteLine($"API Key: {(!string.IsNullOrEmpty(connection.ApiKey) ? "Set" : "✗ Not Set")}");
                Console.WriteLine($"API Key Header: {connection.ApiKeyHeader}");
                break;
                
            case AuthTypeEnum.Basic:
                Console.WriteLine($"Username: {(!string.IsNullOrEmpty(connection.UserID) ? "Set" : "✗ Not Set")}");
                Console.WriteLine($"Password: {(!string.IsNullOrEmpty(connection.Password) ? "Set" : "✗ Not Set")}");
                break;
                
            case AuthTypeEnum.Bearer:
                Console.WriteLine($"Bearer Token: {(!string.IsNullOrEmpty(connection.ApiKey) ? "Set" : "✗ Not Set")}");
                break;
                
            case AuthTypeEnum.OAuth2:
                Console.WriteLine($"Client ID: {(!string.IsNullOrEmpty(connection.ClientId) ? "Set" : "✗ Not Set")}");
                Console.WriteLine($"Client Secret: {(!string.IsNullOrEmpty(connection.ClientSecret) ? "Set" : "✗ Not Set")}");
                Console.WriteLine($"Token URL: {connection.TokenUrl ?? "✗ Not Set"}");
                Console.WriteLine($"Grant Type: {connection.GrantType}");
                Console.WriteLine($"Scope: {connection.Scope ?? "Not Set"}");
                break;
        }
    }
    
    private static void CheckNetworkSettings(WebAPIConnectionProperties connection)
    {
        Console.WriteLine("\n--- Network Settings ---");
        Console.WriteLine($"Timeout: {connection.TimeoutMs}ms");
        Console.WriteLine($"Max Retries: {connection.MaxRetries}");
        Console.WriteLine($"Retry Interval: {connection.RetryIntervalMs}ms");
        Console.WriteLine($"SSL Validation: {(connection.ValidateServerCertificate ? "Enabled" : "Disabled")}");
        Console.WriteLine($"Ignore SSL Errors: {(connection.IgnoreSSLErrors ? "Yes" : "No")}");
        
        if (connection.UseProxy)
        {
            Console.WriteLine($"Proxy: {connection.ProxyUrl}:{connection.ProxyPort}");
            Console.WriteLine($"Proxy Credentials: {(connection.UseDefaultProxyCredentials ? "Default" : "Custom")}");
        }
        else
        {
            Console.WriteLine("Proxy: Not Used");
        }
    }
    
    private static void CheckPerformanceSettings(WebAPIConnectionProperties connection)
    {
        Console.WriteLine("\n--- Performance Settings ---");
        Console.WriteLine($"Caching: {(connection.EnableCaching ? "Enabled" : "Disabled")}");
        if (connection.EnableCaching)
        {
            Console.WriteLine($"Cache Expiry: {connection.CacheExpiryMinutes} minutes");
        }
        
        Console.WriteLine($"Rate Limiting: {(connection.EnableRateLimit ? "Enabled" : "Disabled")}");
        if (connection.EnableRateLimit)
        {
            Console.WriteLine($"Rate Limit: {connection.RateLimitRequestsPerMinute} requests/minute");
        }
        
        Console.WriteLine($"Max Concurrent Requests: {connection.MaxConcurrentRequests}");
        Console.WriteLine($"Compression: {(connection.EnableCompression ? "Enabled" : "Disabled")}");
        Console.WriteLine($"Default Page Size: {connection.DefaultPageSize}");
        Console.WriteLine($"Max Page Size: {connection.MaxPageSize}");
        
        Console.WriteLine($"\nHeaders ({connection.Headers.Count}):");
        foreach (var header in connection.Headers)
        {
            Console.WriteLine($"  {header.Headername}: {header.Headervalue}");
        }
    }
}

// Usage
var connection = new WebAPIConnectionProperties { /* your config */ };
WebAPITroubleshooter.DiagnoseConnection(connection);

// Force schema refresh
var structure = webApiDS.GetEntityStructure("EntityName", true);

// Manually define schema if inference fails
var customStructure = new EntityStructure
{
    EntityName = "CustomEntity",
    Fields = new List<EntityField> {
        new EntityField {FieldName = "id", Fieldtype = "System.Int32" },
        new EntityField {FieldName = "name", Fieldtype = "System.String" }
    }
};

// Before: Manual HTTP client usage
using (var client = new HttpClient())
{
    client.DefaultRequestHeaders.Add("Authorization", "Bearer " + token);
    var response = await client.GetAsync("https://api.example.com/users");
    var content = await response.Content.ReadAsStringAsync();
    // Manual JSON parsing and error handling...
}

// After: WebAPIDataSource usage
var users = webApiDS.GetEntity("Users", null);
// Automatic authentication, error handling, caching, and parsing

// Database-style operations work seamlessly
var filters = new List<AppFilter> {
    new AppFilter {FieldName = "status", FilterValue = "active", Operator = "=" }
};

var activeUsers = webApiDS.GetEntity("Users", filters); // Just like RDBSource!

// CRUD operations
webApiDS.InsertEntity("Users", newUser);
webApiDS.UpdateEntity("Users", updatedUser);
webApiDS.DeleteEntity("Users", userToDelete);

GET — raw response or deserialized

// Raw HttpResponseMessage
HttpResponseMessage resp = await ds.GetAsync(
    "users/me",
    query: new Dictionary<string, string> { ["fields"] = "id,name,email" },
    headers: new Dictionary<string, string> { ["Accept-Language"] = "en" }
);

// Deserialize directly to T
var me = await ds.GetAsync<UserProfile>("users/me");
// Returns default(T) on failure — never throws

POST / PUT / DELETE

// POST with a body
await ds.PostAsync("tweets", body: new { text = "Hello from Beep!" });

// PUT (full replace)
await ds.PutAsync($"users/{userId}", body: updatedUser);

// DELETE
await ds.DeleteAsync($"tweets/{tweetId}");

Minimum viable connector skeleton

// 1. Annotate with [AddinAttribute] so AssemblyHandler registers it automatically
[AddinAttribute(
    Category = DatasourceCategory.Connector,
    DatasourceType = DataSourceType.Twitter  // use the correct enum value for your API
)]
public class MyApiDataSource : WebAPIDataSource
{
    // 2. Map logical entity names to API endpoint paths
    private static readonly Dictionary<string, string> EntityEndpoints
        = new(StringComparer.OrdinalIgnoreCase)
    {
        ["items.list"]   = "items",
        ["items.by_id"]  = "items/{id}",
        ["items.search"] = "items/search"
    };

    // 3. Declare which AppFilter keys are required for each entity
    private static readonly Dictionary<string, string[]> RequiredFilters
        = new(StringComparer.OrdinalIgnoreCase)
    {
        ["items.by_id"]  = new[] { "id" },
        ["items.search"] = new[] { "query" }
    };

    public MyApiDataSource(
        string datasourcename, IDMLogger logger,
        IDMEEditor dmeEditor, DataSourceType databasetype, IErrorsInfo errorObject)
        : base(datasourcename, logger, dmeEditor, databasetype, errorObject)
    {
        EntitiesNames = EntityEndpoints.Keys.ToList();
        Entities = EntitiesNames
            .Select(n => new EntityStructure { EntityName = n, DatasourceEntityName = n })
            .ToList();
    }

    // 4. Override GetEntityAsync to drive URL resolution and response extraction
    public override async Task<IEnumerable<object>> GetEntityAsync(
        string EntityName, List<AppFilter> Filter)
    {
        if (!EntityEndpoints.TryGetValue(EntityName, out var endpoint))
            throw new InvalidOperationException($"Unknown entity '{EntityName}'");

        var q = FiltersToQuery(Filter);
        RequireFilters(EntityName, q,
            RequiredFilters.GetValueOrDefault(EntityName, Array.Empty<string>()));

        var resolvedEndpoint = ResolveEndpoint(endpoint, q);
        using var resp = await GetAsync(resolvedEndpoint, q);
        if (resp is null || !resp.IsSuccessStatusCode) return Array.Empty<object>();

        return ExtractArray(resp, "data");  // key in JSON root where items live
    }

    // 5. Expose sync overload
    public override IEnumerable<object> GetEntity(string EntityName, List<AppFilter> filter)
        => GetEntityAsync(EntityName, filter).GetAwaiter().GetResult() ?? Array.Empty<object>();
}

Register via ConfigEditor (recommended)

var conn = new WebAPIConnectionProperties
{
    ConnectionName   = "Twitter",
    ConnectionString = "https://api.twitter.com/2",
    DatabaseType     = DataSourceType.Twitter,
    Category         = DatasourceCategory.Connector,
    DriverName       = "TwitterDataSource",       // matches [AddinAttribute]

    // Authentication
    ApiKey           = "AAAA...",                 // Bearer token
    // Store auth type in Parameters (read by WebAPIAuthenticationHelper)
};
// Set bearer auth type in extra Parameters
conn.Parameters = "AuthType=Bearer";

// Add a default Authorization header directly (simplest approach)
conn.Headers.Add(new WebApiHeader
{
    Key   = "Authorization",
    Value = $"Bearer {bearerToken}"
});

dmeEditor.ConfigEditor.AddDataConnection(conn);

// Get the data source (AssemblyHandler resolves TwitterDataSource via [AddinAttribute])
var twitter = (TwitterDataSource)dmeEditor.GetDataSource("Twitter");
twitter.Openconnection();

Search recent tweets (IDataSource style)

var filters = new List<AppFilter>
{
    new AppFilter { FieldName = "query",       FilterValue = "BeepDM",  Operator = "=" },
    new AppFilter { FieldName = "max_results", FilterValue = "20",      Operator = "=" }
};

var raw = twitter.GetEntity("tweets.search", filters);
// Returns IEnumerable<object> (each item is a deserialized JSON object)

Get user's followers

var filters = new List<AppFilter>
{
    new AppFilter { FieldName = "id", FilterValue = "12345678", Operator = "=" }
};

var followers = twitter.GetEntity("users.followers", filters);

Get tweets for a list

var filters = new List<AppFilter>
{
    new AppFilter { FieldName = "id", FilterValue = "listId123", Operator = "=" },
    new AppFilter { FieldName = "max_results", FilterValue = "50", Operator = "=" }
};

var tweets = twitter.GetEntity("lists.tweets", filters);

Search tweets — typed

// Returns IEnumerable<TwitterTweet> (from Models.cs)
IEnumerable<TwitterTweet> tweets = await twitter.GetTweets(
    query: "#dotnet lang:en",
    maxResults: 25
);

foreach (var tweet in tweets)
{
    Console.WriteLine($"{tweet.CreatedAt:s}  @{tweet.AuthorId}");
    Console.WriteLine($"  {tweet.Text}");
    Console.WriteLine($"  Likes: {tweet.PublicMetrics?.LikeCount}  " +
                      $"RT: {tweet.PublicMetrics?.RetweetCount}");
}

User timeline

var timeline = await twitter.GetUserTimeline(
    userId: "12345678",
    maxResults: 50
);

var popular = timeline
    .OrderByDescending(t => t.PublicMetrics?.LikeCount ?? 0)
    .Take(5);

foreach (var t in popular)
    Console.WriteLine($"{t.PublicMetrics?.LikeCount,6} likes — {t.Text[..50]}");

User lookup

// By username
var users = await twitter.GetUserByUsername("beep_dm");
var user  = users.FirstOrDefault();
Console.WriteLine($"{user?.Name} — {user?.PublicMetrics?.FollowersCount} followers");

// By ID
var byId = await twitter.GetUserById("12345678");

Paged retrieval (IDataSource PagedResult)

var filters = new List<AppFilter>
{
    new AppFilter { FieldName = "query", FilterValue = "#csharp", Operator = "=" }
};

// Page 1 (first 20 results)
PagedResult page1 = twitter.GetEntity("tweets.search", filters, pageNumber: 1, pageSize: 20);
Console.WriteLine($"Page 1: {page1.Data.Count} items");
Console.WriteLine($"Has next: {page1.HasNextPage}");

// Page 2
PagedResult page2 = twitter.GetEntity("tweets.search", filters, pageNumber: 2, pageSize: 20);

Explicit cursor pass-through (typed method)

string? cursor = null;
do
{
    var batch = await twitter.GetTweets("#dotnet", maxResults: 100, nextToken: cursor);
    ProcessBatch(batch);

    // Extract next_token from the last raw page for the next call
    // (use typed method overload or inspect PagedResult.TotalPages instead)
    cursor = GetNextCursor(batch); // your own helper to read meta.next_token
}
while (cursor != null);