Enhance logging in FdsSqlOptions and related classes
- Updated FdsSqlOptions to accept an optional ILogger parameter for improved error logging. - Modified FdsMfr and FdsMfrClient classes to pass the logger instance to FdsSqlOptions. - Added detailed error logging in various methods to capture SQL execution issues and file handling errors. - Improved documentation for FdsSqlOptions to clarify logging behavior. - Updated Archive class to log compression errors, enhancing traceability of failures. - Adjusted project configuration to suppress specific warnings related to transitive dependencies. - Added NuGet.config to define package sources for dependency management. - Updated submodule references for OCORE and related projects.
This commit is contained in:
Stefan
@@ -1,5 +1,6 @@
|
||||
using System.Data;
|
||||
using System.Diagnostics;
|
||||
using System.Linq;
|
||||
using CAMTParser;
|
||||
using Fuchs.Observability;
|
||||
using Microsoft.Extensions.Logging;
|
||||
@@ -35,6 +36,7 @@ public class BankingService : IBankingService
|
||||
using var act = FuchsTelemetry.StartActivity("banking.parse");
|
||||
var sw = Stopwatch.StartNew();
|
||||
var tbl = schemaDatatable?.Clone() ?? BuildDefaultSchema();
|
||||
var diag = new ParseDiagnostics();
|
||||
|
||||
// Buffer once so we can sniff the format and (re)parse from the bytes.
|
||||
byte[] bytes;
|
||||
@@ -48,46 +50,125 @@ public class BankingService : IBankingService
|
||||
if (CamtParser.LooksLikeZip(bytes))
|
||||
{
|
||||
format = "camt.zip";
|
||||
try { MapCamtEntries(tbl, new CamtParser().ParseZip(bytes)); }
|
||||
try
|
||||
{
|
||||
var statements = new CamtParser().ParseZip(bytes, out var skippedEntries);
|
||||
if (skippedEntries.Count > 0)
|
||||
{
|
||||
diag.ZipEntriesSkipped = skippedEntries.Count;
|
||||
_logger.LogWarning(
|
||||
"CAMT ZIP: {Count} entry(ies) skipped: {Entries}",
|
||||
skippedEntries.Count, string.Join("; ", skippedEntries));
|
||||
}
|
||||
MapCamtEntries(tbl, statements, diag);
|
||||
}
|
||||
catch (Exception ex) { _logger.LogError(ex, "CAMT ZIP statement parse failed."); }
|
||||
}
|
||||
else if (CamtParser.LooksLikeXml(bytes))
|
||||
{
|
||||
format = "camt";
|
||||
try { MapCamtEntries(tbl, new CamtParser().Parse(bytes)); }
|
||||
try { MapCamtEntries(tbl, new CamtParser().Parse(bytes), diag); }
|
||||
catch (Exception ex) { _logger.LogError(ex, "CAMT statement parse failed."); }
|
||||
}
|
||||
else
|
||||
{
|
||||
format = "mt940";
|
||||
using var msMt = new MemoryStream(bytes);
|
||||
FillFromMt940(tbl, msMt);
|
||||
FillFromMt940(tbl, msMt, diag);
|
||||
}
|
||||
|
||||
tbl.AcceptChanges();
|
||||
sw.Stop();
|
||||
|
||||
FuchsTelemetry.Mt940RowsParsed.Add(tbl.Rows.Count, new KeyValuePair<string, object?>("format", format));
|
||||
if (diag.StatementsSkippedNoAccount > 0)
|
||||
FuchsTelemetry.BankingEntriesSkipped.Add(diag.StatementsSkippedNoAccount,
|
||||
new KeyValuePair<string, object?>("reason", "noAccount"));
|
||||
if (diag.EntriesSkippedError > 0)
|
||||
FuchsTelemetry.BankingEntriesSkipped.Add(diag.EntriesSkippedError,
|
||||
new KeyValuePair<string, object?>("reason", "error"));
|
||||
if (diag.ZipEntriesSkipped > 0)
|
||||
FuchsTelemetry.BankingEntriesSkipped.Add(diag.ZipEntriesSkipped,
|
||||
new KeyValuePair<string, object?>("reason", "zipEntry"));
|
||||
if (diag.FieldsTruncated > 0)
|
||||
FuchsTelemetry.BankingFieldsTruncated.Add(diag.FieldsTruncated);
|
||||
|
||||
act?.SetTag("fuchs.banking.format", format);
|
||||
act?.SetTag("fuchs.banking.rows", tbl.Rows.Count);
|
||||
_logger.LogInformation("Bank statement parsed: format={Format} rows={Rows} in {Ms} ms",
|
||||
format, tbl.Rows.Count, sw.ElapsedMilliseconds);
|
||||
act?.SetTag("fuchs.banking.statements_skipped_no_account", diag.StatementsSkippedNoAccount);
|
||||
act?.SetTag("fuchs.banking.entries_skipped_error", diag.EntriesSkippedError);
|
||||
act?.SetTag("fuchs.banking.fields_truncated", diag.FieldsTruncated);
|
||||
|
||||
// A statement/upload that yields zero rows almost always means the import silently
|
||||
// failed upstream (wrong account element for this bank's schema variant, empty file,
|
||||
// unsupported CAMT flavor) rather than that the statement legitimately had no bookings.
|
||||
// Surface that as a warning so it doesn't require deliberately grepping info-level logs.
|
||||
var logLevel = tbl.Rows.Count == 0 ? LogLevel.Warning : LogLevel.Information;
|
||||
_logger.Log(logLevel,
|
||||
"Bank statement parsed: format={Format} rows={Rows} statementsSkippedNoAccount={StatementsSkipped} " +
|
||||
"entriesSkippedError={EntriesSkipped} zipEntriesSkipped={ZipSkipped} fieldsTruncated={Truncated} in {Ms} ms",
|
||||
format, tbl.Rows.Count, diag.StatementsSkippedNoAccount, diag.EntriesSkippedError,
|
||||
diag.ZipEntriesSkipped, diag.FieldsTruncated, sw.ElapsedMilliseconds);
|
||||
|
||||
if (diag.TruncatedByColumn.Count > 0)
|
||||
_logger.LogWarning("Bank statement fields truncated to column width: {Columns}",
|
||||
string.Join(", ", diag.TruncatedByColumn.Select(kv => $"{kv.Key}×{kv.Value}")));
|
||||
|
||||
return tbl;
|
||||
}
|
||||
|
||||
// ── MT940 ─────────────────────────────────────────────────────────────────
|
||||
private void FillFromMt940(DataTable tbl, Stream stream)
|
||||
/// <summary>Per-parse counters used to summarize what got dropped or altered, so a single log line can explain a zero- or low-row result.</summary>
|
||||
private sealed class ParseDiagnostics
|
||||
{
|
||||
void SetNfo(DataRow nr, string key, object? value)
|
||||
public int StatementsSkippedNoAccount;
|
||||
public int EntriesSkippedError;
|
||||
public int ZipEntriesSkipped;
|
||||
public int FieldsTruncated;
|
||||
public readonly Dictionary<string, int> TruncatedByColumn = new();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Assigns a value to a row cell, but only if the column exists and the value
|
||||
/// is non-null. String values are truncated to the column's <see cref="DataColumn.MaxLength"/>
|
||||
/// so that an over-long field (e.g. a remittance line, a long counterparty name, or a
|
||||
/// foreign IBAN) can never overflow the destination column. Without this guard, a single
|
||||
/// over-long value throws on assignment and — because the per-entry mapping is wrapped in a
|
||||
/// catch — silently drops the whole transaction, which can empty an entire import.
|
||||
/// Truncations are counted in <paramref name="diag"/> rather than logged per-cell, to avoid
|
||||
/// flooding the log on a file with many long fields.
|
||||
/// </summary>
|
||||
private static void SetCell(DataTable tbl, DataRow nr, string key, object? value, ParseDiagnostics diag)
|
||||
{
|
||||
if (value == null || !tbl.Columns.Contains(key)) return;
|
||||
var col = tbl.Columns[key]!;
|
||||
if (col.DataType == typeof(string) && col.MaxLength > 0 &&
|
||||
value is string s && s.Length > col.MaxLength)
|
||||
{
|
||||
if (tbl.Columns.Contains(key) && value != null) nr[key] = value;
|
||||
value = s[..col.MaxLength];
|
||||
diag.FieldsTruncated++;
|
||||
diag.TruncatedByColumn[key] = diag.TruncatedByColumn.GetValueOrDefault(key) + 1;
|
||||
}
|
||||
nr[key] = value;
|
||||
}
|
||||
|
||||
// ── MT940 ─────────────────────────────────────────────────────────────────
|
||||
private void FillFromMt940(DataTable tbl, Stream stream, ParseDiagnostics diag)
|
||||
{
|
||||
void SetNfo(DataRow nr, string key, object? value) => SetCell(tbl, nr, key, value, diag);
|
||||
|
||||
using var ps = new Parser(stream: stream);
|
||||
try
|
||||
{
|
||||
foreach (var statement in ps.Parse())
|
||||
{
|
||||
if (string.IsNullOrEmpty(statement.AccountIdentification)) continue;
|
||||
if (string.IsNullOrEmpty(statement.AccountIdentification))
|
||||
{
|
||||
diag.StatementsSkippedNoAccount++;
|
||||
_logger.LogWarning(
|
||||
"MT940 statement skipped: no AccountIdentification ({LineCount} line(s) dropped).",
|
||||
statement.Lines.Count);
|
||||
continue;
|
||||
}
|
||||
foreach (var line in statement.Lines)
|
||||
{
|
||||
try
|
||||
@@ -128,7 +209,13 @@ public class BankingService : IBankingService
|
||||
|
||||
tbl.Rows.Add(nr);
|
||||
}
|
||||
catch (Exception ex) { _logger.LogWarning(ex, "MT940 line parse error — account={Account}", statement.AccountIdentification); }
|
||||
catch (Exception ex)
|
||||
{
|
||||
diag.EntriesSkippedError++;
|
||||
_logger.LogWarning(ex,
|
||||
"MT940 line parse error — account={Account} entryDate={EntryDate} amount={Amount}: dropped.",
|
||||
statement.AccountIdentification, line.EntryDate, line.Amount);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -136,16 +223,20 @@ public class BankingService : IBankingService
|
||||
}
|
||||
|
||||
// ── CAMT (ISO 20022) ───────────────────────────────────────────────────────
|
||||
private void MapCamtEntries(DataTable tbl, List<CamtStatement> statements)
|
||||
private void MapCamtEntries(DataTable tbl, List<CamtStatement> statements, ParseDiagnostics diag)
|
||||
{
|
||||
void SetNfo(DataRow nr, string key, object? value)
|
||||
{
|
||||
if (tbl.Columns.Contains(key) && value != null) nr[key] = value;
|
||||
}
|
||||
void SetNfo(DataRow nr, string key, object? value) => SetCell(tbl, nr, key, value, diag);
|
||||
|
||||
foreach (var stmt in statements)
|
||||
{
|
||||
if (string.IsNullOrEmpty(stmt.AccountIdentification)) continue;
|
||||
if (string.IsNullOrEmpty(stmt.AccountIdentification))
|
||||
{
|
||||
diag.StatementsSkippedNoAccount++;
|
||||
_logger.LogWarning(
|
||||
"CAMT statement skipped: no AccountIdentification ({EntryCount} entrie(s) dropped, docType={DocType}).",
|
||||
stmt.Entries.Count, stmt.DocumentType);
|
||||
continue;
|
||||
}
|
||||
foreach (var e in stmt.Entries)
|
||||
{
|
||||
try
|
||||
@@ -155,7 +246,10 @@ public class BankingService : IBankingService
|
||||
if (e.Amount.HasValue) SetNfo(nr, "Amount", e.Amount);
|
||||
if (e.EntryDate.HasValue) SetNfo(nr, "EntryDate", e.EntryDate);
|
||||
if (e.ValueDate.HasValue) SetNfo(nr, "ValueDate", e.ValueDate);
|
||||
SetNfo(nr, "FundsCode", e.Currency);
|
||||
// FundsCode is a single-character MT940 funds code (VARCHAR(1)); CAMT has no
|
||||
// equivalent, so it is left unset. The ISO currency (e.Currency, e.g. "EUR")
|
||||
// must NOT be written here — it overflows the 1-char column and, before the
|
||||
// width guard in SetCell, silently dropped every CAMT transaction.
|
||||
SetNfo(nr, "DebitCreditMark", e.MarkAbbreviation);
|
||||
SetNfo(nr, "BankReference", e.BankReference);
|
||||
SetNfo(nr, "EndToEndReference", e.EndToEndReference);
|
||||
@@ -175,7 +269,13 @@ public class BankingService : IBankingService
|
||||
|
||||
tbl.Rows.Add(nr);
|
||||
}
|
||||
catch (Exception ex) { _logger.LogWarning(ex, "CAMT entry parse error — account={Account}", stmt.AccountIdentification); }
|
||||
catch (Exception ex)
|
||||
{
|
||||
diag.EntriesSkippedError++;
|
||||
_logger.LogWarning(ex,
|
||||
"CAMT entry parse error — account={Account} entryDate={EntryDate} amount={Amount}: dropped.",
|
||||
stmt.AccountIdentification, e.EntryDate, e.Amount);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
using System.Data;
|
||||
using System.Diagnostics;
|
||||
using Fuchs.intranet;
|
||||
using Fuchs.Notifications;
|
||||
using Fuchs.Observability;
|
||||
using Microsoft.Data.SqlClient;
|
||||
using Microsoft.Extensions.Logging;
|
||||
@@ -23,14 +24,16 @@ public class InvoiceService : IInvoiceService
|
||||
private readonly Fuchs_intranet _intranet;
|
||||
private readonly IPdfService _pdf;
|
||||
private readonly IBlobStorageService _blobStorage;
|
||||
private readonly IEventService _events;
|
||||
private readonly ILogger<InvoiceService> _logger;
|
||||
|
||||
public InvoiceService(Fuchs_intranet intranet, IPdfService pdf, IBlobStorageService blobStorage,
|
||||
ILogger<InvoiceService> logger)
|
||||
IEventService events, ILogger<InvoiceService> logger)
|
||||
{
|
||||
_intranet = intranet;
|
||||
_pdf = pdf;
|
||||
_blobStorage = blobStorage;
|
||||
_events = events;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
@@ -150,6 +153,7 @@ public class InvoiceService : IInvoiceService
|
||||
string fileName = invoice.InvoiceRegistration?.getString("DocumentName")
|
||||
.ne($"Rechnung_{invoice.Id}.pdf") ?? $"Rechnung_{invoice.Id}.pdf";
|
||||
await _blobStorage.UploadInvoicePdfAsync(invoice.Id, fileName, ba, invoice.InvoiceRegistration);
|
||||
await _events.InvoiceFileCreatedAsync(invoice, fileName, userAccountId);
|
||||
return ba;
|
||||
}
|
||||
|
||||
|
||||
@@ -234,16 +234,22 @@ public class ProcessWebComService : IComService
|
||||
}
|
||||
}
|
||||
|
||||
private static string BuildSignature()
|
||||
private string BuildSignature()
|
||||
{
|
||||
string sigPath = Path.Combine(AppContext.BaseDirectory,
|
||||
"email_signature", "sanitaerfuchs_email_signature.txt");
|
||||
try
|
||||
{
|
||||
string sigPath = Path.Combine(AppContext.BaseDirectory,
|
||||
"email_signature", "sanitaerfuchs_email_signature.txt");
|
||||
if (File.Exists(sigPath))
|
||||
return SignatureIntro + File.ReadAllText(sigPath);
|
||||
}
|
||||
catch { /* signature is optional */ }
|
||||
catch (Exception ex)
|
||||
{
|
||||
// The signature is optional (emails still send without it), but a read failure here
|
||||
// usually means a misconfigured deployment (permissions, locked file) that would
|
||||
// otherwise go unnoticed indefinitely — every email would just quietly lack a signature.
|
||||
_logger.LogWarning(ex, "Failed to read email signature file at {SignaturePath}", sigPath);
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
using System.Data;
|
||||
using System.Diagnostics;
|
||||
using Fuchs.intranet;
|
||||
using Fuchs.Notifications;
|
||||
using Fuchs.Observability;
|
||||
using Microsoft.Data.SqlClient;
|
||||
using Microsoft.Extensions.Logging;
|
||||
@@ -23,14 +24,16 @@ public class ReminderService : IReminderService
|
||||
private readonly Fuchs_intranet _intranet;
|
||||
private readonly IPdfService _pdf;
|
||||
private readonly IBlobStorageService _blobStorage;
|
||||
private readonly IEventService _events;
|
||||
private readonly ILogger<ReminderService> _logger;
|
||||
|
||||
public ReminderService(Fuchs_intranet intranet, IPdfService pdf, IBlobStorageService blobStorage,
|
||||
ILogger<ReminderService> logger)
|
||||
IEventService events, ILogger<ReminderService> logger)
|
||||
{
|
||||
_intranet = intranet;
|
||||
_pdf = pdf;
|
||||
_blobStorage = blobStorage;
|
||||
_events = events;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
@@ -153,6 +156,7 @@ public class ReminderService : IReminderService
|
||||
string fileName = reminder.ReminderRegistration?.getString("DocumentName")
|
||||
.ne($"Zahlungserinnerung_{reminder.Id}.pdf") ?? $"Zahlungserinnerung_{reminder.Id}.pdf";
|
||||
await _blobStorage.UploadReminderPdfAsync(reminder.Id, fileName, ba, reminder.ReminderRegistration);
|
||||
await _events.ReminderFileCreatedAsync(reminder, fileName, userAccountId);
|
||||
return ba;
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user