feat: Implement mfr truncation fix and enhance invoice handling
Playwright Tests / test (push) Canceled after 0s

- Added a new SQL script to address the "String or binary data would be truncated" error in mfr sync by widening the Note fields in service objects and companies to NVARCHAR(MAX).
- Updated the table types and associated stored procedures to accommodate the changes, ensuring compatibility with existing data.
- Queued a re-sync for previously rejected rows due to truncation issues.
- Created application logins for fuchs_fds and fuchs_web with placeholder passwords for initial setup.
- Introduced functions for invoice backdating and invoice date retrieval to streamline invoice processing.
- Developed a password generation function with customizable parameters for length and character types.
- Added a random number generation function to support password creation.
This commit is contained in:
Stefan
2026-09-30 16:56:08 +02:00
parent 7357f60120
commit c741535aec
79 changed files with 2739 additions and 143 deletions
+101
View File
@@ -0,0 +1,101 @@
using System.Data;
using System.Globalization;
namespace fds;
/// <summary>
/// Explains an mfr merge that failed with "String or binary data would be truncated": SQL Server
/// (message 8152, or 2628 below compatibility level 150 / without VERBOSE_TRUNCATION_WARNINGS)
/// does not name the column, and the merge proc copies the staging table into both the
/// <c>mfr__tt__*</c> table type and the <c>mfr__*</c> target. After such a failure the staged
/// <see cref="DataTable"/> is compared against the narrower of the two declared widths, so the
/// error log names the column, the longest value and an entity Id.
/// Runs only on the failure path — no cost for a successful merge.
/// </summary>
internal static class FdsColumnWidths
{
/// <summary>One column whose staged values exceed the declared width.</summary>
internal sealed record Overflow(string Column, int MaxAllowed, int LongestLength, int RowsOver, object? LongestRowId)
{
public override string ToString() =>
string.Create(CultureInfo.InvariantCulture,
$"{Column}: {LongestLength} > {MaxAllowed} chars ({RowsOver} row{(RowsOver == 1 ? "" : "s")}{(LongestRowId is null ? "" : $", e.g. Id={LongestRowId}")})");
}
/// <summary>8152 = classic truncation message; 2628 = verbose variant (compat level 150+).</summary>
internal static bool IsTruncationError(int sqlErrorNumber) => sqlErrorNumber is 8152 or 2628;
internal static bool IsTruncationError(Exception? exc) =>
exc is Microsoft.Data.SqlClient.SqlException sx && IsTruncationError(sx.Number);
/// <summary>
/// Char width of every bounded string column of the target table and of its
/// <c>mfr__tt__</c> table type (same naming rule as the <c>mfr__updt__</c> merge proc);
/// where both declare the column the narrower wins. MAX columns (max_length = -1) are omitted.
/// Expects parameter <c>@tablename</c>.
/// </summary>
internal const string WidthsSql = """
SELECT [columnname] = c.[name],
[maxchars] = MIN(CASE WHEN c.[system_type_id] IN (231, 239) THEN c.[max_length] / 2 ELSE c.[max_length] END)
FROM sys.columns AS c
WHERE c.[system_type_id] IN (167, 175, 231, 239)
AND c.[max_length] > 0
AND (c.[object_id] = OBJECT_ID(N'[dbo].' + QUOTENAME(@tablename))
OR c.[object_id] IN (SELECT tt.[type_table_object_id] FROM sys.table_types AS tt
WHERE tt.[schema_id] = SCHEMA_ID(N'dbo')
AND tt.[name] = REPLACE(@tablename, N'__', N'__tt__')))
GROUP BY c.[name];
""";
/// <summary>Rows of <see cref="WidthsSql"/> → column name → max chars (case-insensitive).</summary>
internal static Dictionary<string, int> ToWidthMap(DataTable? widthRows)
{
var map = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase);
if (widthRows == null || !widthRows.Columns.Contains("columnname") || !widthRows.Columns.Contains("maxchars"))
return map;
foreach (DataRow row in widthRows.Rows)
{
if (row["columnname"] is not string name || string.IsNullOrWhiteSpace(name)) continue;
if (row["maxchars"] is DBNull) continue;
int width = Convert.ToInt32(row["maxchars"], CultureInfo.InvariantCulture);
if (width <= 0) continue;
map[name] = map.TryGetValue(name, out int existing) ? Math.Min(existing, width) : width;
}
return map;
}
/// <summary>
/// String columns of <paramref name="batch"/> holding at least one value longer than the
/// declared width, one entry per column with the longest value's length and row Id.
/// Columns without a known width (MAX or not declared) are never reported.
/// </summary>
internal static IReadOnlyList<Overflow> FindOverflows(DataTable batch, IReadOnlyDictionary<string, int> widths)
{
var result = new List<Overflow>();
if (batch.Rows.Count == 0 || widths.Count == 0) return result;
DataColumn? idColumn = batch.Columns.Contains("Id") ? batch.Columns["Id"] : null;
foreach (DataColumn col in batch.Columns)
{
if (col.DataType != typeof(string) || !widths.TryGetValue(col.ColumnName, out int maxAllowed)) continue;
int longest = 0, rowsOver = 0;
object? longestId = null;
foreach (DataRow row in batch.Rows)
{
if (row.RowState == DataRowState.Deleted || row[col] is not string s || s.Length <= maxAllowed) continue;
rowsOver++;
if (s.Length > longest)
{
longest = s.Length;
longestId = idColumn != null && row[idColumn] is not DBNull ? row[idColumn] : null;
}
}
if (rowsOver > 0) result.Add(new Overflow(col.ColumnName, maxAllowed, longest, rowsOver, longestId));
}
return result;
}
/// <summary>Log-friendly summary; "none detected" when the batch fits the declared widths.</summary>
internal static string Describe(IReadOnlyList<Overflow> overflows) =>
overflows.Count == 0 ? "none detected" : string.Join("; ", overflows);
}
+33 -4
View File
@@ -294,10 +294,10 @@ public class FdsMfrClient : IDisposable
dtwa.CommandAfterError = new SqlCommand(
$"DROP TABLE IF EXISTS [{fallbackTable}]; SELECT * INTO [{fallbackTable}] FROM {dtwa.DestinationTableName};")
{ CommandTimeout = MergeCommandTimeoutSeconds };
dtwa.OnCommandAfterError += (sender, caex) =>
_logger.LogError(caex,
"Update__entitytable: merge into {Table} failed ({Entity}, setid={SetId}, rows={Rows}) — staged batch kept in {FallbackTable}",
tbl.TableName, thisEntityName, setId, trc, fallbackTable);
// Logged after DoSubmit returns so a truncation can be explained
// (column + length + Id) with an awaited width lookup.
Exception? mergeError = null;
dtwa.OnCommandAfterError += (sender, caex) => mergeError = caex;
dtwa.OnError += (sender, exc, message) =>
_logger.LogError(exc,
"Update__entitytable: DatatableWriterAsync submission failed for {Table} ({Entity}, setid={SetId}) — {Message}",
@@ -319,6 +319,15 @@ public class FdsMfrClient : IDisposable
new("@tgtid", entityId.Length > 0 ? entityId[0].ToString() : (object)DBNull.Value)
});
dtwa.DoSubmit();
if (mergeError != null)
{
string overlong = FdsColumnWidths.IsTruncationError(mergeError)
? await DescribeOverlongColumnsAsync(tbl)
: "n/a";
_logger.LogError(mergeError,
"Update__entitytable: merge into {Table} failed ({Entity}, setid={SetId}, rows={Rows}) — staged batch kept in {FallbackTable}; over-long columns: {OverlongColumns}",
tbl.TableName, thisEntityName, setId, trc, fallbackTable, overlong);
}
}
catch (Exception dsex)
{
@@ -341,6 +350,26 @@ public class FdsMfrClient : IDisposable
return true;
}
/// <summary>
/// Failure path only: names the staged columns wider than the target table / table type allow.
/// Never throws — a failed lookup is reported in the returned text instead.
/// </summary>
private async Task<string> DescribeOverlongColumnsAsync(DataTable batch)
{
try
{
var widths = await getSQLDatatable_async(FdsColumnWidths.WidthsSql, FdsShared.FDSConnectionString(),
SqlParameterList: new List<SqlParameter> { new("@tablename", batch.TableName) },
options: new FdsSqlOptions(logger: _logger));
if (!string.IsNullOrEmpty(widths.Exception)) return "column widths unavailable: " + widths.Exception;
return FdsColumnWidths.Describe(FdsColumnWidths.FindOverflows(batch, FdsColumnWidths.ToWidthMap(widths.DataTable)));
}
catch (Exception ex)
{
return "column widths unavailable: " + ex.Message;
}
}
private static void SetRowValue(DataRow row, string key, object value)
{
if (!row.Table.Columns.Contains(key)) return;
+47
View File
@@ -4,6 +4,47 @@
<name>Fuchs_DataService</name>
</assembly>
<members>
<member name="T:fds.FdsColumnWidths">
<summary>
Explains an mfr merge that failed with "String or binary data would be truncated": SQL Server
(message 8152, or 2628 below compatibility level 150 / without VERBOSE_TRUNCATION_WARNINGS)
does not name the column, and the merge proc copies the staging table into both the
<c>mfr__tt__*</c> table type and the <c>mfr__*</c> target. After such a failure the staged
<see cref="T:System.Data.DataTable"/> is compared against the narrower of the two declared widths, so the
error log names the column, the longest value and an entity Id.
Runs only on the failure path — no cost for a successful merge.
</summary>
</member>
<member name="T:fds.FdsColumnWidths.Overflow">
<summary>One column whose staged values exceed the declared width.</summary>
</member>
<member name="M:fds.FdsColumnWidths.Overflow.#ctor(System.String,System.Int32,System.Int32,System.Int32,System.Object)">
<summary>One column whose staged values exceed the declared width.</summary>
</member>
<member name="M:fds.FdsColumnWidths.IsTruncationError(System.Int32)">
<summary>8152 = classic truncation message; 2628 = verbose variant (compat level 150+).</summary>
</member>
<member name="F:fds.FdsColumnWidths.WidthsSql">
<summary>
Char width of every bounded string column of the target table and of its
<c>mfr__tt__</c> table type (same naming rule as the <c>mfr__updt__</c> merge proc);
where both declare the column the narrower wins. MAX columns (max_length = -1) are omitted.
Expects parameter <c>@tablename</c>.
</summary>
</member>
<member name="M:fds.FdsColumnWidths.ToWidthMap(System.Data.DataTable)">
<summary>Rows of <see cref="F:fds.FdsColumnWidths.WidthsSql"/> → column name → max chars (case-insensitive).</summary>
</member>
<member name="M:fds.FdsColumnWidths.FindOverflows(System.Data.DataTable,System.Collections.Generic.IReadOnlyDictionary{System.String,System.Int32})">
<summary>
String columns of <paramref name="batch"/> holding at least one value longer than the
declared width, one entry per column with the longest value's length and row Id.
Columns without a known width (MAX or not declared) are never reported.
</summary>
</member>
<member name="M:fds.FdsColumnWidths.Describe(System.Collections.Generic.IReadOnlyList{fds.FdsColumnWidths.Overflow})">
<summary>Log-friendly summary; "none detected" when the batch fits the declared widths.</summary>
</member>
<member name="T:fds.FdsSqlOptions">
<summary>
Logs every SQL error through the worker's structured <see cref="T:Microsoft.Extensions.Logging.ILogger"/> alone — which, when
@@ -29,6 +70,12 @@
(empty) shape is loaded. Empty when the entity table itself is unknown.
</summary>
</member>
<member name="M:fds.FdsMfrClient.DescribeOverlongColumnsAsync(System.Data.DataTable)">
<summary>
Failure path only: names the staged columns wider than the target table / table type allow.
Never throws — a failed lookup is reported in the returned text instead.
</summary>
</member>
<member name="T:fds.FdsConfig">
<summary>
Holds the application <see cref="T:Microsoft.Extensions.Configuration.IConfiguration"/> supplied by the host (the Fuchs web app).