feat: Implement mfr truncation fix and enhance invoice handling
Playwright Tests / test (push) Canceled after 0s
Playwright Tests / test (push) Canceled after 0s
- Added a new SQL script to address the "String or binary data would be truncated" error in mfr sync by widening the Note fields in service objects and companies to NVARCHAR(MAX). - Updated the table types and associated stored procedures to accommodate the changes, ensuring compatibility with existing data. - Queued a re-sync for previously rejected rows due to truncation issues. - Created application logins for fuchs_fds and fuchs_web with placeholder passwords for initial setup. - Introduced functions for invoice backdating and invoice date retrieval to streamline invoice processing. - Developed a password generation function with customizable parameters for length and character types. - Added a random number generation function to support password creation.
This commit is contained in:
@@ -0,0 +1,101 @@
|
||||
using System.Data;
|
||||
using System.Globalization;
|
||||
|
||||
namespace fds;
|
||||
|
||||
/// <summary>
|
||||
/// Explains an mfr merge that failed with "String or binary data would be truncated": SQL Server
|
||||
/// (message 8152, or 2628 below compatibility level 150 / without VERBOSE_TRUNCATION_WARNINGS)
|
||||
/// does not name the column, and the merge proc copies the staging table into both the
|
||||
/// <c>mfr__tt__*</c> table type and the <c>mfr__*</c> target. After such a failure the staged
|
||||
/// <see cref="DataTable"/> is compared against the narrower of the two declared widths, so the
|
||||
/// error log names the column, the longest value and an entity Id.
|
||||
/// Runs only on the failure path — no cost for a successful merge.
|
||||
/// </summary>
|
||||
internal static class FdsColumnWidths
|
||||
{
|
||||
/// <summary>One column whose staged values exceed the declared width.</summary>
|
||||
internal sealed record Overflow(string Column, int MaxAllowed, int LongestLength, int RowsOver, object? LongestRowId)
|
||||
{
|
||||
public override string ToString() =>
|
||||
string.Create(CultureInfo.InvariantCulture,
|
||||
$"{Column}: {LongestLength} > {MaxAllowed} chars ({RowsOver} row{(RowsOver == 1 ? "" : "s")}{(LongestRowId is null ? "" : $", e.g. Id={LongestRowId}")})");
|
||||
}
|
||||
|
||||
/// <summary>8152 = classic truncation message; 2628 = verbose variant (compat level 150+).</summary>
|
||||
internal static bool IsTruncationError(int sqlErrorNumber) => sqlErrorNumber is 8152 or 2628;
|
||||
|
||||
internal static bool IsTruncationError(Exception? exc) =>
|
||||
exc is Microsoft.Data.SqlClient.SqlException sx && IsTruncationError(sx.Number);
|
||||
|
||||
/// <summary>
|
||||
/// Char width of every bounded string column of the target table and of its
|
||||
/// <c>mfr__tt__</c> table type (same naming rule as the <c>mfr__updt__</c> merge proc);
|
||||
/// where both declare the column the narrower wins. MAX columns (max_length = -1) are omitted.
|
||||
/// Expects parameter <c>@tablename</c>.
|
||||
/// </summary>
|
||||
internal const string WidthsSql = """
|
||||
SELECT [columnname] = c.[name],
|
||||
[maxchars] = MIN(CASE WHEN c.[system_type_id] IN (231, 239) THEN c.[max_length] / 2 ELSE c.[max_length] END)
|
||||
FROM sys.columns AS c
|
||||
WHERE c.[system_type_id] IN (167, 175, 231, 239)
|
||||
AND c.[max_length] > 0
|
||||
AND (c.[object_id] = OBJECT_ID(N'[dbo].' + QUOTENAME(@tablename))
|
||||
OR c.[object_id] IN (SELECT tt.[type_table_object_id] FROM sys.table_types AS tt
|
||||
WHERE tt.[schema_id] = SCHEMA_ID(N'dbo')
|
||||
AND tt.[name] = REPLACE(@tablename, N'__', N'__tt__')))
|
||||
GROUP BY c.[name];
|
||||
""";
|
||||
|
||||
/// <summary>Rows of <see cref="WidthsSql"/> → column name → max chars (case-insensitive).</summary>
|
||||
internal static Dictionary<string, int> ToWidthMap(DataTable? widthRows)
|
||||
{
|
||||
var map = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase);
|
||||
if (widthRows == null || !widthRows.Columns.Contains("columnname") || !widthRows.Columns.Contains("maxchars"))
|
||||
return map;
|
||||
foreach (DataRow row in widthRows.Rows)
|
||||
{
|
||||
if (row["columnname"] is not string name || string.IsNullOrWhiteSpace(name)) continue;
|
||||
if (row["maxchars"] is DBNull) continue;
|
||||
int width = Convert.ToInt32(row["maxchars"], CultureInfo.InvariantCulture);
|
||||
if (width <= 0) continue;
|
||||
map[name] = map.TryGetValue(name, out int existing) ? Math.Min(existing, width) : width;
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// String columns of <paramref name="batch"/> holding at least one value longer than the
|
||||
/// declared width, one entry per column with the longest value's length and row Id.
|
||||
/// Columns without a known width (MAX or not declared) are never reported.
|
||||
/// </summary>
|
||||
internal static IReadOnlyList<Overflow> FindOverflows(DataTable batch, IReadOnlyDictionary<string, int> widths)
|
||||
{
|
||||
var result = new List<Overflow>();
|
||||
if (batch.Rows.Count == 0 || widths.Count == 0) return result;
|
||||
DataColumn? idColumn = batch.Columns.Contains("Id") ? batch.Columns["Id"] : null;
|
||||
|
||||
foreach (DataColumn col in batch.Columns)
|
||||
{
|
||||
if (col.DataType != typeof(string) || !widths.TryGetValue(col.ColumnName, out int maxAllowed)) continue;
|
||||
int longest = 0, rowsOver = 0;
|
||||
object? longestId = null;
|
||||
foreach (DataRow row in batch.Rows)
|
||||
{
|
||||
if (row.RowState == DataRowState.Deleted || row[col] is not string s || s.Length <= maxAllowed) continue;
|
||||
rowsOver++;
|
||||
if (s.Length > longest)
|
||||
{
|
||||
longest = s.Length;
|
||||
longestId = idColumn != null && row[idColumn] is not DBNull ? row[idColumn] : null;
|
||||
}
|
||||
}
|
||||
if (rowsOver > 0) result.Add(new Overflow(col.ColumnName, maxAllowed, longest, rowsOver, longestId));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>Log-friendly summary; "none detected" when the batch fits the declared widths.</summary>
|
||||
internal static string Describe(IReadOnlyList<Overflow> overflows) =>
|
||||
overflows.Count == 0 ? "none detected" : string.Join("; ", overflows);
|
||||
}
|
||||
@@ -294,10 +294,10 @@ public class FdsMfrClient : IDisposable
|
||||
dtwa.CommandAfterError = new SqlCommand(
|
||||
$"DROP TABLE IF EXISTS [{fallbackTable}]; SELECT * INTO [{fallbackTable}] FROM {dtwa.DestinationTableName};")
|
||||
{ CommandTimeout = MergeCommandTimeoutSeconds };
|
||||
dtwa.OnCommandAfterError += (sender, caex) =>
|
||||
_logger.LogError(caex,
|
||||
"Update__entitytable: merge into {Table} failed ({Entity}, setid={SetId}, rows={Rows}) — staged batch kept in {FallbackTable}",
|
||||
tbl.TableName, thisEntityName, setId, trc, fallbackTable);
|
||||
// Logged after DoSubmit returns so a truncation can be explained
|
||||
// (column + length + Id) with an awaited width lookup.
|
||||
Exception? mergeError = null;
|
||||
dtwa.OnCommandAfterError += (sender, caex) => mergeError = caex;
|
||||
dtwa.OnError += (sender, exc, message) =>
|
||||
_logger.LogError(exc,
|
||||
"Update__entitytable: DatatableWriterAsync submission failed for {Table} ({Entity}, setid={SetId}) — {Message}",
|
||||
@@ -319,6 +319,15 @@ public class FdsMfrClient : IDisposable
|
||||
new("@tgtid", entityId.Length > 0 ? entityId[0].ToString() : (object)DBNull.Value)
|
||||
});
|
||||
dtwa.DoSubmit();
|
||||
if (mergeError != null)
|
||||
{
|
||||
string overlong = FdsColumnWidths.IsTruncationError(mergeError)
|
||||
? await DescribeOverlongColumnsAsync(tbl)
|
||||
: "n/a";
|
||||
_logger.LogError(mergeError,
|
||||
"Update__entitytable: merge into {Table} failed ({Entity}, setid={SetId}, rows={Rows}) — staged batch kept in {FallbackTable}; over-long columns: {OverlongColumns}",
|
||||
tbl.TableName, thisEntityName, setId, trc, fallbackTable, overlong);
|
||||
}
|
||||
}
|
||||
catch (Exception dsex)
|
||||
{
|
||||
@@ -341,6 +350,26 @@ public class FdsMfrClient : IDisposable
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Failure path only: names the staged columns wider than the target table / table type allow.
|
||||
/// Never throws — a failed lookup is reported in the returned text instead.
|
||||
/// </summary>
|
||||
private async Task<string> DescribeOverlongColumnsAsync(DataTable batch)
|
||||
{
|
||||
try
|
||||
{
|
||||
var widths = await getSQLDatatable_async(FdsColumnWidths.WidthsSql, FdsShared.FDSConnectionString(),
|
||||
SqlParameterList: new List<SqlParameter> { new("@tablename", batch.TableName) },
|
||||
options: new FdsSqlOptions(logger: _logger));
|
||||
if (!string.IsNullOrEmpty(widths.Exception)) return "column widths unavailable: " + widths.Exception;
|
||||
return FdsColumnWidths.Describe(FdsColumnWidths.FindOverflows(batch, FdsColumnWidths.ToWidthMap(widths.DataTable)));
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
return "column widths unavailable: " + ex.Message;
|
||||
}
|
||||
}
|
||||
|
||||
private static void SetRowValue(DataRow row, string key, object value)
|
||||
{
|
||||
if (!row.Table.Columns.Contains(key)) return;
|
||||
|
||||
@@ -4,6 +4,47 @@
|
||||
<name>Fuchs_DataService</name>
|
||||
</assembly>
|
||||
<members>
|
||||
<member name="T:fds.FdsColumnWidths">
|
||||
<summary>
|
||||
Explains an mfr merge that failed with "String or binary data would be truncated": SQL Server
|
||||
(message 8152, or 2628 below compatibility level 150 / without VERBOSE_TRUNCATION_WARNINGS)
|
||||
does not name the column, and the merge proc copies the staging table into both the
|
||||
<c>mfr__tt__*</c> table type and the <c>mfr__*</c> target. After such a failure the staged
|
||||
<see cref="T:System.Data.DataTable"/> is compared against the narrower of the two declared widths, so the
|
||||
error log names the column, the longest value and an entity Id.
|
||||
Runs only on the failure path — no cost for a successful merge.
|
||||
</summary>
|
||||
</member>
|
||||
<member name="T:fds.FdsColumnWidths.Overflow">
|
||||
<summary>One column whose staged values exceed the declared width.</summary>
|
||||
</member>
|
||||
<member name="M:fds.FdsColumnWidths.Overflow.#ctor(System.String,System.Int32,System.Int32,System.Int32,System.Object)">
|
||||
<summary>One column whose staged values exceed the declared width.</summary>
|
||||
</member>
|
||||
<member name="M:fds.FdsColumnWidths.IsTruncationError(System.Int32)">
|
||||
<summary>8152 = classic truncation message; 2628 = verbose variant (compat level 150+).</summary>
|
||||
</member>
|
||||
<member name="F:fds.FdsColumnWidths.WidthsSql">
|
||||
<summary>
|
||||
Char width of every bounded string column of the target table and of its
|
||||
<c>mfr__tt__</c> table type (same naming rule as the <c>mfr__updt__</c> merge proc);
|
||||
where both declare the column the narrower wins. MAX columns (max_length = -1) are omitted.
|
||||
Expects parameter <c>@tablename</c>.
|
||||
</summary>
|
||||
</member>
|
||||
<member name="M:fds.FdsColumnWidths.ToWidthMap(System.Data.DataTable)">
|
||||
<summary>Rows of <see cref="F:fds.FdsColumnWidths.WidthsSql"/> → column name → max chars (case-insensitive).</summary>
|
||||
</member>
|
||||
<member name="M:fds.FdsColumnWidths.FindOverflows(System.Data.DataTable,System.Collections.Generic.IReadOnlyDictionary{System.String,System.Int32})">
|
||||
<summary>
|
||||
String columns of <paramref name="batch"/> holding at least one value longer than the
|
||||
declared width, one entry per column with the longest value's length and row Id.
|
||||
Columns without a known width (MAX or not declared) are never reported.
|
||||
</summary>
|
||||
</member>
|
||||
<member name="M:fds.FdsColumnWidths.Describe(System.Collections.Generic.IReadOnlyList{fds.FdsColumnWidths.Overflow})">
|
||||
<summary>Log-friendly summary; "none detected" when the batch fits the declared widths.</summary>
|
||||
</member>
|
||||
<member name="T:fds.FdsSqlOptions">
|
||||
<summary>
|
||||
Logs every SQL error through the worker's structured <see cref="T:Microsoft.Extensions.Logging.ILogger"/> alone — which, when
|
||||
@@ -29,6 +70,12 @@
|
||||
(empty) shape is loaded. Empty when the entity table itself is unknown.
|
||||
</summary>
|
||||
</member>
|
||||
<member name="M:fds.FdsMfrClient.DescribeOverlongColumnsAsync(System.Data.DataTable)">
|
||||
<summary>
|
||||
Failure path only: names the staged columns wider than the target table / table type allow.
|
||||
Never throws — a failed lookup is reported in the returned text instead.
|
||||
</summary>
|
||||
</member>
|
||||
<member name="T:fds.FdsConfig">
|
||||
<summary>
|
||||
Holds the application <see cref="T:Microsoft.Extensions.Configuration.IConfiguration"/> supplied by the host (the Fuchs web app).
|
||||
|
||||
Reference in New Issue
Block a user