From 03a330aee85cf05bcd597bd423dbe3fd3aa570e6 Mon Sep 17 00:00:00 2001 From: Christian Wade Date: Tue, 31 Dec 2019 15:54:00 -0800 Subject: [PATCH] Validate M dependencies with data sources, PBIT hardening validation --- .../TabularCompare/ComparisonFactory.cs | 2 +- .../MultidimensionalMetadata/Measure.cs | 4 +- .../CalcDependencyCollection.cs | 13 ++-- .../TabularMetadata/CalculationItem.cs | 4 +- .../TabularMetadata/Comparison.cs | 74 +++++++++++++------ .../TabularCompare/TabularMetadata/Measure.cs | 4 +- .../TabularMetadata/TabularModel.cs | 66 ++++++++++++----- 7 files changed, 116 insertions(+), 51 deletions(-) diff --git a/BismNormalizer/BismNormalizer/TabularCompare/ComparisonFactory.cs b/BismNormalizer/BismNormalizer/TabularCompare/ComparisonFactory.cs index f8e9052..d96c2d3 100644 --- a/BismNormalizer/BismNormalizer/TabularCompare/ComparisonFactory.cs +++ b/BismNormalizer/BismNormalizer/TabularCompare/ComparisonFactory.cs @@ -66,7 +66,7 @@ namespace BismNormalizer.TabularCompare Telemetry.TrackEvent("CreateComparisonInitialized", new Dictionary { { "App", comparisonInfo.AppName.Replace(" ", "") } }); //If composite models not allowed on AS, check DQ/Import at model level matches: - if (comparisonInfo.ConnectionInfoSource.ServerName != null && !comparisonInfo.ConnectionInfoSource.ServerName.StartsWith("powerbi://") && !Settings.Default.OptionCompositeModelsOverride && comparisonInfo.SourceDirectQuery != comparisonInfo.TargetDirectQuery) + if (comparisonInfo.AppName == "BISM Normalizer" && comparisonInfo.ConnectionInfoTarget.ServerName != null && !comparisonInfo.ConnectionInfoTarget.ServerName.StartsWith("powerbi://") && !Settings.Default.OptionCompositeModelsOverride && comparisonInfo.SourceDirectQuery != comparisonInfo.TargetDirectQuery) { throw new ConnectionException($"Mixed DirectQuery settings are not supported for AS skus.\nSource is {(comparisonInfo.SourceDirectQuery ? "On" : "Off")} and target is {(comparisonInfo.TargetDirectQuery ? "On" : "Off")}."); } diff --git a/BismNormalizer/BismNormalizer/TabularCompare/MultidimensionalMetadata/Measure.cs b/BismNormalizer/BismNormalizer/TabularCompare/MultidimensionalMetadata/Measure.cs index 6d6fc69..2a44ef1 100644 --- a/BismNormalizer/BismNormalizer/TabularCompare/MultidimensionalMetadata/Measure.cs +++ b/BismNormalizer/BismNormalizer/TabularCompare/MultidimensionalMetadata/Measure.cs @@ -234,7 +234,7 @@ namespace BismNormalizer.TabularCompare.MultidimensionalMetadata while (whatsLeftOfLine.Contains('[') && whatsLeftOfLine.Contains(']')) { int openSquareBracketPosition = whatsLeftOfLine.IndexOf('[', 0); - //brilliant person at microsoft has ]] instead of ] + //someone has ]] instead of ] int closeSquareBracketPosition = whatsLeftOfLine.Replace("]]", " ").IndexOf(']', openSquareBracketPosition + 1); if (openSquareBracketPosition < closeSquareBracketPosition - 1) @@ -242,7 +242,7 @@ namespace BismNormalizer.TabularCompare.MultidimensionalMetadata string potentialDependency = whatsLeftOfLine.Substring(openSquareBracketPosition + 1, closeSquareBracketPosition - openSquareBracketPosition - 1); if (!potentialDependency.Contains('"') && !dependencies.Contains(potentialDependency)) { - //unbelievable: some genius at m$ did a replace on ] with ]] + //someone did a replace on ] with ]] dependencies.Add(potentialDependency); } } diff --git a/BismNormalizer/BismNormalizer/TabularCompare/TabularMetadata/CalcDependencyCollection.cs b/BismNormalizer/BismNormalizer/TabularCompare/TabularMetadata/CalcDependencyCollection.cs index bc8e12d..a02e772 100644 --- a/BismNormalizer/BismNormalizer/TabularCompare/TabularMetadata/CalcDependencyCollection.cs +++ b/BismNormalizer/BismNormalizer/TabularCompare/TabularMetadata/CalcDependencyCollection.cs @@ -39,20 +39,23 @@ namespace BismNormalizer.TabularCompare.TabularMetadata /// Type of the object to look up dependencies. /// Name of the object to look up dependencies. /// - public CalcDependencyCollection DependenciesReferenceTo(CalcDependencyObjectType referencedObjectType, string referencedObjectName) + public CalcDependencyCollection DependenciesReferenceTo(CalcDependencyObjectType referencedObjectType, string referencedObjectName, string referencedTableName) { CalcDependencyCollection returnVal = new CalcDependencyCollection(); - LookUpDependenciesReferenceTo(referencedObjectType, referencedObjectName, returnVal); + LookUpDependenciesReferenceTo(referencedObjectType, referencedObjectName, referencedTableName, returnVal); return returnVal; } - private void LookUpDependenciesReferenceTo(CalcDependencyObjectType referencedObjectType, string referencedObjectName, CalcDependencyCollection returnVal) + private void LookUpDependenciesReferenceTo(CalcDependencyObjectType referencedObjectType, string referencedObjectName, string referencedTableName, CalcDependencyCollection returnVal) { foreach (CalcDependency calcDependency in this) { - if (calcDependency.ReferencedObjectType == referencedObjectType && calcDependency.ReferencedObjectName == referencedObjectName) + if ( + (calcDependency.ReferencedObjectType == referencedObjectType && referencedObjectType != CalcDependencyObjectType.Partition && calcDependency.ReferencedObjectName == referencedObjectName) || + (calcDependency.ReferencedObjectType == referencedObjectType && referencedObjectType == CalcDependencyObjectType.Partition && calcDependency.ReferencedTableName == referencedTableName) //References to table-partition expressions are by table name, not partition name + ) { - LookUpDependenciesReferenceTo(calcDependency.ObjectType, calcDependency.ObjectName, returnVal); + LookUpDependenciesReferenceTo(calcDependency.ObjectType, calcDependency.ObjectName, calcDependency.TableName, returnVal); returnVal.Add(calcDependency); } } diff --git a/BismNormalizer/BismNormalizer/TabularCompare/TabularMetadata/CalculationItem.cs b/BismNormalizer/BismNormalizer/TabularCompare/TabularMetadata/CalculationItem.cs index 71e79da..a135a42 100644 --- a/BismNormalizer/BismNormalizer/TabularCompare/TabularMetadata/CalculationItem.cs +++ b/BismNormalizer/BismNormalizer/TabularCompare/TabularMetadata/CalculationItem.cs @@ -68,7 +68,7 @@ namespace BismNormalizer.TabularCompare.TabularMetadata while (whatsRemainingOfLine.Contains('[') && whatsRemainingOfLine.Contains(']')) { int openSquareBracketPosition = whatsRemainingOfLine.IndexOf('[', 0); - //brilliant person at microsoft has ]] instead of ] + //someone has ]] instead of ] int closeSquareBracketPosition = whatsRemainingOfLine.Replace("]]", " ").IndexOf(']', openSquareBracketPosition + 1); if (openSquareBracketPosition < closeSquareBracketPosition - 1) @@ -78,7 +78,7 @@ namespace BismNormalizer.TabularCompare.TabularMetadata !_tomCalculationItem.Expression.Contains($"\"{potentialDependency}\"") && //it's possible the calculationItem itself is deriving the column name from an ADDCOLUMNS for example !dependencies.Contains(potentialDependency)) { - //unbelievable: some genius at m$ did a replace on ] with ]] + //someone did a replace on ] with ]] dependencies.Add(potentialDependency); } } diff --git a/BismNormalizer/BismNormalizer/TabularCompare/TabularMetadata/Comparison.cs b/BismNormalizer/BismNormalizer/TabularCompare/TabularMetadata/Comparison.cs index 59a530d..cc96780 100644 --- a/BismNormalizer/BismNormalizer/TabularCompare/TabularMetadata/Comparison.cs +++ b/BismNormalizer/BismNormalizer/TabularCompare/TabularMetadata/Comparison.cs @@ -635,8 +635,8 @@ namespace BismNormalizer.TabularCompare.TabularMetadata bool reconnect = false; try { - _sourceTabularModel.TomDatabase.Refresh(); - _targetTabularModel.TomDatabase.Refresh(); + if (!_sourceTabularModel.ConnectionInfo.UseBimFile) _sourceTabularModel.TomDatabase.Refresh(); + if (!_targetTabularModel.ConnectionInfo.UseBimFile) _targetTabularModel.TomDatabase.Refresh(); } catch (Exception) { @@ -646,16 +646,11 @@ namespace BismNormalizer.TabularCompare.TabularMetadata if (reconnect || _uncommitedChanges) { // Reconnect to re-initialize - if (!_comparisonInfo.ConnectionInfoSource.UseBimFile) - { - _sourceTabularModel = new TabularModel(this, _comparisonInfo.ConnectionInfoSource, _comparisonInfo); - _sourceTabularModel.Connect(); - } - if (!_comparisonInfo.ConnectionInfoTarget.UseBimFile) - { - _targetTabularModel = new TabularModel(this, _comparisonInfo.ConnectionInfoTarget, _comparisonInfo); - _targetTabularModel.Connect(); - } + _sourceTabularModel = new TabularModel(this, _comparisonInfo.ConnectionInfoSource, _comparisonInfo); + _sourceTabularModel.Connect(); + + _targetTabularModel = new TabularModel(this, _comparisonInfo.ConnectionInfoTarget, _comparisonInfo); + _targetTabularModel.Connect(); } if (!_sourceTabularModel.ConnectionInfo.UseProject && _sourceTabularModel.TomDatabase.LastSchemaUpdate > _lastSourceSchemaUpdate) @@ -989,13 +984,13 @@ namespace BismNormalizer.TabularCompare.TabularMetadata #region Calc dependencies validation - private bool HasBlockingToDependenciesInTarget(string targetObjectName, CalcDependencyObjectType targetObjectType, ref List warningObjectList) + private bool HasBlockingToDependenciesInTarget(string targetObjectName, string referencedTableName, CalcDependencyObjectType targetObjectType, ref List warningObjectList) { //For deletion. //Check any objects in target that depend on this object are also going to be deleted or updated. bool returnVal = false; - CalcDependencyCollection targetToDepdendencies = _targetTabularModel.MDependencies.DependenciesReferenceTo(targetObjectType, targetObjectName); + CalcDependencyCollection targetToDepdendencies = _targetTabularModel.MDependencies.DependenciesReferenceTo(targetObjectType, targetObjectName, referencedTableName); foreach (CalcDependency targetToDependency in targetToDepdendencies) { foreach (ComparisonObject comparisonObjectToCheck in _comparisonObjects) @@ -1086,7 +1081,7 @@ namespace BismNormalizer.TabularCompare.TabularMetadata comparisonObjectToCheck.SourceObjectName == sourceFromDependency.ReferencedObjectName && comparisonObjectToCheck.Status == ComparisonObjectStatus.MissingInTarget && //Creates being skipped (dependency will be missing). comparisonObjectToCheck.MergeAction == MergeAction.Skip) - //Deletes are impossible for this object to depend on, so don't need to detect. Other Skips can assume are fine, so don't need to detect. + //Deletes are impossible for this object to depend on, so don't need to detect. Other Skips can assume are fine, so don't need to detect. { string warningObject = $"Expression {comparisonObjectToCheck.SourceObjectName}"; if (!warningObjectList.Contains(warningObject)) @@ -1096,6 +1091,25 @@ namespace BismNormalizer.TabularCompare.TabularMetadata returnVal = true; } + break; + case CalcDependencyObjectType.Partition: + //Does the object about to be created/updated (sourceObjectName) have a source dependency on this table (comparisonObjectToCheck)? + + if (!_targetTabularModel.Tables.ContainsName(sourceFromDependency.ReferencedTableName) && + comparisonObjectToCheck.ComparisonObjectType == ComparisonObjectType.Table && + comparisonObjectToCheck.SourceObjectName == sourceFromDependency.ReferencedTableName && + comparisonObjectToCheck.Status == ComparisonObjectStatus.MissingInTarget && //Creates being skipped (dependency will be missing). + comparisonObjectToCheck.MergeAction == MergeAction.Skip) + //Deletes are impossible for this object to depend on, so don't need to detect. Other Skips can assume are fine, so don't need to detect. + { + string warningObject = $"Table {comparisonObjectToCheck.SourceObjectName}"; + if (!warningObjectList.Contains(warningObject)) + { + warningObjectList.Add(warningObject); + } + returnVal = true; + } + break; case CalcDependencyObjectType.DataSource: //Does the object about to be created/updated (sourceObjectName) have a source dependency on this data source (comparisonObjectToCheck)? @@ -1257,7 +1271,7 @@ namespace BismNormalizer.TabularCompare.TabularMetadata //Check any objects in target that depend on the DataSource are also going to be deleted List warningObjectList = new List(); - bool toDependencies = HasBlockingToDependenciesInTarget(comparisonObject.TargetObjectName, CalcDependencyObjectType.DataSource, ref warningObjectList); + bool toDependencies = HasBlockingToDependenciesInTarget(comparisonObject.TargetObjectName, "", CalcDependencyObjectType.DataSource, ref warningObjectList); //For old non-M partitions, check if any such tables have reference to this DataSource, and will not be deleted foreach (Table table in _targetTabularModel.Tables) @@ -1393,7 +1407,7 @@ namespace BismNormalizer.TabularCompare.TabularMetadata //Check any objects in target that depend on the expression are also going to be deleted List warningObjectList = new List(); - if (!HasBlockingToDependenciesInTarget(comparisonObject.TargetObjectName, CalcDependencyObjectType.Expression, ref warningObjectList)) + if (!HasBlockingToDependenciesInTarget(comparisonObject.TargetObjectName, "", CalcDependencyObjectType.Expression, ref warningObjectList)) { _targetTabularModel.DeleteExpression(comparisonObject.TargetObjectName); OnValidationMessage(new ValidationMessageEventArgs($"Delete expression [{comparisonObject.TargetObjectName}].", ValidationMessageType.Expression, ValidationMessageStatus.Informational)); @@ -1495,8 +1509,23 @@ namespace BismNormalizer.TabularCompare.TabularMetadata { return; }; - _targetTabularModel.DeleteTable(comparisonObject.TargetObjectName); - OnValidationMessage(new ValidationMessageEventArgs($"Delete {(isCalculationGroup ? "calculation group" : "table")} '{comparisonObject.TargetObjectName}'.", ValidationMessageType.Table, ValidationMessageStatus.Informational)); + + //Check any objects in target that depend on the table expression are also going to be deleted + List warningObjectList = new List(); + if (!HasBlockingToDependenciesInTarget("", comparisonObject.TargetObjectName, CalcDependencyObjectType.Partition, ref warningObjectList)) + { + _targetTabularModel.DeleteTable(comparisonObject.TargetObjectName); + OnValidationMessage(new ValidationMessageEventArgs($"Delete {(isCalculationGroup ? "calculation group" : "table")} '{comparisonObject.TargetObjectName}'.", ValidationMessageType.Table, ValidationMessageStatus.Informational)); + } + else + { + string message = $"Unable to delete table {comparisonObject.TargetObjectName} because the following objects depend on it: {String.Join(", ", warningObjectList)}."; + if (_comparisonInfo.OptionsInfo.OptionRetainPartitions && !_comparisonInfo.OptionsInfo.OptionRetainPolicyPartitions) + { + message += " Note: the option to retain partitions is on, which may be affecting this."; + } + OnValidationMessage(new ValidationMessageEventArgs(message, ValidationMessageType.Table, ValidationMessageStatus.Warning)); + } } } @@ -1855,10 +1884,13 @@ namespace BismNormalizer.TabularCompare.TabularMetadata private bool DesktopHardened(ComparisonObject comparisonObject, ValidationMessageType validationMessageType) { - if (_targetTabularModel.ConnectionInfo.UseDesktop && _targetTabularModel.ConnectionInfo.ServerMode == Microsoft.AnalysisServices.ServerMode.SharePoint) + if ( + (_targetTabularModel.ConnectionInfo.UseDesktop && _targetTabularModel.ConnectionInfo.ServerMode == Microsoft.AnalysisServices.ServerMode.SharePoint) || + (_targetTabularModel.ConnectionInfo.UseBimFile && _targetTabularModel.ConnectionInfo.BimFile != null && _targetTabularModel.ConnectionInfo.BimFile.ToUpper().EndsWith(".PBIT")) + ) { //V3 hardening - OnValidationMessage(new ValidationMessageEventArgs($"Unable to {comparisonObject.MergeAction.ToString().ToLower()} {comparisonObject.ComparisonObjectType.ToString()} {comparisonObject.TargetObjectName} because target is Power BI Desktop, which does not yet support modifications for this object type.", validationMessageType, ValidationMessageStatus.Warning)); + OnValidationMessage(new ValidationMessageEventArgs($"Unable to {comparisonObject.MergeAction.ToString().ToLower()} {comparisonObject.ComparisonObjectType.ToString()} {comparisonObject.TargetObjectName} because target is Power BI Desktop or .PBIT, which does not yet support modifications for this object type.", validationMessageType, ValidationMessageStatus.Warning)); return false; } else diff --git a/BismNormalizer/BismNormalizer/TabularCompare/TabularMetadata/Measure.cs b/BismNormalizer/BismNormalizer/TabularCompare/TabularMetadata/Measure.cs index 67fd945..811dd0a 100644 --- a/BismNormalizer/BismNormalizer/TabularCompare/TabularMetadata/Measure.cs +++ b/BismNormalizer/BismNormalizer/TabularCompare/TabularMetadata/Measure.cs @@ -79,7 +79,7 @@ namespace BismNormalizer.TabularCompare.TabularMetadata while (whatsRemainingOfLine.Contains('[') && whatsRemainingOfLine.Contains(']')) { int openSquareBracketPosition = whatsRemainingOfLine.IndexOf('[', 0); - //brilliant person at microsoft has ]] instead of ] + //someone has ]] instead of ] int closeSquareBracketPosition = whatsRemainingOfLine.Replace("]]", " ").IndexOf(']', openSquareBracketPosition + 1); if (openSquareBracketPosition < closeSquareBracketPosition - 1) @@ -89,7 +89,7 @@ namespace BismNormalizer.TabularCompare.TabularMetadata !_tomMeasure.Expression.Contains($"\"{potentialDependency}\"") && //it's possible the measure itself is deriving the column name from an ADDCOLUMNS for example !dependencies.Contains(potentialDependency)) { - //unbelievable: some genius at m$ did a replace on ] with ]] + //someone did a replace on ] with ]] dependencies.Add(potentialDependency); } } diff --git a/BismNormalizer/BismNormalizer/TabularCompare/TabularMetadata/TabularModel.cs b/BismNormalizer/BismNormalizer/TabularCompare/TabularMetadata/TabularModel.cs index 3d83c3a..e4dfb21 100644 --- a/BismNormalizer/BismNormalizer/TabularCompare/TabularMetadata/TabularModel.cs +++ b/BismNormalizer/BismNormalizer/TabularCompare/TabularMetadata/TabularModel.cs @@ -239,7 +239,8 @@ namespace BismNormalizer.TabularCompare.TabularMetadata _calcDependencies.Clear(); List mObjects = new List(); - //Add table partitions to mObjects collection + #region Add M-dependent objects to collection + foreach (Table table in _tables) { foreach (Partition partition in table.TomTable.Partitions) @@ -258,7 +259,6 @@ namespace BismNormalizer.TabularCompare.TabularMetadata } } - //Add other M expressions to mObjects collection foreach (Expression expression in _expressions) { mObjects.Add( @@ -271,7 +271,24 @@ namespace BismNormalizer.TabularCompare.TabularMetadata ); } - char[] delimiterChars = { ' ', ',', ':', '\t', '\n', '[', ']', '(', ')', '{', '}' }; + foreach (DataSource dataSource in _dataSources) + { + if (dataSource.TomDataSource.Type == DataSourceType.Structured) + { + mObjects.Add( + new MObject( + objectType: "DATA_SOURCE", + tableName: "", + objectName: dataSource.Name, + expression: "" + ) + ); + } + } + + #endregion + + char[] delimiterChars = { ' ', ',', ':', '=', '\t', '\n', '[', ']', '(', ')', '{', '}' }; List keywords = new List() { "and", "as", "each", "else", "error", "false", "if", "in", "is", "let", "meta", "not", "otherwise", "or", "section", "shared", "then", "true", "try", "type", "#binary", "#date", "#datetime", "#datetimezone", "#duration", "#infinity", "#nan", "#sections", "#shared", "#table", "#time" }; foreach (MObject mObject in mObjects) @@ -291,16 +308,16 @@ namespace BismNormalizer.TabularCompare.TabularMetadata mObject.TableName == referencedMObject.TableName )) { - if ( //if M_EXPRESSION name contains spaces or is a keyword, only need to check for occurrence like #"My Query" or #"let" - (referencedMObject.ObjectType == "M_EXPRESSION" && (referencedMObject.ObjectName.Contains(" ") || keywords.Contains(referencedMObject.ObjectName))) && - (mObject.Expression.Contains("\"" + referencedMObject.ObjectName + "\"")) + if ( //if M_EXPRESSION or DATA_SOURCE, check for occurrence like #"My Query" or #"let" + (referencedMObject.ObjectType == "M_EXPRESSION" || referencedMObject.ObjectType == "DATA_SOURCE") && + (mObject.Expression.Contains("#\"" + referencedMObject.ObjectName + "\"")) ) { foundDependency = true; } - else if ( //if table name contains spaces or is a keyword, only need to check for occurrence like #"My Query" or #"let" - (referencedMObject.ObjectType == "PARTITION" && (referencedMObject.TableName.Contains(" ") || keywords.Contains(referencedMObject.TableName))) && - (mObject.Expression.Contains("\"" + referencedMObject.TableName + "\"")) + else if ( //if table name, check for occurrence like #"My Query" or #"let" + referencedMObject.ObjectType == "PARTITION" && + (mObject.Expression.Contains("#\"" + referencedMObject.TableName + "\"")) ) { foundDependency = true; @@ -310,8 +327,14 @@ namespace BismNormalizer.TabularCompare.TabularMetadata foreach (string word in words) { if ( - (referencedMObject.ObjectType == "M_EXPRESSION" && word == referencedMObject.ObjectName && !keywords.Contains(referencedMObject.ObjectName)) || - (referencedMObject.ObjectType == "PARTITION" && word == referencedMObject.TableName && !keywords.Contains(referencedMObject.TableName)) + ( + (referencedMObject.ObjectType == "M_EXPRESSION" || referencedMObject.ObjectType == "DATA_SOURCE") && + word == referencedMObject.ObjectName && !keywords.Contains(referencedMObject.ObjectName) + ) || + ( + referencedMObject.ObjectType == "PARTITION" && + word == referencedMObject.TableName && !keywords.Contains(referencedMObject.TableName) + ) ) { foundDependency = true; @@ -1846,13 +1869,7 @@ namespace BismNormalizer.TabularCompare.TabularMetadata /// Boolean indicating whether update was successful. public bool Update() { - //Set model annotation for telemetry tagging later - const string AnnotationName = "__BNorm"; - Tom.Annotation annotationBNorm = new Tom.Annotation(); - annotationBNorm.Name = AnnotationName; - annotationBNorm.Value = "1"; - if (!_model.TomModel.Annotations.Contains(AnnotationName)) - _model.TomModel.Annotations.Add(annotationBNorm); + SetBNormAnnotation(); if (_connectionInfo.UseBimFile) { @@ -1884,6 +1901,17 @@ namespace BismNormalizer.TabularCompare.TabularMetadata return true; } + private void SetBNormAnnotation() + { + //Set model annotation for telemetry tagging later + const string AnnotationName = "__BNorm"; + Tom.Annotation annotationBNorm = new Tom.Annotation(); + annotationBNorm.Name = AnnotationName; + annotationBNorm.Value = "1"; + if (!_model.TomModel.Annotations.Contains(AnnotationName)) + _model.TomModel.Annotations.Add(annotationBNorm); + } + private void UpdateProject() { UpdateWithScript(); @@ -2315,6 +2343,8 @@ namespace BismNormalizer.TabularCompare.TabularMetadata /// JSON script of tabular model defintion. public string ScriptDatabase() { + SetBNormAnnotation(); + //script db to json string json = JsonScripter.ScriptCreateOrReplace(_database);