Validate M dependencies with data sources, PBIT hardening validation

BismNormalizer/BismNormalizer/TabularCompare/ComparisonFactory.cs

@@ -66,7 +66,7 @@ namespace BismNormalizer.TabularCompare
             Telemetry.TrackEvent("CreateComparisonInitialized", new Dictionary<string, string> { { "App", comparisonInfo.AppName.Replace(" ", "") } });
 
             //If composite models not allowed on AS, check DQ/Import at model level matches:
-            if (comparisonInfo.ConnectionInfoSource.ServerName != null && !comparisonInfo.ConnectionInfoSource.ServerName.StartsWith("powerbi://") && !Settings.Default.OptionCompositeModelsOverride && comparisonInfo.SourceDirectQuery != comparisonInfo.TargetDirectQuery)
+            if (comparisonInfo.AppName == "BISM Normalizer" && comparisonInfo.ConnectionInfoTarget.ServerName != null && !comparisonInfo.ConnectionInfoTarget.ServerName.StartsWith("powerbi://") && !Settings.Default.OptionCompositeModelsOverride && comparisonInfo.SourceDirectQuery != comparisonInfo.TargetDirectQuery)
             {
                 throw new ConnectionException($"Mixed DirectQuery settings are not supported for AS skus.\nSource is {(comparisonInfo.SourceDirectQuery ? "On" : "Off")} and target is {(comparisonInfo.TargetDirectQuery ? "On" : "Off")}.");
             }

BismNormalizer/BismNormalizer/TabularCompare/MultidimensionalMetadata/Measure.cs

@@ -234,7 +234,7 @@ namespace BismNormalizer.TabularCompare.MultidimensionalMetadata
                     while (whatsLeftOfLine.Contains('[') && whatsLeftOfLine.Contains(']'))
                     {
                         int openSquareBracketPosition = whatsLeftOfLine.IndexOf('[', 0);
-                        //brilliant person at microsoft has ]] instead of ]
+                        //someone has ]] instead of ]
                         int closeSquareBracketPosition = whatsLeftOfLine.Replace("]]", "  ").IndexOf(']', openSquareBracketPosition + 1);
 
                         if (openSquareBracketPosition < closeSquareBracketPosition - 1)
@@ -242,7 +242,7 @@ namespace BismNormalizer.TabularCompare.MultidimensionalMetadata
                             string potentialDependency = whatsLeftOfLine.Substring(openSquareBracketPosition + 1, closeSquareBracketPosition - openSquareBracketPosition - 1);
                             if (!potentialDependency.Contains('"') && !dependencies.Contains(potentialDependency))
                             {
-                                //unbelievable: some genius at m$ did a replace on ] with ]]
+                                //someone did a replace on ] with ]]
                                 dependencies.Add(potentialDependency);
                             }
                         }

BismNormalizer/BismNormalizer/TabularCompare/TabularMetadata/CalcDependencyCollection.cs

@@ -39,20 +39,23 @@ namespace BismNormalizer.TabularCompare.TabularMetadata
         /// <param name="objectType">Type of the object to look up dependencies.</param>
         /// <param name="objectName">Name of the object to look up dependencies.</param>
         /// <returns></returns>
-        public CalcDependencyCollection DependenciesReferenceTo(CalcDependencyObjectType referencedObjectType, string referencedObjectName)
+        public CalcDependencyCollection DependenciesReferenceTo(CalcDependencyObjectType referencedObjectType, string referencedObjectName, string referencedTableName)
         {
             CalcDependencyCollection returnVal = new CalcDependencyCollection();
-            LookUpDependenciesReferenceTo(referencedObjectType, referencedObjectName, returnVal);
+            LookUpDependenciesReferenceTo(referencedObjectType, referencedObjectName, referencedTableName, returnVal);
             return returnVal;
         }
 
-        private void LookUpDependenciesReferenceTo(CalcDependencyObjectType referencedObjectType, string referencedObjectName, CalcDependencyCollection returnVal)
+        private void LookUpDependenciesReferenceTo(CalcDependencyObjectType referencedObjectType, string referencedObjectName, string referencedTableName, CalcDependencyCollection returnVal)
         {
             foreach (CalcDependency calcDependency in this)
             {
-                if (calcDependency.ReferencedObjectType == referencedObjectType && calcDependency.ReferencedObjectName == referencedObjectName)
+                if (
+                      (calcDependency.ReferencedObjectType == referencedObjectType && referencedObjectType != CalcDependencyObjectType.Partition && calcDependency.ReferencedObjectName == referencedObjectName) ||
+                      (calcDependency.ReferencedObjectType == referencedObjectType && referencedObjectType == CalcDependencyObjectType.Partition && calcDependency.ReferencedTableName == referencedTableName) //References to table-partition expressions are by table name, not partition name
+                   )
                 {
-                    LookUpDependenciesReferenceTo(calcDependency.ObjectType, calcDependency.ObjectName, returnVal);
+                    LookUpDependenciesReferenceTo(calcDependency.ObjectType, calcDependency.ObjectName, calcDependency.TableName, returnVal);
                     returnVal.Add(calcDependency);
                 }
             }

BismNormalizer/BismNormalizer/TabularCompare/TabularMetadata/CalculationItem.cs

@@ -68,7 +68,7 @@ namespace BismNormalizer.TabularCompare.TabularMetadata
                         while (whatsRemainingOfLine.Contains('[') && whatsRemainingOfLine.Contains(']'))
                         {
                             int openSquareBracketPosition = whatsRemainingOfLine.IndexOf('[', 0);
-                            //brilliant person at microsoft has ]] instead of ]
+                            //someone has ]] instead of ]
                             int closeSquareBracketPosition = whatsRemainingOfLine.Replace("]]", "  ").IndexOf(']', openSquareBracketPosition + 1);
 
                             if (openSquareBracketPosition < closeSquareBracketPosition - 1)
@@ -78,7 +78,7 @@ namespace BismNormalizer.TabularCompare.TabularMetadata
                                     !_tomCalculationItem.Expression.Contains($"\"{potentialDependency}\"") && //it's possible the calculationItem itself is deriving the column name from an ADDCOLUMNS for example
                                     !dependencies.Contains(potentialDependency))
                                 {
-                                    //unbelievable: some genius at m$ did a replace on ] with ]]
+                                    //someone did a replace on ] with ]]
                                     dependencies.Add(potentialDependency);
                                 }
                             }

BismNormalizer/BismNormalizer/TabularCompare/TabularMetadata/Comparison.cs

@@ -635,8 +635,8 @@ namespace BismNormalizer.TabularCompare.TabularMetadata
             bool reconnect = false;
             try
             {
-                _sourceTabularModel.TomDatabase.Refresh();
+                if (!_sourceTabularModel.ConnectionInfo.UseBimFile) _sourceTabularModel.TomDatabase.Refresh();
-                _targetTabularModel.TomDatabase.Refresh();
+                if (!_targetTabularModel.ConnectionInfo.UseBimFile) _targetTabularModel.TomDatabase.Refresh();
             }
             catch (Exception)
             {
@@ -646,17 +646,12 @@ namespace BismNormalizer.TabularCompare.TabularMetadata
             if (reconnect || _uncommitedChanges)
             {
                 // Reconnect to re-initialize
-                if (!_comparisonInfo.ConnectionInfoSource.UseBimFile)
-                {
                 _sourceTabularModel = new TabularModel(this, _comparisonInfo.ConnectionInfoSource, _comparisonInfo);
                 _sourceTabularModel.Connect();
-                }
+
-                if (!_comparisonInfo.ConnectionInfoTarget.UseBimFile)
-                {
                 _targetTabularModel = new TabularModel(this, _comparisonInfo.ConnectionInfoTarget, _comparisonInfo);
                 _targetTabularModel.Connect();
             }
-            }
 
             if (!_sourceTabularModel.ConnectionInfo.UseProject && _sourceTabularModel.TomDatabase.LastSchemaUpdate > _lastSourceSchemaUpdate)
             {
@@ -989,13 +984,13 @@ namespace BismNormalizer.TabularCompare.TabularMetadata
 
         #region Calc dependencies validation
 
-        private bool HasBlockingToDependenciesInTarget(string targetObjectName, CalcDependencyObjectType targetObjectType, ref List<string> warningObjectList)
+        private bool HasBlockingToDependenciesInTarget(string targetObjectName, string referencedTableName, CalcDependencyObjectType targetObjectType, ref List<string> warningObjectList)
         {
             //For deletion.
             //Check any objects in target that depend on this object are also going to be deleted or updated.
 
             bool returnVal = false;
-            CalcDependencyCollection targetToDepdendencies = _targetTabularModel.MDependencies.DependenciesReferenceTo(targetObjectType, targetObjectName);
+            CalcDependencyCollection targetToDepdendencies = _targetTabularModel.MDependencies.DependenciesReferenceTo(targetObjectType, targetObjectName, referencedTableName);
             foreach (CalcDependency targetToDependency in targetToDepdendencies)
             {
                 foreach (ComparisonObject comparisonObjectToCheck in _comparisonObjects)
@@ -1096,6 +1091,25 @@ namespace BismNormalizer.TabularCompare.TabularMetadata
                                 returnVal = true;
                             }
 
+                            break;
+                        case CalcDependencyObjectType.Partition:
+                            //Does the object about to be created/updated (sourceObjectName) have a source dependency on this table (comparisonObjectToCheck)?
+
+                            if (!_targetTabularModel.Tables.ContainsName(sourceFromDependency.ReferencedTableName) &&
+                                comparisonObjectToCheck.ComparisonObjectType == ComparisonObjectType.Table &&
+                                comparisonObjectToCheck.SourceObjectName == sourceFromDependency.ReferencedTableName &&
+                                comparisonObjectToCheck.Status == ComparisonObjectStatus.MissingInTarget &&  //Creates being skipped (dependency will be missing).
+                                comparisonObjectToCheck.MergeAction == MergeAction.Skip)
+                            //Deletes are impossible for this object to depend on, so don't need to detect. Other Skips can assume are fine, so don't need to detect.
+                            {
+                                string warningObject = $"Table {comparisonObjectToCheck.SourceObjectName}";
+                                if (!warningObjectList.Contains(warningObject))
+                                {
+                                    warningObjectList.Add(warningObject);
+                                }
+                                returnVal = true;
+                            }
+
                             break;
                         case CalcDependencyObjectType.DataSource:
                             //Does the object about to be created/updated (sourceObjectName) have a source dependency on this data source (comparisonObjectToCheck)?
@@ -1257,7 +1271,7 @@ namespace BismNormalizer.TabularCompare.TabularMetadata
 
                 //Check any objects in target that depend on the DataSource are also going to be deleted
                 List<string> warningObjectList = new List<string>();
-                bool toDependencies = HasBlockingToDependenciesInTarget(comparisonObject.TargetObjectName, CalcDependencyObjectType.DataSource, ref warningObjectList);
+                bool toDependencies = HasBlockingToDependenciesInTarget(comparisonObject.TargetObjectName, "", CalcDependencyObjectType.DataSource, ref warningObjectList);
 
                 //For old non-M partitions, check if any such tables have reference to this DataSource, and will not be deleted
                 foreach (Table table in _targetTabularModel.Tables)
@@ -1393,7 +1407,7 @@ namespace BismNormalizer.TabularCompare.TabularMetadata
 
                 //Check any objects in target that depend on the expression are also going to be deleted
                 List<string> warningObjectList = new List<string>();
-                if (!HasBlockingToDependenciesInTarget(comparisonObject.TargetObjectName, CalcDependencyObjectType.Expression, ref warningObjectList))
+                if (!HasBlockingToDependenciesInTarget(comparisonObject.TargetObjectName, "", CalcDependencyObjectType.Expression, ref warningObjectList))
                 {
                     _targetTabularModel.DeleteExpression(comparisonObject.TargetObjectName);
                     OnValidationMessage(new ValidationMessageEventArgs($"Delete expression [{comparisonObject.TargetObjectName}].", ValidationMessageType.Expression, ValidationMessageStatus.Informational));
@@ -1495,9 +1509,24 @@ namespace BismNormalizer.TabularCompare.TabularMetadata
                 {
                     return;
                 };
+
+                //Check any objects in target that depend on the table expression are also going to be deleted
+                List<string> warningObjectList = new List<string>();
+                if (!HasBlockingToDependenciesInTarget("", comparisonObject.TargetObjectName, CalcDependencyObjectType.Partition, ref warningObjectList))
+                {
                     _targetTabularModel.DeleteTable(comparisonObject.TargetObjectName);
                     OnValidationMessage(new ValidationMessageEventArgs($"Delete {(isCalculationGroup ? "calculation group" : "table")} '{comparisonObject.TargetObjectName}'.", ValidationMessageType.Table, ValidationMessageStatus.Informational));
                 }
+                else
+                {
+                    string message = $"Unable to delete table {comparisonObject.TargetObjectName} because the following objects depend on it: {String.Join(", ", warningObjectList)}.";
+                    if (_comparisonInfo.OptionsInfo.OptionRetainPartitions && !_comparisonInfo.OptionsInfo.OptionRetainPolicyPartitions)
+                    {
+                        message += " Note: the option to retain partitions is on, which may be affecting this.";
+                    }
+                    OnValidationMessage(new ValidationMessageEventArgs(message, ValidationMessageType.Table, ValidationMessageStatus.Warning));
+                }
+            }
         }
 
         private void CreateTable(ComparisonObject comparisonObject)
@@ -1855,10 +1884,13 @@ namespace BismNormalizer.TabularCompare.TabularMetadata
 
         private bool DesktopHardened(ComparisonObject comparisonObject, ValidationMessageType validationMessageType)
         {
-            if (_targetTabularModel.ConnectionInfo.UseDesktop && _targetTabularModel.ConnectionInfo.ServerMode == Microsoft.AnalysisServices.ServerMode.SharePoint)
+            if (
+                  (_targetTabularModel.ConnectionInfo.UseDesktop && _targetTabularModel.ConnectionInfo.ServerMode == Microsoft.AnalysisServices.ServerMode.SharePoint) ||
+                  (_targetTabularModel.ConnectionInfo.UseBimFile && _targetTabularModel.ConnectionInfo.BimFile != null && _targetTabularModel.ConnectionInfo.BimFile.ToUpper().EndsWith(".PBIT"))
+               )
             {
                 //V3 hardening
-                OnValidationMessage(new ValidationMessageEventArgs($"Unable to {comparisonObject.MergeAction.ToString().ToLower()} {comparisonObject.ComparisonObjectType.ToString()} {comparisonObject.TargetObjectName} because target is Power BI Desktop, which does not yet support modifications for this object type.", validationMessageType, ValidationMessageStatus.Warning));
+                OnValidationMessage(new ValidationMessageEventArgs($"Unable to {comparisonObject.MergeAction.ToString().ToLower()} {comparisonObject.ComparisonObjectType.ToString()} {comparisonObject.TargetObjectName} because target is Power BI Desktop or .PBIT, which does not yet support modifications for this object type.", validationMessageType, ValidationMessageStatus.Warning));
                 return false;
             }
             else

BismNormalizer/BismNormalizer/TabularCompare/TabularMetadata/Measure.cs

@@ -79,7 +79,7 @@ namespace BismNormalizer.TabularCompare.TabularMetadata
                         while (whatsRemainingOfLine.Contains('[') && whatsRemainingOfLine.Contains(']'))
                         {
                             int openSquareBracketPosition = whatsRemainingOfLine.IndexOf('[', 0);
-                            //brilliant person at microsoft has ]] instead of ]
+                            //someone has ]] instead of ]
                             int closeSquareBracketPosition = whatsRemainingOfLine.Replace("]]", "  ").IndexOf(']', openSquareBracketPosition + 1);
 
                             if (openSquareBracketPosition < closeSquareBracketPosition - 1)
@@ -89,7 +89,7 @@ namespace BismNormalizer.TabularCompare.TabularMetadata
                                     !_tomMeasure.Expression.Contains($"\"{potentialDependency}\"") && //it's possible the measure itself is deriving the column name from an ADDCOLUMNS for example
                                     !dependencies.Contains(potentialDependency))
                                 {
-                                    //unbelievable: some genius at m$ did a replace on ] with ]]
+                                    //someone did a replace on ] with ]]
                                     dependencies.Add(potentialDependency);
                                 }
                             }

BismNormalizer/BismNormalizer/TabularCompare/TabularMetadata/TabularModel.cs

@@ -239,7 +239,8 @@ namespace BismNormalizer.TabularCompare.TabularMetadata
             _calcDependencies.Clear();
             List<MObject> mObjects = new List<MObject>();
 
-            //Add table partitions to mObjects collection
+            #region Add M-dependent objects to collection
+
             foreach (Table table in _tables)
             {
                 foreach (Partition partition in table.TomTable.Partitions)
@@ -258,7 +259,6 @@ namespace BismNormalizer.TabularCompare.TabularMetadata
                 }
             }
 
-            //Add other M expressions to mObjects collection
             foreach (Expression expression in _expressions)
             {
                 mObjects.Add(
@@ -271,7 +271,24 @@ namespace BismNormalizer.TabularCompare.TabularMetadata
                 );
             }
 
-            char[] delimiterChars = { ' ', ',', ':', '\t', '\n', '[', ']', '(', ')', '{', '}' };
+            foreach (DataSource dataSource in _dataSources)
+            {
+                if (dataSource.TomDataSource.Type == DataSourceType.Structured)
+                {
+                    mObjects.Add(
+                        new MObject(
+                            objectType: "DATA_SOURCE",
+                            tableName: "",
+                            objectName: dataSource.Name,
+                            expression: ""
+                        )
+                    );
+                }
+            }
+
+            #endregion
+
+            char[] delimiterChars = { ' ', ',', ':', '=', '\t', '\n', '[', ']', '(', ')', '{', '}' };
             List<string> keywords = new List<string>() { "and", "as", "each", "else", "error", "false", "if", "in", "is", "let", "meta", "not", "otherwise", "or", "section", "shared", "then", "true", "try", "type", "#binary", "#date", "#datetime", "#datetimezone", "#duration", "#infinity", "#nan", "#sections", "#shared", "#table", "#time" };
                                                                                                      
             foreach (MObject mObject in mObjects)
@@ -291,16 +308,16 @@ namespace BismNormalizer.TabularCompare.TabularMetadata
                             mObject.TableName == referencedMObject.TableName
                          ))
                     {
-                        if (  //if M_EXPRESSION name contains spaces or is a keyword, only need to check for occurrence like #"My Query" or #"let"
+                        if (  //if M_EXPRESSION or DATA_SOURCE, check for occurrence like #"My Query" or #"let"
-                            (referencedMObject.ObjectType == "M_EXPRESSION" && (referencedMObject.ObjectName.Contains(" ") || keywords.Contains(referencedMObject.ObjectName))) &&
+                            (referencedMObject.ObjectType == "M_EXPRESSION" || referencedMObject.ObjectType == "DATA_SOURCE") &&
-                            (mObject.Expression.Contains("\"" + referencedMObject.ObjectName + "\""))
+                            (mObject.Expression.Contains("#\"" + referencedMObject.ObjectName + "\""))
                         )
                         {
                             foundDependency = true;
                         }
-                        else if ( //if table name contains spaces or is a keyword, only need to check for occurrence like #"My Query" or #"let"
+                        else if ( //if table name, check for occurrence like #"My Query" or #"let"
-                            (referencedMObject.ObjectType == "PARTITION" && (referencedMObject.TableName.Contains(" ") || keywords.Contains(referencedMObject.TableName))) &&
+                            referencedMObject.ObjectType == "PARTITION" &&
-                            (mObject.Expression.Contains("\"" + referencedMObject.TableName + "\""))
+                            (mObject.Expression.Contains("#\"" + referencedMObject.TableName + "\""))
                         )
                         {
                             foundDependency = true;
@@ -310,8 +327,14 @@ namespace BismNormalizer.TabularCompare.TabularMetadata
                             foreach (string word in words)
                             {
                                 if (
-                                        (referencedMObject.ObjectType == "M_EXPRESSION" && word == referencedMObject.ObjectName && !keywords.Contains(referencedMObject.ObjectName)) ||
+                                        (
-                                        (referencedMObject.ObjectType == "PARTITION" && word == referencedMObject.TableName && !keywords.Contains(referencedMObject.TableName))
+                                            (referencedMObject.ObjectType == "M_EXPRESSION" || referencedMObject.ObjectType == "DATA_SOURCE") &&
+                                            word == referencedMObject.ObjectName && !keywords.Contains(referencedMObject.ObjectName)
+                                        ) ||
+                                        (
+                                            referencedMObject.ObjectType == "PARTITION" &&
+                                            word == referencedMObject.TableName && !keywords.Contains(referencedMObject.TableName)
+                                        )
                                    )
                                 {
                                     foundDependency = true;
@@ -1846,13 +1869,7 @@ namespace BismNormalizer.TabularCompare.TabularMetadata
         /// <returns>Boolean indicating whether update was successful.</returns>
         public bool Update()
         {
-            //Set model annotation for telemetry tagging later
+            SetBNormAnnotation();
-            const string AnnotationName = "__BNorm";
-            Tom.Annotation annotationBNorm = new Tom.Annotation();
-            annotationBNorm.Name = AnnotationName;
-            annotationBNorm.Value = "1";
-            if (!_model.TomModel.Annotations.Contains(AnnotationName))
-                _model.TomModel.Annotations.Add(annotationBNorm);
 
             if (_connectionInfo.UseBimFile)
             {
@@ -1884,6 +1901,17 @@ namespace BismNormalizer.TabularCompare.TabularMetadata
             return true;
         }
 
+        private void SetBNormAnnotation()
+        {
+            //Set model annotation for telemetry tagging later
+            const string AnnotationName = "__BNorm";
+            Tom.Annotation annotationBNorm = new Tom.Annotation();
+            annotationBNorm.Name = AnnotationName;
+            annotationBNorm.Value = "1";
+            if (!_model.TomModel.Annotations.Contains(AnnotationName))
+                _model.TomModel.Annotations.Add(annotationBNorm);
+        }
+
         private void UpdateProject()
         {
             UpdateWithScript();
@@ -2315,6 +2343,8 @@ namespace BismNormalizer.TabularCompare.TabularMetadata
         /// <returns>JSON script of tabular model defintion.</returns>
         public string ScriptDatabase()
         {
+            SetBNormAnnotation();
+
             //script db to json
             string json = JsonScripter.ScriptCreateOrReplace(_database);