From f8847197316a1585fbc5a5a6f4d88659bbde5207 Mon Sep 17 00:00:00 2001 From: Christian Wade Date: Mon, 30 Dec 2019 23:21:07 -0800 Subject: [PATCH] parse m expressions for dependencies --- .../TabularCompare/ConnectionInfo.cs | 3 +- .../TabularCompare/Core/Comparison.cs | 11 +- .../TabularMetadata/TabularModel.cs | 208 +++++++++++++++--- 3 files changed, 186 insertions(+), 36 deletions(-) diff --git a/BismNormalizer/BismNormalizer/TabularCompare/ConnectionInfo.cs b/BismNormalizer/BismNormalizer/TabularCompare/ConnectionInfo.cs index d88a02b..a8abe75 100644 --- a/BismNormalizer/BismNormalizer/TabularCompare/ConnectionInfo.cs +++ b/BismNormalizer/BismNormalizer/TabularCompare/ConnectionInfo.cs @@ -576,7 +576,8 @@ namespace BismNormalizer.TabularCompare string dataDir = amoServer.ServerProperties["DataDir"].Value; if (dataDir.EndsWith("\\")) dataDir = dataDir.Substring(0, dataDir.Length - 1); string commandStatement = String.Format("SystemGetSubdirs '{0}'", dataDir); - XmlNodeList rows = Core.Comparison.ExecuteXmlaCommand(amoServer, "", commandStatement); + bool foundFault = false; + XmlNodeList rows = Core.Comparison.ExecuteXmlaCommand(amoServer, "", commandStatement, ref foundFault); string dbDir = ""; foreach (XmlNode row in rows) diff --git a/BismNormalizer/BismNormalizer/TabularCompare/Core/Comparison.cs b/BismNormalizer/BismNormalizer/TabularCompare/Core/Comparison.cs index 340eadd..e38546f 100644 --- a/BismNormalizer/BismNormalizer/TabularCompare/Core/Comparison.cs +++ b/BismNormalizer/BismNormalizer/TabularCompare/Core/Comparison.cs @@ -522,7 +522,8 @@ namespace BismNormalizer.TabularCompare.Core public static int FindRowCount(Microsoft.AnalysisServices.Core.Server server, string tableName, string databaseName) { string dax = String.Format("EVALUATE ROW( \"RowCount\", COUNTROWS('{0}'))", tableName); - XmlNodeList rows = ExecuteXmlaCommand(server, databaseName, dax); + bool foundFault = false; + XmlNodeList rows = ExecuteXmlaCommand(server, databaseName, dax, ref foundFault); foreach (XmlNode row in rows) { @@ -552,7 +553,7 @@ namespace BismNormalizer.TabularCompare.Core /// /// /// XmlNodeList containing results of the command execution. - public static XmlNodeList ExecuteXmlaCommand(Microsoft.AnalysisServices.Core.Server server, string catalog, string commandStatement) + public static XmlNodeList ExecuteXmlaCommand(Microsoft.AnalysisServices.Core.Server server, string catalog, string commandStatement, ref bool foundFault) { XmlWriter xmlWriter = server.StartXmlaRequest(XmlaRequestType.Undefined); WriteSoapEnvelopeWithCommandStatement(xmlWriter, server.SessionID, catalog, commandStatement); @@ -567,6 +568,12 @@ namespace BismNormalizer.TabularCompare.Core nsmgr.AddNamespace("myns1", "urn:schemas-microsoft-com:xml-analysis"); nsmgr.AddNamespace("myns2", "urn:schemas-microsoft-com:xml-analysis:rowset"); XmlNodeList rows = documentResponse.SelectNodes("//myns1:ExecuteResponse/myns1:return/myns2:root/myns2:row", nsmgr); + + if (rows.Count == 0 && documentResponse.GetElementsByTagName("faultcode").Count > 0) + { + foundFault = true; + } + return rows; } diff --git a/BismNormalizer/BismNormalizer/TabularCompare/TabularMetadata/TabularModel.cs b/BismNormalizer/BismNormalizer/TabularCompare/TabularMetadata/TabularModel.cs index f3960fe..5f54b7a 100644 --- a/BismNormalizer/BismNormalizer/TabularCompare/TabularMetadata/TabularModel.cs +++ b/BismNormalizer/BismNormalizer/TabularCompare/TabularMetadata/TabularModel.cs @@ -7,6 +7,8 @@ using Tom = Microsoft.AnalysisServices.Tabular; using Amo = Microsoft.AnalysisServices; using Newtonsoft.Json.Linq; using BismNormalizer.TabularCompare.Core; +using System.Web.UI.WebControls.WebParts; +using System.Text.RegularExpressions; namespace BismNormalizer.TabularCompare.TabularMetadata { @@ -72,7 +74,6 @@ namespace BismNormalizer.TabularCompare.TabularMetadata //Don't need try to load from project here as will already be done before instantiated Comparison throw new Amo.ConnectionException($"Could not connect to database {_connectionInfo.DatabaseName}"); } - InitializeCalcDependencies(); } //Shell model @@ -148,9 +149,18 @@ namespace BismNormalizer.TabularCompare.TabularMetadata { _cultures.Add(new Culture(this, culture)); } + + if (_connectionInfo.UseBimFile) + { + InitializeCalcDependenciesFromM(); + } + else + { + InitializeCalcDependenciesFromServer(); + } } - private void InitializeCalcDependencies() + private void InitializeCalcDependenciesFromServer() { _calcDependencies.Clear(); string command = @@ -159,43 +169,175 @@ namespace BismNormalizer.TabularCompare.TabularMetadata "NOT (OBJECT_TYPE = REFERENCED_OBJECT_TYPE AND " + " [TABLE] = REFERENCED_TABLE AND" + " OBJECT = REFERENCED_OBJECT);"; //Ignore recursive M expression dependencies - XmlNodeList rows = Core.Comparison.ExecuteXmlaCommand(_server, _connectionInfo.DatabaseName, command); - - foreach (XmlNode row in rows) + bool foundFault = false; + XmlNodeList rows = Core.Comparison.ExecuteXmlaCommand(_server, _connectionInfo.DatabaseName, command, ref foundFault); + + if (foundFault) { - string objectType = ""; - string tableName = ""; - string objectName = ""; - string expression = ""; - string referencedObjectType = ""; - string referencedTableName = ""; - string referencedObjectName = ""; - string referencedExpression = ""; - - foreach (XmlNode col in row.ChildNodes) + InitializeCalcDependenciesFromM(); + } + else + { + foreach (XmlNode row in rows) { - if (col.Name == "OBJECT_TYPE") objectType = col.InnerText; - if (col.Name == "TABLE") tableName = col.InnerText; - if (col.Name == "OBJECT") objectName = col.InnerText; - if (col.Name == "EXPRESSION") expression = col.InnerText; - if (col.Name == "REFERENCED_OBJECT_TYPE") referencedObjectType = col.InnerText; - if (col.Name == "REFERENCED_TABLE") referencedTableName = col.InnerText; - if (col.Name == "REFERENCED_OBJECT") referencedObjectName = col.InnerText; - if (col.Name == "REFERENCED_EXPRESSION") referencedExpression = col.InnerText; - } + string objectType = ""; + string tableName = ""; + string objectName = ""; + string expression = ""; + string referencedObjectType = ""; + string referencedTableName = ""; + string referencedObjectName = ""; + string referencedExpression = ""; - _calcDependencies.Add(new CalcDependency(this, - objectType, - tableName, - objectName, - expression, - referencedObjectType, - referencedTableName, - referencedObjectName, - referencedExpression + foreach (XmlNode col in row.ChildNodes) + { + if (col.Name == "OBJECT_TYPE") objectType = col.InnerText; + if (col.Name == "TABLE") tableName = col.InnerText; + if (col.Name == "OBJECT") objectName = col.InnerText; + if (col.Name == "EXPRESSION") expression = col.InnerText; + if (col.Name == "REFERENCED_OBJECT_TYPE") referencedObjectType = col.InnerText; + if (col.Name == "REFERENCED_TABLE") referencedTableName = col.InnerText; + if (col.Name == "REFERENCED_OBJECT") referencedObjectName = col.InnerText; + if (col.Name == "REFERENCED_EXPRESSION") referencedExpression = col.InnerText; + } + + _calcDependencies.Add(new CalcDependency(this, + objectType, + tableName, + objectName, + expression, + referencedObjectType, + referencedTableName, + referencedObjectName, + referencedExpression + ) + ); + } + } + } + + private struct MObject + { + public string ObjectType; + public string TableName; + public string ObjectName; + public string Expression; + + public MObject(string objectType, string tableName, string objectName, string expression) + { + ObjectType = objectType; + TableName = tableName; + ObjectName = objectName; + Expression = expression; + } + } + + private void InitializeCalcDependenciesFromM() + { + //TODO: Doesn't deal with structured data sources (DSRs) yet + + _calcDependencies.Clear(); + List mObjects = new List(); + + //Add table partitions to mObjects collection + foreach (Table table in _tables) + { + foreach (Partition partition in table.TomTable.Partitions) + { + if (partition.SourceType == PartitionSourceType.M) + { + mObjects.Add( + new MObject( + objectType: "PARTITION", + tableName: table.Name, + objectName: partition.Name, + expression: ((MPartitionSource)partition.Source).Expression + ) + ); + } + } + } + + //Add other M expressions to mObjects collection + foreach (Expression expression in _expressions) + { + mObjects.Add( + new MObject( + objectType: "M_EXPRESSION", + tableName: "", + objectName: expression.Name, + expression: expression.TomExpression.Expression ) ); } + + char[] delimiterChars = { ' ', ',', ':', '\t', '\n', '[', ']', '(', ')', '{', '}' }; + List keywords = new List() { "let", "in" }; //TODO: need list of all M keywords + + foreach (MObject mObject in mObjects) + { + string regex = "(#\"(.*?)\")"; + //Expression with double quote references removed + string expressionRegex = Regex.Replace(mObject.Expression, regex, ""); + string[] words = expressionRegex.Split(delimiterChars); + + foreach (MObject referencedMObject in mObjects) + { + bool foundDependency = false; + + if (!( //Ignore circular dependencies + mObject.ObjectName == referencedMObject.ObjectName && + mObject.ObjectType == referencedMObject.ObjectType && + mObject.TableName == referencedMObject.TableName + )) + { + if ( //if M_EXPRESSION name contains spaces or is a keyword, only need to check for occurrence like #"My Query" or #"let" + (referencedMObject.ObjectType == "M_EXPRESSION" && (referencedMObject.ObjectName.Contains(" ") || keywords.Contains(referencedMObject.ObjectName))) && + (mObject.Expression.Contains("\"" + referencedMObject.ObjectName + "\"")) + ) + { + foundDependency = true; + } + else if ( //if table name contains spaces or is a keyword, only need to check for occurrence like #"My Query" or #"let" + (referencedMObject.ObjectType == "PARTITION" && (referencedMObject.TableName.Contains(" ") || keywords.Contains(referencedMObject.TableName))) && + (mObject.Expression.Contains("\"" + referencedMObject.TableName + "\"")) + ) + { + foundDependency = true; + } + else + { + foreach (string word in words) + { + if ( + (referencedMObject.ObjectType == "M_EXPRESSION" && word == referencedMObject.ObjectName && !keywords.Contains(referencedMObject.ObjectName)) || + (referencedMObject.ObjectType == "PARTITION" && word == referencedMObject.TableName && !keywords.Contains(referencedMObject.TableName)) + ) + { + foundDependency = true; + } + } + } + + if (foundDependency) + { + _calcDependencies.Add( + new CalcDependency( + this, + objectType: mObject.ObjectType, + tableName: mObject.TableName, + objectName: mObject.ObjectName, + expression: mObject.Expression, + referencedObjectType: referencedMObject.ObjectType, + referencedTableName: referencedMObject.TableName, + referencedObjectName: referencedMObject.ObjectName, + referencedExpression: referencedMObject.Expression + ) + ); + } + } + } + } } ///