mujiannan/Analysis-Services
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Validate M dependencies with data sources, PBIT hardening validation

Browse Source
This commit is contained in:
Christian Wade 2019-12-31 15:54:00 -08:00
parent ba37cf38ec
commit 03a330aee8
7 changed files with 116 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
BismNormalizer/BismNormalizer/TabularCompare/ComparisonFactory.cs
View File

@ -66,7 +66,7 @@ namespace BismNormalizer.TabularCompare
Telemetry.TrackEvent("CreateComparisonInitialized", new Dictionary<string, string> { { "App", comparisonInfo.AppName.Replace(" ", "") } });
//If composite models not allowed on AS, check DQ/Import at model level matches:
if (comparisonInfo.ConnectionInfoSource.ServerName != null && !comparisonInfo.ConnectionInfoSource.ServerName.StartsWith("powerbi://") && !Settings.Default.OptionCompositeModelsOverride && comparisonInfo.SourceDirectQuery != comparisonInfo.TargetDirectQuery)
if (comparisonInfo.AppName == "BISM Normalizer" && comparisonInfo.ConnectionInfoTarget.ServerName != null && !comparisonInfo.ConnectionInfoTarget.ServerName.StartsWith("powerbi://") && !Settings.Default.OptionCompositeModelsOverride && comparisonInfo.SourceDirectQuery != comparisonInfo.TargetDirectQuery)
{
throw new ConnectionException($"Mixed DirectQuery settings are not supported for AS skus.\nSource is {(comparisonInfo.SourceDirectQuery ? "On" : "Off")} and target is {(comparisonInfo.TargetDirectQuery ? "On" : "Off")}.");
}

4
BismNormalizer/BismNormalizer/TabularCompare/MultidimensionalMetadata/Measure.cs
View File

@ -234,7 +234,7 @@ namespace BismNormalizer.TabularCompare.MultidimensionalMetadata
while (whatsLeftOfLine.Contains('[') && whatsLeftOfLine.Contains(']'))
{
int openSquareBracketPosition = whatsLeftOfLine.IndexOf('[', 0);
//brilliant person at microsoft has ]] instead of ]
//someone has ]] instead of ]
int closeSquareBracketPosition = whatsLeftOfLine.Replace("]]", " ").IndexOf(']', openSquareBracketPosition + 1);
if (openSquareBracketPosition < closeSquareBracketPosition - 1)
@ -242,7 +242,7 @@ namespace BismNormalizer.TabularCompare.MultidimensionalMetadata
string potentialDependency = whatsLeftOfLine.Substring(openSquareBracketPosition + 1, closeSquareBracketPosition - openSquareBracketPosition - 1);
if (!potentialDependency.Contains('"') && !dependencies.Contains(potentialDependency))
{
//unbelievable: some genius at m$ did a replace on ] with ]]
//someone did a replace on ] with ]]
dependencies.Add(potentialDependency);
}
}

13
BismNormalizer/BismNormalizer/TabularCompare/TabularMetadata/CalcDependencyCollection.cs
View File

@ -39,20 +39,23 @@ namespace BismNormalizer.TabularCompare.TabularMetadata
/// <param name="objectType">Type of the object to look up dependencies.</param>
/// <param name="objectName">Name of the object to look up dependencies.</param>
/// <returns></returns>
public CalcDependencyCollection DependenciesReferenceTo(CalcDependencyObjectType referencedObjectType, string referencedObjectName)
public CalcDependencyCollection DependenciesReferenceTo(CalcDependencyObjectType referencedObjectType, string referencedObjectName, string referencedTableName)
{
CalcDependencyCollection returnVal = new CalcDependencyCollection();
LookUpDependenciesReferenceTo(referencedObjectType, referencedObjectName, returnVal);
LookUpDependenciesReferenceTo(referencedObjectType, referencedObjectName, referencedTableName, returnVal);
return returnVal;
}
private void LookUpDependenciesReferenceTo(CalcDependencyObjectType referencedObjectType, string referencedObjectName, CalcDependencyCollection returnVal)
private void LookUpDependenciesReferenceTo(CalcDependencyObjectType referencedObjectType, string referencedObjectName, string referencedTableName, CalcDependencyCollection returnVal)
{
foreach (CalcDependency calcDependency in this)
{
if (calcDependency.ReferencedObjectType == referencedObjectType && calcDependency.ReferencedObjectName == referencedObjectName)
if (
(calcDependency.ReferencedObjectType == referencedObjectType && referencedObjectType != CalcDependencyObjectType.Partition && calcDependency.ReferencedObjectName == referencedObjectName) ||
(calcDependency.ReferencedObjectType == referencedObjectType && referencedObjectType == CalcDependencyObjectType.Partition && calcDependency.ReferencedTableName == referencedTableName) //References to table-partition expressions are by table name, not partition name
)
{
LookUpDependenciesReferenceTo(calcDependency.ObjectType, calcDependency.ObjectName, returnVal);
LookUpDependenciesReferenceTo(calcDependency.ObjectType, calcDependency.ObjectName, calcDependency.TableName, returnVal);
returnVal.Add(calcDependency);
}
}

4
BismNormalizer/BismNormalizer/TabularCompare/TabularMetadata/CalculationItem.cs
View File

@ -68,7 +68,7 @@ namespace BismNormalizer.TabularCompare.TabularMetadata
while (whatsRemainingOfLine.Contains('[') && whatsRemainingOfLine.Contains(']'))
{
int openSquareBracketPosition = whatsRemainingOfLine.IndexOf('[', 0);
//brilliant person at microsoft has ]] instead of ]
//someone has ]] instead of ]
int closeSquareBracketPosition = whatsRemainingOfLine.Replace("]]", " ").IndexOf(']', openSquareBracketPosition + 1);
if (openSquareBracketPosition < closeSquareBracketPosition - 1)
@ -78,7 +78,7 @@ namespace BismNormalizer.TabularCompare.TabularMetadata
!_tomCalculationItem.Expression.Contains($"\"{potentialDependency}\"") && //it's possible the calculationItem itself is deriving the column name from an ADDCOLUMNS for example
!dependencies.Contains(potentialDependency))
{
//unbelievable: some genius at m$ did a replace on ] with ]]
//someone did a replace on ] with ]]
dependencies.Add(potentialDependency);
}
}

74
BismNormalizer/BismNormalizer/TabularCompare/TabularMetadata/Comparison.cs
View File

@ -635,8 +635,8 @@ namespace BismNormalizer.TabularCompare.TabularMetadata
bool reconnect = false;
try
{
_sourceTabularModel.TomDatabase.Refresh();
_targetTabularModel.TomDatabase.Refresh();
if (!_sourceTabularModel.ConnectionInfo.UseBimFile) _sourceTabularModel.TomDatabase.Refresh();
if (!_targetTabularModel.ConnectionInfo.UseBimFile) _targetTabularModel.TomDatabase.Refresh();
}
catch (Exception)
{
@ -646,16 +646,11 @@ namespace BismNormalizer.TabularCompare.TabularMetadata
if (reconnect || _uncommitedChanges)
{
// Reconnect to re-initialize
if (!_comparisonInfo.ConnectionInfoSource.UseBimFile)
{
_sourceTabularModel = new TabularModel(this, _comparisonInfo.ConnectionInfoSource, _comparisonInfo);
_sourceTabularModel.Connect();
}
if (!_comparisonInfo.ConnectionInfoTarget.UseBimFile)
{
_targetTabularModel = new TabularModel(this, _comparisonInfo.ConnectionInfoTarget, _comparisonInfo);
_targetTabularModel.Connect();
}
_sourceTabularModel = new TabularModel(this, _comparisonInfo.ConnectionInfoSource, _comparisonInfo);
_sourceTabularModel.Connect();
_targetTabularModel = new TabularModel(this, _comparisonInfo.ConnectionInfoTarget, _comparisonInfo);
_targetTabularModel.Connect();
}
if (!_sourceTabularModel.ConnectionInfo.UseProject && _sourceTabularModel.TomDatabase.LastSchemaUpdate > _lastSourceSchemaUpdate)
@ -989,13 +984,13 @@ namespace BismNormalizer.TabularCompare.TabularMetadata
#region Calc dependencies validation
private bool HasBlockingToDependenciesInTarget(string targetObjectName, CalcDependencyObjectType targetObjectType, ref List<string> warningObjectList)
private bool HasBlockingToDependenciesInTarget(string targetObjectName, string referencedTableName, CalcDependencyObjectType targetObjectType, ref List<string> warningObjectList)
{
//For deletion.
//Check any objects in target that depend on this object are also going to be deleted or updated.
bool returnVal = false;
CalcDependencyCollection targetToDepdendencies = _targetTabularModel.MDependencies.DependenciesReferenceTo(targetObjectType, targetObjectName);
CalcDependencyCollection targetToDepdendencies = _targetTabularModel.MDependencies.DependenciesReferenceTo(targetObjectType, targetObjectName, referencedTableName);
foreach (CalcDependency targetToDependency in targetToDepdendencies)
{
foreach (ComparisonObject comparisonObjectToCheck in _comparisonObjects)
@ -1086,7 +1081,7 @@ namespace BismNormalizer.TabularCompare.TabularMetadata
comparisonObjectToCheck.SourceObjectName == sourceFromDependency.ReferencedObjectName &&
comparisonObjectToCheck.Status == ComparisonObjectStatus.MissingInTarget && //Creates being skipped (dependency will be missing).
comparisonObjectToCheck.MergeAction == MergeAction.Skip)
//Deletes are impossible for this object to depend on, so don't need to detect. Other Skips can assume are fine, so don't need to detect.
//Deletes are impossible for this object to depend on, so don't need to detect. Other Skips can assume are fine, so don't need to detect.
{
string warningObject = $"Expression {comparisonObjectToCheck.SourceObjectName}";
if (!warningObjectList.Contains(warningObject))
@ -1096,6 +1091,25 @@ namespace BismNormalizer.TabularCompare.TabularMetadata
returnVal = true;
}
break;
case CalcDependencyObjectType.Partition:
//Does the object about to be created/updated (sourceObjectName) have a source dependency on this table (comparisonObjectToCheck)?
if (!_targetTabularModel.Tables.ContainsName(sourceFromDependency.ReferencedTableName) &&
comparisonObjectToCheck.ComparisonObjectType == ComparisonObjectType.Table &&
comparisonObjectToCheck.SourceObjectName == sourceFromDependency.ReferencedTableName &&
comparisonObjectToCheck.Status == ComparisonObjectStatus.MissingInTarget && //Creates being skipped (dependency will be missing).
comparisonObjectToCheck.MergeAction == MergeAction.Skip)
//Deletes are impossible for this object to depend on, so don't need to detect. Other Skips can assume are fine, so don't need to detect.
{
string warningObject = $"Table {comparisonObjectToCheck.SourceObjectName}";
if (!warningObjectList.Contains(warningObject))
{
warningObjectList.Add(warningObject);
}
returnVal = true;
}
break;
case CalcDependencyObjectType.DataSource:
//Does the object about to be created/updated (sourceObjectName) have a source dependency on this data source (comparisonObjectToCheck)?
@ -1257,7 +1271,7 @@ namespace BismNormalizer.TabularCompare.TabularMetadata
//Check any objects in target that depend on the DataSource are also going to be deleted
List<string> warningObjectList = new List<string>();
bool toDependencies = HasBlockingToDependenciesInTarget(comparisonObject.TargetObjectName, CalcDependencyObjectType.DataSource, ref warningObjectList);
bool toDependencies = HasBlockingToDependenciesInTarget(comparisonObject.TargetObjectName, "", CalcDependencyObjectType.DataSource, ref warningObjectList);
//For old non-M partitions, check if any such tables have reference to this DataSource, and will not be deleted
foreach (Table table in _targetTabularModel.Tables)
@ -1393,7 +1407,7 @@ namespace BismNormalizer.TabularCompare.TabularMetadata
//Check any objects in target that depend on the expression are also going to be deleted
List<string> warningObjectList = new List<string>();
if (!HasBlockingToDependenciesInTarget(comparisonObject.TargetObjectName, CalcDependencyObjectType.Expression, ref warningObjectList))
if (!HasBlockingToDependenciesInTarget(comparisonObject.TargetObjectName, "", CalcDependencyObjectType.Expression, ref warningObjectList))
{
_targetTabularModel.DeleteExpression(comparisonObject.TargetObjectName);
OnValidationMessage(new ValidationMessageEventArgs($"Delete expression [{comparisonObject.TargetObjectName}].", ValidationMessageType.Expression, ValidationMessageStatus.Informational));
@ -1495,8 +1509,23 @@ namespace BismNormalizer.TabularCompare.TabularMetadata
{
return;
};
_targetTabularModel.DeleteTable(comparisonObject.TargetObjectName);
OnValidationMessage(new ValidationMessageEventArgs($"Delete {(isCalculationGroup ? "calculation group" : "table")} '{comparisonObject.TargetObjectName}'.", ValidationMessageType.Table, ValidationMessageStatus.Informational));
//Check any objects in target that depend on the table expression are also going to be deleted
List<string> warningObjectList = new List<string>();
if (!HasBlockingToDependenciesInTarget("", comparisonObject.TargetObjectName, CalcDependencyObjectType.Partition, ref warningObjectList))
{
_targetTabularModel.DeleteTable(comparisonObject.TargetObjectName);
OnValidationMessage(new ValidationMessageEventArgs($"Delete {(isCalculationGroup ? "calculation group" : "table")} '{comparisonObject.TargetObjectName}'.", ValidationMessageType.Table, ValidationMessageStatus.Informational));
}
else
{
string message = $"Unable to delete table {comparisonObject.TargetObjectName} because the following objects depend on it: {String.Join(", ", warningObjectList)}.";
if (_comparisonInfo.OptionsInfo.OptionRetainPartitions && !_comparisonInfo.OptionsInfo.OptionRetainPolicyPartitions)
{
message += " Note: the option to retain partitions is on, which may be affecting this.";
}
OnValidationMessage(new ValidationMessageEventArgs(message, ValidationMessageType.Table, ValidationMessageStatus.Warning));
}
}
}
@ -1855,10 +1884,13 @@ namespace BismNormalizer.TabularCompare.TabularMetadata
private bool DesktopHardened(ComparisonObject comparisonObject, ValidationMessageType validationMessageType)
{
if (_targetTabularModel.ConnectionInfo.UseDesktop && _targetTabularModel.ConnectionInfo.ServerMode == Microsoft.AnalysisServices.ServerMode.SharePoint)
if (
(_targetTabularModel.ConnectionInfo.UseDesktop && _targetTabularModel.ConnectionInfo.ServerMode == Microsoft.AnalysisServices.ServerMode.SharePoint) ||
(_targetTabularModel.ConnectionInfo.UseBimFile && _targetTabularModel.ConnectionInfo.BimFile != null && _targetTabularModel.ConnectionInfo.BimFile.ToUpper().EndsWith(".PBIT"))
)
{
//V3 hardening
OnValidationMessage(new ValidationMessageEventArgs($"Unable to {comparisonObject.MergeAction.ToString().ToLower()} {comparisonObject.ComparisonObjectType.ToString()} {comparisonObject.TargetObjectName} because target is Power BI Desktop, which does not yet support modifications for this object type.", validationMessageType, ValidationMessageStatus.Warning));
OnValidationMessage(new ValidationMessageEventArgs($"Unable to {comparisonObject.MergeAction.ToString().ToLower()} {comparisonObject.ComparisonObjectType.ToString()} {comparisonObject.TargetObjectName} because target is Power BI Desktop or .PBIT, which does not yet support modifications for this object type.", validationMessageType, ValidationMessageStatus.Warning));
return false;
}
else

4
BismNormalizer/BismNormalizer/TabularCompare/TabularMetadata/Measure.cs
View File

@ -79,7 +79,7 @@ namespace BismNormalizer.TabularCompare.TabularMetadata
while (whatsRemainingOfLine.Contains('[') && whatsRemainingOfLine.Contains(']'))
{
int openSquareBracketPosition = whatsRemainingOfLine.IndexOf('[', 0);
//brilliant person at microsoft has ]] instead of ]
//someone has ]] instead of ]
int closeSquareBracketPosition = whatsRemainingOfLine.Replace("]]", " ").IndexOf(']', openSquareBracketPosition + 1);
if (openSquareBracketPosition < closeSquareBracketPosition - 1)
@ -89,7 +89,7 @@ namespace BismNormalizer.TabularCompare.TabularMetadata
!_tomMeasure.Expression.Contains($"\"{potentialDependency}\"") && //it's possible the measure itself is deriving the column name from an ADDCOLUMNS for example
!dependencies.Contains(potentialDependency))
{
//unbelievable: some genius at m$ did a replace on ] with ]]
//someone did a replace on ] with ]]
dependencies.Add(potentialDependency);
}
}

66
BismNormalizer/BismNormalizer/TabularCompare/TabularMetadata/TabularModel.cs
View File

@ -239,7 +239,8 @@ namespace BismNormalizer.TabularCompare.TabularMetadata
_calcDependencies.Clear();
List<MObject> mObjects = new List<MObject>();
//Add table partitions to mObjects collection
#region Add M-dependent objects to collection
foreach (Table table in _tables)
{
foreach (Partition partition in table.TomTable.Partitions)
@ -258,7 +259,6 @@ namespace BismNormalizer.TabularCompare.TabularMetadata
}
}
//Add other M expressions to mObjects collection
foreach (Expression expression in _expressions)
{
mObjects.Add(
@ -271,7 +271,24 @@ namespace BismNormalizer.TabularCompare.TabularMetadata
);
}
char[] delimiterChars = { ' ', ',', ':', '\t', '\n', '[', ']', '(', ')', '{', '}' };
foreach (DataSource dataSource in _dataSources)
{
if (dataSource.TomDataSource.Type == DataSourceType.Structured)
{
mObjects.Add(
new MObject(
objectType: "DATA_SOURCE",
tableName: "",
objectName: dataSource.Name,
expression: ""
)
);
}
}
#endregion
char[] delimiterChars = { ' ', ',', ':', '=', '\t', '\n', '[', ']', '(', ')', '{', '}' };
List<string> keywords = new List<string>() { "and", "as", "each", "else", "error", "false", "if", "in", "is", "let", "meta", "not", "otherwise", "or", "section", "shared", "then", "true", "try", "type", "#binary", "#date", "#datetime", "#datetimezone", "#duration", "#infinity", "#nan", "#sections", "#shared", "#table", "#time" };
foreach (MObject mObject in mObjects)
@ -291,16 +308,16 @@ namespace BismNormalizer.TabularCompare.TabularMetadata
mObject.TableName == referencedMObject.TableName
))
{
if ( //if M_EXPRESSION name contains spaces or is a keyword, only need to check for occurrence like #"My Query" or #"let"
(referencedMObject.ObjectType == "M_EXPRESSION" && (referencedMObject.ObjectName.Contains(" ") || keywords.Contains(referencedMObject.ObjectName))) &&
(mObject.Expression.Contains("\"" + referencedMObject.ObjectName + "\""))
if ( //if M_EXPRESSION or DATA_SOURCE, check for occurrence like #"My Query" or #"let"
(referencedMObject.ObjectType == "M_EXPRESSION" || referencedMObject.ObjectType == "DATA_SOURCE") &&
(mObject.Expression.Contains("#\"" + referencedMObject.ObjectName + "\""))
)
{
foundDependency = true;
}
else if ( //if table name contains spaces or is a keyword, only need to check for occurrence like #"My Query" or #"let"
(referencedMObject.ObjectType == "PARTITION" && (referencedMObject.TableName.Contains(" ") || keywords.Contains(referencedMObject.TableName))) &&
(mObject.Expression.Contains("\"" + referencedMObject.TableName + "\""))
else if ( //if table name, check for occurrence like #"My Query" or #"let"
referencedMObject.ObjectType == "PARTITION" &&
(mObject.Expression.Contains("#\"" + referencedMObject.TableName + "\""))
)
{
foundDependency = true;
@ -310,8 +327,14 @@ namespace BismNormalizer.TabularCompare.TabularMetadata
foreach (string word in words)
{
if (
(referencedMObject.ObjectType == "M_EXPRESSION" && word == referencedMObject.ObjectName && !keywords.Contains(referencedMObject.ObjectName)) ||
(referencedMObject.ObjectType == "PARTITION" && word == referencedMObject.TableName && !keywords.Contains(referencedMObject.TableName))
(
(referencedMObject.ObjectType == "M_EXPRESSION" || referencedMObject.ObjectType == "DATA_SOURCE") &&
word == referencedMObject.ObjectName && !keywords.Contains(referencedMObject.ObjectName)
) ||
(
referencedMObject.ObjectType == "PARTITION" &&
word == referencedMObject.TableName && !keywords.Contains(referencedMObject.TableName)
)
)
{
foundDependency = true;
@ -1846,13 +1869,7 @@ namespace BismNormalizer.TabularCompare.TabularMetadata
/// <returns>Boolean indicating whether update was successful.</returns>
public bool Update()
{
//Set model annotation for telemetry tagging later
const string AnnotationName = "__BNorm";
Tom.Annotation annotationBNorm = new Tom.Annotation();
annotationBNorm.Name = AnnotationName;
annotationBNorm.Value = "1";
if (!_model.TomModel.Annotations.Contains(AnnotationName))
_model.TomModel.Annotations.Add(annotationBNorm);
SetBNormAnnotation();
if (_connectionInfo.UseBimFile)
{
@ -1884,6 +1901,17 @@ namespace BismNormalizer.TabularCompare.TabularMetadata
return true;
}
private void SetBNormAnnotation()
{
//Set model annotation for telemetry tagging later
const string AnnotationName = "__BNorm";
Tom.Annotation annotationBNorm = new Tom.Annotation();
annotationBNorm.Name = AnnotationName;
annotationBNorm.Value = "1";
if (!_model.TomModel.Annotations.Contains(AnnotationName))
_model.TomModel.Annotations.Add(annotationBNorm);
}
private void UpdateProject()
{
UpdateWithScript();
@ -2315,6 +2343,8 @@ namespace BismNormalizer.TabularCompare.TabularMetadata
/// <returns>JSON script of tabular model defintion.</returns>
public string ScriptDatabase()
{
SetBNormAnnotation();
//script db to json
string json = JsonScripter.ScriptCreateOrReplace(_database);