464 lines
18 KiB
C#
464 lines
18 KiB
C#
using Microsoft.AnalysisServices.Tabular;
|
|
using System;
|
|
using System.Collections.Generic;
|
|
using System.IO;
|
|
using System.Linq;
|
|
using System.Text;
|
|
using System.Threading.Tasks;
|
|
using PowerArgs;
|
|
using System.Globalization;
|
|
using Newtonsoft.Json;
|
|
using Newtonsoft.Json.Converters;
|
|
|
|
namespace SetUpAggs
|
|
{
|
|
#region JsonHelper Class
|
|
|
|
public static class JsonHelper
|
|
{
|
|
public static string FromClass<T>(T data, bool isEmptyToNull = false,
|
|
JsonSerializerSettings jsonSettings = null)
|
|
{
|
|
string response = string.Empty;
|
|
|
|
if (!EqualityComparer<T>.Default.Equals(data, default(T)))
|
|
response = JsonConvert.SerializeObject(data, jsonSettings);
|
|
|
|
return isEmptyToNull ? (response == "{}" ? "null" : response) : response;
|
|
}
|
|
|
|
public static T ToClass<T>(string data, JsonSerializerSettings jsonSettings = null)
|
|
{
|
|
var response = default(T);
|
|
|
|
if (!string.IsNullOrEmpty(data))
|
|
response = jsonSettings == null
|
|
? JsonConvert.DeserializeObject<T>(data)
|
|
: JsonConvert.DeserializeObject<T>(data, jsonSettings);
|
|
|
|
return response;
|
|
}
|
|
}
|
|
#endregion
|
|
|
|
#region Json Configuration Classes
|
|
|
|
public partial class AggsConfiguration
|
|
{
|
|
[JsonProperty("database")]
|
|
public Database Database { get; set; }
|
|
}
|
|
|
|
public partial class Database
|
|
{
|
|
[JsonProperty("name")]
|
|
public string Name { get; set; }
|
|
|
|
[JsonProperty("tables")]
|
|
public Table[] Tables { get; set; }
|
|
}
|
|
|
|
public partial class Table
|
|
{
|
|
[JsonProperty("name")]
|
|
public string Name { get; set; }
|
|
|
|
[JsonProperty("mode")]
|
|
public string Mode { get; set; }
|
|
|
|
[JsonProperty("aggregationRules", NullValueHandling = NullValueHandling.Ignore)]
|
|
public AggregationRule[] AggregationRules { get; set; }
|
|
}
|
|
|
|
public partial class AggregationRule
|
|
{
|
|
[JsonProperty("aggTableColumn")]
|
|
public string AggTableColumn { get; set; }
|
|
|
|
[JsonProperty("summarization")]
|
|
public string Summarization { get; set; }
|
|
|
|
[JsonProperty("detailTable")]
|
|
public string DetailTable { get; set; }
|
|
|
|
[JsonProperty("detailTableColumn", NullValueHandling = NullValueHandling.Ignore)]
|
|
public string DetailTableColumn { get; set; }
|
|
}
|
|
|
|
public partial class Object
|
|
{
|
|
[JsonProperty("database")]
|
|
public string Database { get; set; }
|
|
}
|
|
|
|
public partial class AggsConfiguration
|
|
{
|
|
public static AggsConfiguration FromJson(string json) => JsonConvert.DeserializeObject<AggsConfiguration>(json, SetUpAggs.Converter.Settings);
|
|
}
|
|
|
|
public static class Serialize
|
|
{
|
|
public static string ToJson(this AggsConfiguration self) => JsonConvert.SerializeObject(self, SetUpAggs.Converter.Settings);
|
|
}
|
|
|
|
internal static class Converter
|
|
{
|
|
public static readonly JsonSerializerSettings Settings = new JsonSerializerSettings
|
|
{
|
|
MetadataPropertyHandling = MetadataPropertyHandling.Ignore,
|
|
DateParseHandling = DateParseHandling.None,
|
|
Converters =
|
|
{
|
|
new IsoDateTimeConverter { DateTimeStyles = DateTimeStyles.AssumeUniversal }
|
|
},
|
|
};
|
|
}
|
|
#endregion
|
|
|
|
#region Command Line Utility Arguments Handling
|
|
|
|
[ArgExceptionBehavior(ArgExceptionPolicy.StandardExceptionHandling)]
|
|
public class AggConfigurationUtility
|
|
{
|
|
Microsoft.AnalysisServices.Tabular.Server server = new Microsoft.AnalysisServices.Tabular.Server();
|
|
Microsoft.AnalysisServices.Tabular.Database database = new Microsoft.AnalysisServices.Tabular.Database();
|
|
AggsConfiguration aggsConfig;
|
|
|
|
[HelpHook, ArgShortcut("-?"), ArgDescription("Shows this help")]
|
|
public bool Help { get; set; }
|
|
|
|
|
|
/// <summary>
|
|
/// Validate the configuration file against the model before applying changes
|
|
/// </summary>
|
|
/// <param name="tableConfig"></param>
|
|
private void ValidateConfigurationOfTable(Table tableConfig)
|
|
{
|
|
var table = database.Model.Tables.Find(tableConfig.Name);
|
|
|
|
if (table == null)
|
|
throw new Exception($"Could not find table [{tableConfig.Name}]");
|
|
|
|
try
|
|
{
|
|
Enum.Parse(typeof(ModeType), tableConfig.Mode);
|
|
}
|
|
catch(Exception)
|
|
{
|
|
throw new Exception($"Table [{tableConfig.Name}] has invalid mode type {tableConfig.Mode}, must be one of Default, DirectQuery, Dual, or Import");
|
|
}
|
|
|
|
if (table.Partitions.Count == 0)
|
|
throw new Exception($"No partitions found for table [{table.Name}]");
|
|
|
|
if(tableConfig.AggregationRules != null)
|
|
foreach (AggregationRule rule in tableConfig.AggregationRules)
|
|
{
|
|
if(rule.Summarization != "CountTableRows")
|
|
try
|
|
{
|
|
Enum.Parse(typeof(SummarizationType), rule.Summarization);
|
|
}
|
|
catch (Exception)
|
|
{
|
|
throw new Exception($"Aggregation rule summarization invalid [{rule.Summarization}], must be one of GroupBy, Sum, Min, Max, Count, or CountTableRows");
|
|
}
|
|
|
|
Microsoft.AnalysisServices.Tabular.Column aggColumn = table.Columns.Find(rule.AggTableColumn);
|
|
|
|
if (aggColumn == null)
|
|
throw new Exception($"Cound not find aggregation column [{table.Name}].[{rule.AggTableColumn}] in model");
|
|
|
|
var detailTable = database.Model.Tables.Find(rule.DetailTable);
|
|
|
|
if (detailTable == null)
|
|
throw new Exception($"Cound not find detail table [{rule.DetailTable}] in model");
|
|
|
|
var detailColumn = (rule.DetailTableColumn == null ? null : detailTable.Columns.Find(rule.DetailTableColumn));
|
|
|
|
bool isCountAggregate = rule.Summarization != "Count" || rule.Summarization != "CountTableRows";
|
|
|
|
if (detailColumn == null && !isCountAggregate)
|
|
throw new Exception($"Cound not find detail column [{rule.DetailTable}].[{rule.DetailTableColumn}] in model");
|
|
|
|
if (!isCountAggregate && aggColumn.DataType != detailColumn.DataType)
|
|
throw new Exception($"Data type mismatch for aggregation column [{table.Name}].[{rule.AggTableColumn}] and detail column [{rule.DetailTable}].[{rule.DetailTableColumn}]");
|
|
|
|
}
|
|
|
|
Console.ForegroundColor = ConsoleColor.White;
|
|
Console.WriteLine($"Validated aggregation rules for [{table.Name}], mode currently set to {table.Partitions[0].Mode.ToString()}");
|
|
}
|
|
|
|
/// <summary>
|
|
/// Applies the differences between the configuration file and the model
|
|
/// </summary>
|
|
/// <param name="tableConfig"></param>
|
|
private int ApplyConfigurationOfTable(Table tableConfig)
|
|
{
|
|
// Update mode of each partition of the table if a difference is found between the model and the configuration
|
|
var table = database.Model.Tables.Find(tableConfig.Name);
|
|
int partitionModesChanged = 0;
|
|
table.Partitions.ForEach(p =>
|
|
{
|
|
var mode = (ModeType)Enum.Parse(typeof(ModeType), tableConfig.Mode);
|
|
if (p.Mode != mode)
|
|
{
|
|
p.Mode = mode;
|
|
partitionModesChanged++;
|
|
}
|
|
}
|
|
);
|
|
|
|
// Remove all alternateOf definitions from the model that do not match an aggregation rule in the configuration
|
|
int alternateOfRemoved = 0;
|
|
table.Columns.ForEach(c =>
|
|
{
|
|
int ruleMatch = 0;
|
|
|
|
if(tableConfig.AggregationRules != null && c.AlternateOf != null)
|
|
ruleMatch = tableConfig.AggregationRules.Where(r =>
|
|
r.Summarization != "CountTableRows" &&
|
|
r.AggTableColumn == c.Name &&
|
|
r.DetailTable == c.AlternateOf.BaseColumn.Table.Name &&
|
|
r.DetailTableColumn == c.AlternateOf.BaseColumn.Name &&
|
|
(SummarizationType)Enum.Parse(typeof(SummarizationType), r.Summarization) == c.AlternateOf.Summarization
|
|
).Count() +
|
|
tableConfig.AggregationRules.Where(r =>
|
|
r.Summarization == "CountTableRows" &&
|
|
r.AggTableColumn == c.Name &&
|
|
r.DetailTable == c.AlternateOf.BaseTable.Name &&
|
|
c.AlternateOf.Summarization == SummarizationType.Count
|
|
).Count();
|
|
|
|
if (ruleMatch == 0 && c.AlternateOf != null)
|
|
{
|
|
var baseObjectName = c.AlternateOf.BaseColumn == null ?
|
|
String.Format("[{0}]", c.AlternateOf.BaseTable.Name) :
|
|
String.Format("[{0}].[{1}]", c.AlternateOf.BaseColumn.Table.Name, c.AlternateOf.BaseColumn.Name);
|
|
|
|
Console.ForegroundColor = ConsoleColor.Yellow;
|
|
Console.WriteLine($"Removing aggregation rule from column [{c.Name}] with alternate of {baseObjectName} aggregation {c.AlternateOf.Summarization}");
|
|
|
|
c.AlternateOf = null;
|
|
alternateOfRemoved++;
|
|
}
|
|
}
|
|
);
|
|
|
|
// Add all alternateOf definitions from the configuration that do not match an alternateOf definition in the model
|
|
int alternateOfAdded = 0;
|
|
if(tableConfig.AggregationRules != null)
|
|
tableConfig.AggregationRules.ForEach(r =>
|
|
{
|
|
var column = table.Columns.First(c => c.Name == r.AggTableColumn);
|
|
var detailTable = database.Model.Tables.Find(r.DetailTable);
|
|
var detailColumn = (r.DetailTableColumn != null) ? detailTable.Columns.Find(r.DetailTableColumn) : null;
|
|
|
|
if (column.AlternateOf == null)
|
|
{
|
|
if (r.Summarization == "CountTableRows")
|
|
column.AlternateOf = new AlternateOf
|
|
{
|
|
Summarization = SummarizationType.Count,
|
|
BaseTable = detailTable
|
|
};
|
|
else
|
|
column.AlternateOf = new AlternateOf
|
|
{
|
|
Summarization = (SummarizationType)Enum.Parse(typeof(SummarizationType), r.Summarization),
|
|
BaseColumn = detailColumn
|
|
};
|
|
|
|
alternateOfAdded++;
|
|
|
|
Console.ForegroundColor = ConsoleColor.White;
|
|
Console.WriteLine($"Adding aggregation rule for column [{column.Name}] with alternate of [{detailTable.Name + (detailColumn == null ? string.Empty : "].[" + detailColumn.Name)}] aggregation {column.AlternateOf.Summarization}");
|
|
}
|
|
}
|
|
);
|
|
|
|
return partitionModesChanged + alternateOfRemoved + alternateOfAdded;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Connect to analysis services instance and database, throw if error is encountered
|
|
/// </summary>
|
|
/// <param name="serverName"></param>
|
|
private void Connect(string serverName)
|
|
{
|
|
//string connectionString = $"Provider=MSOLAP;Data Source={args.Server};";
|
|
string connectionString = $"Provider=MSOLAP;Data Source={serverName};Integrated Security=SSPI;Persist Security Info=True;";
|
|
server.Connect(connectionString);
|
|
|
|
database = server.Databases.FindByName(aggsConfig.Database.Name);
|
|
|
|
if (database == null)
|
|
{
|
|
throw new Microsoft.AnalysisServices.ConnectionException($"Could not find database [{aggsConfig.Database.Name}]");
|
|
}
|
|
}
|
|
|
|
[ArgActionMethod, ArgDescription("Applies the Aggs configuration to the model specified")]
|
|
public void Apply(ServerAndConfigurationArgs args)
|
|
{
|
|
List<Microsoft.AnalysisServices.Tabular.Table> updatedTableCollection = new List<Microsoft.AnalysisServices.Tabular.Table>();
|
|
|
|
// Read configuration file
|
|
aggsConfig = JsonHelper.ToClass<AggsConfiguration>(File.ReadAllText(args.ConfigFile));
|
|
|
|
#region Connect
|
|
|
|
Connect(args.Server);
|
|
|
|
Console.ForegroundColor = ConsoleColor.White;
|
|
Console.WriteLine($"Start: {DateTime.Now.ToString("hh:mm:ss tt")}");
|
|
Console.WriteLine($"Server: {args.Server}");
|
|
Console.WriteLine($"Database: {aggsConfig.Database.Name}");
|
|
|
|
#endregion
|
|
|
|
#region Validate Configuration Against Model
|
|
|
|
aggsConfig.Database.Tables.ForEach(x => ValidateConfigurationOfTable(x));
|
|
|
|
Console.ForegroundColor = ConsoleColor.Green;
|
|
Console.WriteLine($"Validation successful");
|
|
|
|
#endregion
|
|
|
|
#region Update Model Compatibility
|
|
|
|
//Set database compat level
|
|
if (database.CompatibilityLevel < 1465)
|
|
{
|
|
database.CompatibilityLevel = 1465;
|
|
database.Update();
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Apply Model Changes
|
|
|
|
var changesApplied = 0;
|
|
aggsConfig.Database.Tables.ForEach(x =>
|
|
{
|
|
|
|
var delta = ApplyConfigurationOfTable(x);
|
|
|
|
if (delta > 0)
|
|
updatedTableCollection.Add(database.Model.Tables.Find(x.Name));
|
|
|
|
changesApplied += delta;
|
|
}
|
|
);
|
|
|
|
if (changesApplied == 0)
|
|
{
|
|
Console.ForegroundColor = ConsoleColor.Yellow;
|
|
Console.WriteLine($"No changes applied to model");
|
|
}
|
|
else
|
|
{
|
|
Console.ForegroundColor = ConsoleColor.Green;
|
|
Console.WriteLine($"Applied model changes");
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Expand Full
|
|
|
|
if (changesApplied == 0)
|
|
{
|
|
Console.ForegroundColor = ConsoleColor.White;
|
|
Console.WriteLine("Skipping write to database, no changes detected");
|
|
}
|
|
else
|
|
{
|
|
// Set database UpdateOptions to ExpandFull
|
|
try
|
|
{
|
|
Console.ForegroundColor = ConsoleColor.White;
|
|
Console.Write("Writing model changes to database ");
|
|
database.Update(Microsoft.AnalysisServices.UpdateOptions.ExpandFull);
|
|
Console.ForegroundColor = ConsoleColor.Green;
|
|
Console.WriteLine("SUCCESS");
|
|
}
|
|
catch (Exception e)
|
|
{
|
|
Console.ForegroundColor = ConsoleColor.Red;
|
|
Console.WriteLine("FAILED");
|
|
|
|
throw e;
|
|
}
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Refresh Tables as Required
|
|
|
|
if (changesApplied == 0)
|
|
{
|
|
Console.ForegroundColor = ConsoleColor.White;
|
|
Console.WriteLine("Skipping table refresh, no changes detected");
|
|
}
|
|
else
|
|
{
|
|
Console.ForegroundColor = ConsoleColor.White;
|
|
Console.WriteLine("Refreshing tables - begin: " + DateTime.Now.ToString("hh:mm:ss tt"));
|
|
|
|
foreach (var tableObj in updatedTableCollection)
|
|
{
|
|
Console.ForegroundColor = ConsoleColor.White;
|
|
Console.Write($"Refreshing table [{tableObj.Name}] ");
|
|
tableObj.RequestRefresh(RefreshType.Full);
|
|
Console.ForegroundColor = ConsoleColor.Green;
|
|
Console.WriteLine("COMPLETE");
|
|
}
|
|
|
|
database.Model.SaveChanges();
|
|
|
|
Console.ForegroundColor = ConsoleColor.White;
|
|
Console.WriteLine("Refreshing tables - end: " + DateTime.Now.ToString("hh:mm:ss tt"));
|
|
}
|
|
|
|
#endregion
|
|
}
|
|
}
|
|
|
|
public class ServerAndConfigurationArgs
|
|
{
|
|
[ArgRequired, ArgDescription("The URL to the Azure Analysis Services instance e.g.: asazure://host/instance"), ArgPosition(1)]
|
|
public string Server { get; set; }
|
|
|
|
[ArgRequired, ArgDescription("The file path to the JSON configuration file to apply to the database"), ArgPosition(2)]
|
|
public string ConfigFile { get; set; }
|
|
}
|
|
|
|
#endregion
|
|
|
|
class Program
|
|
{
|
|
static void Main(string[] args)
|
|
{
|
|
try
|
|
{
|
|
Args.InvokeAction<AggConfigurationUtility>(args);
|
|
}
|
|
catch (Exception exc)
|
|
{
|
|
Console.ForegroundColor = ConsoleColor.Red;
|
|
Console.WriteLine("");
|
|
Console.WriteLine($"Exception occurred: {DateTime.Now.ToString("hh:mm:ss tt")}");
|
|
Console.WriteLine($"Exception message: {exc.Message}");
|
|
}
|
|
finally
|
|
{
|
|
Console.ForegroundColor = ConsoleColor.White;
|
|
Console.WriteLine("Press any key to exit");
|
|
Console.ReadKey();
|
|
}
|
|
}
|
|
}
|
|
}
|