mujiannan/Analysis-Services
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

merge partitions

Browse Source
This commit is contained in:
christianwade 2016-12-05 21:37:16 -08:00
parent 101f371ed9
commit ff012a5c4f
2 changed files with 110 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
AsPartitionProcessing/AsPartitionProcessing.SampleClient/Program.cs
View File

@ -5,26 +5,32 @@ using Microsoft.AnalysisServices.Tabular;
namespace AsPartitionProcessing.SampleClient namespace AsPartitionProcessing.SampleClient
{ {
enum SampleExecutionMode
{
InitializeInline,
InitializeFromDatabase,
MergePartitions
}
class Program class Program
{ {
const bool UseDatabase = false; //Set sample execution mode here:
const SampleExecutionMode execMode = SampleExecutionMode.InitializeInline;
static void Main(string[] args) static void Main(string[] args)
{ {
try try
{ {
List<ModelConfiguration> modelsConfig; List<ModelConfiguration> modelsConfig;
if (!UseDatabase) if (execMode == SampleExecutionMode.InitializeInline)
{ {
modelsConfig = InitializeAdventureWorksInline(); modelsConfig = InitializeInline();
} }
else else
{ {
modelsConfig = InitializeFromDatabase(); modelsConfig = InitializeFromDatabase();
} }
//PartitionProcessor.MergeMonthsToYear(modelsConfig[0], LogMessage, "Internet Sales", "2012");
foreach (ModelConfiguration modelConfig in modelsConfig) foreach (ModelConfiguration modelConfig in modelsConfig)
{ {
if (!modelConfig.IntegratedAuth) //For Azure AS if (!modelConfig.IntegratedAuth) //For Azure AS
@ -36,8 +42,14 @@ namespace AsPartitionProcessing.SampleClient
modelConfig.Password = ReadPassword(); modelConfig.Password = ReadPassword();
} }
//Most important method: if (execMode == SampleExecutionMode.MergePartitions)
PartitionProcessor.PerformProcessing(modelConfig, LogMessage); {
PartitionProcessor.MergePartitions(modelConfig, LogMessage, "Internet Sales", Granularity.Yearly, "2012");
}
else
{
PartitionProcessor.PerformProcessing(modelConfig, LogMessage);
}
} }
} }
catch (Exception exc) catch (Exception exc)
@ -51,7 +63,7 @@ namespace AsPartitionProcessing.SampleClient
Console.ReadKey(); Console.ReadKey();
} }
private static List<ModelConfiguration> InitializeAdventureWorksInline() private static List<ModelConfiguration> InitializeInline()
{ {
ModelConfiguration partitionedModel = new ModelConfiguration( ModelConfiguration partitionedModel = new ModelConfiguration(
modelConfigurationID: 1, modelConfigurationID: 1,
@ -140,10 +152,13 @@ namespace AsPartitionProcessing.SampleClient
private static void LogMessage(string message, ModelConfiguration partitionedModel) private static void LogMessage(string message, ModelConfiguration partitionedModel)
{ {
//Can provide custom logger here //Can provide custom logger here
try try
{ {
if (UseDatabase) if (!(execMode == SampleExecutionMode.InitializeInline))
{
ConfigDatabaseHelper.LogMessage(message, partitionedModel); ConfigDatabaseHelper.LogMessage(message, partitionedModel);
}
Console.WriteLine(message); Console.WriteLine(message);
} }

142
AsPartitionProcessing/AsPartitionProcessing/PartitionProcessor.cs
View File

@ -100,9 +100,9 @@ namespace AsPartitionProcessing
LogMessage(new String('-', tableConfiguration.AnalysisServicesTable.Length + 38), false); LogMessage(new String('-', tableConfiguration.AnalysisServicesTable.Length + 38), false);
//Figure out what processing needs to be done //Figure out what processing needs to be done
List<string> partitionKeysCurrent = GetPartitionKeysTable(table, partitioningConfiguration.Granularity); List<string> partitionKeysCurrent = GetPartitionKeysCurrent(table, partitioningConfiguration.Granularity);
List<string> partitionKeysNew = GetPartitionKeys(false, partitioningConfiguration, partitioningConfiguration.Granularity); List<string> partitionKeysNew = GetPartitionKeysTarget(false, partitioningConfiguration, partitioningConfiguration.Granularity);
List<string> partitionKeysForProcessing = GetPartitionKeys(true, partitioningConfiguration, partitioningConfiguration.Granularity); List<string> partitionKeysForProcessing = GetPartitionKeysTarget(true, partitioningConfiguration, partitioningConfiguration.Granularity);
DisplayPartitionRange(partitionKeysCurrent, true, partitioningConfiguration.Granularity); DisplayPartitionRange(partitionKeysCurrent, true, partitioningConfiguration.Granularity);
DisplayPartitionRange(partitionKeysNew, false, partitioningConfiguration.Granularity); DisplayPartitionRange(partitionKeysNew, false, partitioningConfiguration.Granularity);
LogMessage("", false); LogMessage("", false);
@ -119,19 +119,13 @@ namespace AsPartitionProcessing
} }
//Process partitions //Process partitions
string selectQueryTemplate = DeriveSelectQueryTemplate(partitioningConfiguration.Granularity);
foreach (string partitionKey in partitionKeysForProcessing) foreach (string partitionKey in partitionKeysForProcessing)
{ {
Partition partitionToProcess = table.Partitions.Find(partitionKey); Partition partitionToProcess = table.Partitions.Find(partitionKey);
if (partitionToProcess == null) if (partitionToProcess == null)
{ {
//Doesn't exist so create it partitionToProcess = CreateNewPartition(table, templatePartition, partitioningConfiguration, partitionKey);
partitionToProcess = new Partition();
templatePartition.CopyTo(partitionToProcess);
partitionToProcess.Name = partitionKey;
((QueryPartitionSource)partitionToProcess.Source).Query = String.Format(selectQueryTemplate, partitioningConfiguration.SourceTableName, partitioningConfiguration.SourcePartitionColumn, partitionKey);
table.Partitions.Add(partitionToProcess);
LogMessage($"Create new partition {DateFormatPartitionKey(partitionKey, partitioningConfiguration.Granularity)}", true); LogMessage($"Create new partition {DateFormatPartitionKey(partitionKey, partitioningConfiguration.Granularity)}", true);
if (!_modelConfiguration.InitialSetUp) if (!_modelConfiguration.InitialSetUp)
@ -226,14 +220,28 @@ namespace AsPartitionProcessing
} }
} }
private static Partition CreateNewPartition(Table table, Partition templatePartition, PartitioningConfiguration partitioningConfiguration, string partitionKey)
{
Partition partitionToProcess;
//Doesn't exist so create it
string selectQueryTemplate = DeriveSelectQueryTemplate(partitioningConfiguration.Granularity);
partitionToProcess = new Partition();
templatePartition.CopyTo(partitionToProcess);
partitionToProcess.Name = partitionKey;
((QueryPartitionSource)partitionToProcess.Source).Query = String.Format(selectQueryTemplate, partitioningConfiguration.SourceTableName, partitioningConfiguration.SourcePartitionColumn, partitionKey);
table.Partitions.Add(partitionToProcess);
return partitionToProcess;
}
/// <summary> /// <summary>
/// Merge month partitions into a year partition. /// Merge months into a year, or days into a month.
/// </summary> /// </summary>
/// <param name="modelConfiguration">Configuration info for the model</param> /// <param name="modelConfiguration">Configuration info for the model</param>
/// <param name="messageLogger">Pointer to logging method</param> /// <param name="messageLogger">Pointer to logging method</param>
/// <param name="analysisServicesTable">Name of the partitioned table in the tabular model.</param> /// <param name="analysisServicesTable">Name of the partitioned table in the tabular model.</param>
/// <param name="yearPartitionKey">Year partition key following yyyy</param> /// <param name="targetGranularity">Granularity of the newly created partition. Must be year or month.</param>
public static void MergeMonthsToYear(ModelConfiguration modelConfiguration, LogMessageDelegate messageLogger, string analysisServicesTable, string yearPartitionKey) /// <param name="partitionKey">Target partition key. If year, follow yyyy; if month follow yyyymm.</param>
public static void MergePartitions(ModelConfiguration modelConfiguration, LogMessageDelegate messageLogger, string analysisServicesTable, Granularity targetGranularity, string partitionKey)
{ {
_modelConfiguration = modelConfiguration; _modelConfiguration = modelConfiguration;
_messageLogger = messageLogger; _messageLogger = messageLogger;
@ -241,14 +249,24 @@ namespace AsPartitionProcessing
Server server = new Server(); Server server = new Server();
try try
{ {
//Check new partition key is expected format //Check target granularity
int partitionKeyParsed; if (targetGranularity == Granularity.Daily)
if (yearPartitionKey.Length != 4 || !int.TryParse(yearPartitionKey, out partitionKeyParsed))
{ {
throw new InvalidOperationException($"Partition key {yearPartitionKey} is not of expected format."); throw new InvalidOperationException($"Target granularity for merging must be year or month.");
} }
//Check the model configuration contains the partitioned table //Check new partition key is expected format
int partitionKeyParsed;
if ( !(
(partitionKey.Length == 4 && targetGranularity == Granularity.Yearly) ||
(partitionKey.Length == 6 && targetGranularity == Granularity.Monthly)
) || !int.TryParse(partitionKey, out partitionKeyParsed)
)
{
throw new InvalidOperationException($"Partition key {partitionKey} is not of expected format.");
}
//Check configuration contains the partitioned table
bool foundMatch = false; bool foundMatch = false;
PartitioningConfiguration partitionConfig = null; PartitioningConfiguration partitionConfig = null;
foreach (TableConfiguration tableConfig in modelConfiguration.TableConfigurations) foreach (TableConfiguration tableConfig in modelConfiguration.TableConfigurations)
@ -262,7 +280,7 @@ namespace AsPartitionProcessing
} }
if (!foundMatch) if (!foundMatch)
{ {
throw new InvalidOperationException($"Table {analysisServicesTable} not found in model configuration with at least one partitioning configuration (required for source table and column names)."); throw new InvalidOperationException($"Table {analysisServicesTable} not found in configuration with at least one partitioning configuration defined.");
} }
Database database; Database database;
@ -282,47 +300,39 @@ namespace AsPartitionProcessing
} }
//See if there is already a partition with same key - do not want to delete an existing partition in case of inadvertent data loss //See if there is already a partition with same key - do not want to delete an existing partition in case of inadvertent data loss
if (table.Partitions.Find(yearPartitionKey) != null) if (table.Partitions.Find(partitionKey) != null)
{ {
throw new InvalidOperationException($"Table {analysisServicesTable} already contains a partition with key {yearPartitionKey}. Please delete this partition and retry."); throw new InvalidOperationException($"Table {analysisServicesTable} already contains a partition with key {partitionKey}. Please delete this partition and retry.");
} }
LogMessage("", false);
LogMessage($"Create new merged partition {yearPartitionKey} for table {analysisServicesTable}", false);
//Create new partition //Check there are partitions to be merged
string selectQueryTemplate = DeriveSelectQueryTemplate(Granularity.Yearly); Granularity childGranularity = targetGranularity == Granularity.Yearly ? Granularity.Monthly : Granularity.Daily;
Partition newPartition = new Partition(); List<Partition> partitionsToBeMerged = GetPartitionsCurrent(table, childGranularity, partitionKey);
templatePartition.CopyTo(newPartition); if (partitionsToBeMerged.Count == 0)
newPartition.Name = yearPartitionKey;
((QueryPartitionSource)newPartition.Source).Query = String.Format(selectQueryTemplate, partitionConfig.SourceTableName, partitionConfig.SourcePartitionColumn, yearPartitionKey);
table.Partitions.Add(newPartition);
LogMessage($"Create new partition {DateFormatPartitionKey(yearPartitionKey, Granularity.Yearly)}", true);
//Merge each month partition and delete it
List<string> monthKeysToBeMerged = GetPartitionKeysTable(table, Granularity.Monthly, yearPartitionKey);
if (monthKeysToBeMerged.Count == 0)
{ {
LogMessage($"No partitinos found in {analysisServicesTable} to be merged into {yearPartitionKey} ...", true); LogMessage($"No partitinos found in {analysisServicesTable} to be merged into {partitionKey}.", false);
} }
else else
{ {
List<Partition> partitionsToBeMerged = new List<Partition>(); //Done with validation, so go ahead ...
foreach (string monthKey in monthKeysToBeMerged) LogMessage("", false);
LogMessage($"Create new merged partition {DateFormatPartitionKey(partitionKey, targetGranularity)} for table {analysisServicesTable}", true);
Partition newPartition = CreateNewPartition(table, templatePartition, partitionConfig, partitionKey);
foreach (Partition partition in partitionsToBeMerged)
{ {
LogMessage($"Partition {DateFormatPartitionKey(monthKey, Granularity.Monthly)} to be merged into {DateFormatPartitionKey(yearPartitionKey, Granularity.Yearly)}", true); LogMessage($"Partition {partition.Name} to be merged into {DateFormatPartitionKey(partitionKey, targetGranularity)}", true);
partitionsToBeMerged.Add(table.Partitions.Find(monthKey));
} }
newPartition.RequestMerge(partitionsToBeMerged); newPartition.RequestMerge(partitionsToBeMerged);
LogMessage($"Save changes for table {analysisServicesTable} ...", true); LogMessage($"Save changes for table {analysisServicesTable} ...", true);
database.Model.SaveChanges(); database.Model.SaveChanges();
}
Console.ForegroundColor = ConsoleColor.White; Console.ForegroundColor = ConsoleColor.White;
LogMessage("", false); LogMessage("", false);
LogMessage("Finish: " + DateTime.Now.ToString("hh:mm:ss tt"), false); LogMessage("Finish: " + DateTime.Now.ToString("hh:mm:ss tt"), false);
}
} }
catch (Exception exc) catch (Exception exc)
{ {
@ -442,7 +452,7 @@ namespace AsPartitionProcessing
} }
} }
private static List<string> GetPartitionKeys(bool forProcessing, PartitioningConfiguration partitioningConfiguration, Granularity granularity) private static List<string> GetPartitionKeysTarget(bool forProcessing, PartitioningConfiguration partitioningConfiguration, Granularity granularity)
{ {
//forProcessing: false to return complete target set of partitions, true to return partitons to be processed (may be less if incremental mode). //forProcessing: false to return complete target set of partitions, true to return partitons to be processed (may be less if incremental mode).
@ -473,21 +483,14 @@ namespace AsPartitionProcessing
return partitionKeys; return partitionKeys;
} }
private static List<string> GetPartitionKeysTable(Table table, Granularity granularity, string filter = "") private static List<string> GetPartitionKeysCurrent(Table table, Granularity granularity, string filter = "")
{ {
List<string> partitionKeysExisting = new List<string>(); List<string> partitionKeysExisting = new List<string>();
foreach (Partition partition in table.Partitions) foreach (Partition partition in table.Partitions)
{ {
//Ignore partitions that don't follow the convention yyyy, yyyymm or yyyymmdd int partitionKey = -1;
int partitionKey; if (IncludePartition(granularity, filter, partition, ref partitionKey))
if (
( (partition.Name.Length == 4 && granularity == Granularity.Yearly) ||
(partition.Name.Length == 6 && granularity == Granularity.Monthly) ||
(partition.Name.Length == 8 && granularity == Granularity.Daily)
) && int.TryParse(partition.Name, out partitionKey) &&
partition.Name.StartsWith(filter)
)
{ {
partitionKeysExisting.Add(Convert.ToString(partitionKey)); partitionKeysExisting.Add(Convert.ToString(partitionKey));
} }
@ -497,6 +500,33 @@ namespace AsPartitionProcessing
return partitionKeysExisting; return partitionKeysExisting;
} }
private static List<Partition> GetPartitionsCurrent(Table table, Granularity granularity, string filter = "")
{
List<Partition> partitionsExisting = new List<Partition>();
foreach (Partition partition in table.Partitions)
{
int partitionKey = -1;
if (IncludePartition(granularity, filter, partition, ref partitionKey))
{
partitionsExisting.Add(partition);
}
}
partitionsExisting.Sort();
return partitionsExisting;
}
private static bool IncludePartition(Granularity granularity, string filter, Partition partition, ref int partitionKey)
{
//Ignore partitions that don't follow the convention yyyy, yyyymm or yyyymmdd, or are not included in the filter expression
return ( (partition.Name.Length == 4 && granularity == Granularity.Yearly) ||
(partition.Name.Length == 6 && granularity == Granularity.Monthly) ||
(partition.Name.Length == 8 && granularity == Granularity.Daily)
) && int.TryParse(partition.Name, out partitionKey) &&
partition.Name.StartsWith(filter);
}
private static string DeriveSelectQueryTemplate(Granularity granularity) private static string DeriveSelectQueryTemplate(Granularity granularity)
{ {
string selectQueryTemplate; string selectQueryTemplate;