From 101f371ed99854e4bb0fd121276cd3928e655da9 Mon Sep 17 00:00:00 2001 From: christianwade Date: Sat, 3 Dec 2016 22:30:36 -0800 Subject: [PATCH] Merge v0 --- .../Program.cs | 2 + .../ConfigDatabaseHelper.cs | 4 +- .../PartitionProcessor.cs | 193 ++++++++++++++++-- 3 files changed, 178 insertions(+), 21 deletions(-) diff --git a/AsPartitionProcessing/AsPartitionProcessing.SampleClient/Program.cs b/AsPartitionProcessing/AsPartitionProcessing.SampleClient/Program.cs index 702d619..b4f8cfe 100644 --- a/AsPartitionProcessing/AsPartitionProcessing.SampleClient/Program.cs +++ b/AsPartitionProcessing/AsPartitionProcessing.SampleClient/Program.cs @@ -23,6 +23,8 @@ namespace AsPartitionProcessing.SampleClient modelsConfig = InitializeFromDatabase(); } + //PartitionProcessor.MergeMonthsToYear(modelsConfig[0], LogMessage, "Internet Sales", "2012"); + foreach (ModelConfiguration modelConfig in modelsConfig) { if (!modelConfig.IntegratedAuth) //For Azure AS diff --git a/AsPartitionProcessing/AsPartitionProcessing/ConfigDatabaseHelper.cs b/AsPartitionProcessing/AsPartitionProcessing/ConfigDatabaseHelper.cs index 5753125..c673ea4 100644 --- a/AsPartitionProcessing/AsPartitionProcessing/ConfigDatabaseHelper.cs +++ b/AsPartitionProcessing/AsPartitionProcessing/ConfigDatabaseHelper.cs @@ -123,7 +123,7 @@ namespace AsPartitionProcessing { command.Connection = connection; command.CommandType = CommandType.Text; - command.CommandText = "DELETE FROM [dbo].[PartitionedModelLog];"; + command.CommandText = "DELETE FROM [dbo].[ProcessingLog];"; command.ExecuteNonQuery(); } } @@ -144,7 +144,7 @@ namespace AsPartitionProcessing command.Connection = connection; command.CommandType = CommandType.Text; command.CommandText = @" - INSERT INTO [dbo].[PartitionedModelLog] + INSERT INTO [dbo].[ProcessingLog] ([ModelConfigurationID] ,[ExecutionID] ,[LogDateTime] diff --git a/AsPartitionProcessing/AsPartitionProcessing/PartitionProcessor.cs b/AsPartitionProcessing/AsPartitionProcessing/PartitionProcessor.cs index 35618ee..0660614 100644 --- a/AsPartitionProcessing/AsPartitionProcessing/PartitionProcessor.cs +++ b/AsPartitionProcessing/AsPartitionProcessing/PartitionProcessor.cs @@ -27,9 +27,15 @@ namespace AsPartitionProcessing /// public static class PartitionProcessor { + #region Member Variables + private static ModelConfiguration _modelConfiguration; private static LogMessageDelegate _messageLogger; + #endregion + + #region Public Methods + /// /// Partitions tables in a tabular model based on configuration /// @@ -80,11 +86,11 @@ namespace AsPartitionProcessing } else { - //Non-partitioned table. Process based on partitioning configuration(s). + //Partitioned table. Process based on partitioning configuration(s). Partition templatePartition = table.Partitions.Find(tableConfiguration.AnalysisServicesTable); if (templatePartition == null) { - throw new Microsoft.AnalysisServices.ConnectionException($"Table {tableConfiguration.AnalysisServicesTable} does not contain a partition with the same name to act as the template partition."); + throw new InvalidOperationException($"Table {tableConfiguration.AnalysisServicesTable} does not contain a partition with the same name to act as the template partition."); } foreach (PartitioningConfiguration partitioningConfiguration in tableConfiguration.PartitioningConfigurations) @@ -113,20 +119,7 @@ namespace AsPartitionProcessing } //Process partitions - string selectQueryTemplate; - switch (partitioningConfiguration.Granularity) - { - case Granularity.Daily: - selectQueryTemplate = "SELECT * FROM {0} WHERE {1} = {2} ORDER BY {1}"; - break; - case Granularity.Monthly: - selectQueryTemplate = "SELECT * FROM {0} WHERE FLOOR({1} / 100) = {2} ORDER BY {1}"; - break; - default: //Granularity.Yearly: - selectQueryTemplate = "SELECT * FROM {0} WHERE FLOOR({1} / 10000) = {2} ORDER BY {1}"; - break; - } - + string selectQueryTemplate = DeriveSelectQueryTemplate(partitioningConfiguration.Granularity); foreach (string partitionKey in partitionKeysForProcessing) { Partition partitionToProcess = table.Partitions.Find(partitionKey); @@ -216,7 +209,10 @@ namespace AsPartitionProcessing LogMessage("", false); LogMessage($"Exception occurred: {DateTime.Now.ToString("hh:mm:ss tt")}", false); LogMessage($"Exception message: {exc.Message}", false); - LogMessage($"Inner exception message: {exc.InnerException.Message}", false); + if (exc.InnerException != null) + { + LogMessage($"Inner exception message: {exc.InnerException.Message}", false); + } } finally { @@ -230,6 +226,143 @@ namespace AsPartitionProcessing } } + /// + /// Merge month partitions into a year partition. + /// + /// Configuration info for the model + /// Pointer to logging method + /// Name of the partitioned table in the tabular model. + /// Year partition key following yyyy + public static void MergeMonthsToYear(ModelConfiguration modelConfiguration, LogMessageDelegate messageLogger, string analysisServicesTable, string yearPartitionKey) + { + _modelConfiguration = modelConfiguration; + _messageLogger = messageLogger; + + Server server = new Server(); + try + { + //Check new partition key is expected format + int partitionKeyParsed; + if (yearPartitionKey.Length != 4 || !int.TryParse(yearPartitionKey, out partitionKeyParsed)) + { + throw new InvalidOperationException($"Partition key {yearPartitionKey} is not of expected format."); + } + + //Check the model configuration contains the partitioned table + bool foundMatch = false; + PartitioningConfiguration partitionConfig = null; + foreach (TableConfiguration tableConfig in modelConfiguration.TableConfigurations) + { + if (tableConfig.AnalysisServicesTable == analysisServicesTable && tableConfig.PartitioningConfigurations.Count > 0) + { + partitionConfig = tableConfig.PartitioningConfigurations[0]; + foundMatch = true; + break; + } + } + if (!foundMatch) + { + throw new InvalidOperationException($"Table {analysisServicesTable} not found in model configuration with at least one partitioning configuration (required for source table and column names)."); + } + + Database database; + Connect(server, out database); + + Table table = database.Model.Tables.Find(analysisServicesTable); + if (table == null) + { + throw new Microsoft.AnalysisServices.ConnectionException($"Could not connect to table {analysisServicesTable}."); + } + + //Find template partition + Partition templatePartition = table.Partitions.Find(analysisServicesTable); + if (templatePartition == null) + { + throw new InvalidOperationException($"Table {analysisServicesTable} does not contain a partition with the same name to act as the template partition."); + } + + //See if there is already a partition with same key - do not want to delete an existing partition in case of inadvertent data loss + if (table.Partitions.Find(yearPartitionKey) != null) + { + throw new InvalidOperationException($"Table {analysisServicesTable} already contains a partition with key {yearPartitionKey}. Please delete this partition and retry."); + } + + LogMessage("", false); + LogMessage($"Create new merged partition {yearPartitionKey} for table {analysisServicesTable}", false); + + //Create new partition + string selectQueryTemplate = DeriveSelectQueryTemplate(Granularity.Yearly); + Partition newPartition = new Partition(); + templatePartition.CopyTo(newPartition); + newPartition.Name = yearPartitionKey; + ((QueryPartitionSource)newPartition.Source).Query = String.Format(selectQueryTemplate, partitionConfig.SourceTableName, partitionConfig.SourcePartitionColumn, yearPartitionKey); + table.Partitions.Add(newPartition); + LogMessage($"Create new partition {DateFormatPartitionKey(yearPartitionKey, Granularity.Yearly)}", true); + + + //Merge each month partition and delete it + List monthKeysToBeMerged = GetPartitionKeysTable(table, Granularity.Monthly, yearPartitionKey); + if (monthKeysToBeMerged.Count == 0) + { + LogMessage($"No partitinos found in {analysisServicesTable} to be merged into {yearPartitionKey} ...", true); + } + else + { + List partitionsToBeMerged = new List(); + foreach (string monthKey in monthKeysToBeMerged) + { + LogMessage($"Partition {DateFormatPartitionKey(monthKey, Granularity.Monthly)} to be merged into {DateFormatPartitionKey(yearPartitionKey, Granularity.Yearly)}", true); + partitionsToBeMerged.Add(table.Partitions.Find(monthKey)); + } + + newPartition.RequestMerge(partitionsToBeMerged); + LogMessage($"Save changes for table {analysisServicesTable} ...", true); + database.Model.SaveChanges(); + } + + Console.ForegroundColor = ConsoleColor.White; + LogMessage("", false); + LogMessage("Finish: " + DateTime.Now.ToString("hh:mm:ss tt"), false); + } + catch (Exception exc) + { + Console.ForegroundColor = ConsoleColor.Red; + LogMessage("", false); + LogMessage($"Exception occurred: {DateTime.Now.ToString("hh:mm:ss tt")}", false); + LogMessage($"Exception message: {exc.Message}", false); + if (exc.InnerException != null) + { + LogMessage($"Inner exception message: {exc.InnerException.Message}", false); + } + } + finally + { + try + { + _modelConfiguration = null; + _messageLogger = null; + if (server != null) server.Disconnect(); + } + catch { } + } + } + + /// + /// Merge day partitions into a month partition. + /// + /// Configuration info for the model + /// Pointer to logging method + /// Month partition key following yyyymm + public static void MergeDaysToMonth(ModelConfiguration modelConfiguration, LogMessageDelegate messageLogger, string monthPartitionKey) + { + + + } + + #endregion + + #region Private Methods + private static void IncrementalProcessPartition(string partitionKey, Partition partitionToProcess, Granularity granularity) { if (_modelConfiguration.IncrementalOnline) @@ -340,7 +473,7 @@ namespace AsPartitionProcessing return partitionKeys; } - private static List GetPartitionKeysTable(Table table, Granularity granularity) + private static List GetPartitionKeysTable(Table table, Granularity granularity, string filter = "") { List partitionKeysExisting = new List(); @@ -352,7 +485,8 @@ namespace AsPartitionProcessing ( (partition.Name.Length == 4 && granularity == Granularity.Yearly) || (partition.Name.Length == 6 && granularity == Granularity.Monthly) || (partition.Name.Length == 8 && granularity == Granularity.Daily) - ) && int.TryParse(partition.Name, out partitionKey) + ) && int.TryParse(partition.Name, out partitionKey) && + partition.Name.StartsWith(filter) ) { partitionKeysExisting.Add(Convert.ToString(partitionKey)); @@ -362,6 +496,27 @@ namespace AsPartitionProcessing return partitionKeysExisting; } + + private static string DeriveSelectQueryTemplate(Granularity granularity) + { + string selectQueryTemplate; + switch (granularity) + { + case Granularity.Daily: + selectQueryTemplate = "SELECT * FROM {0} WHERE {1} = {2} ORDER BY {1}"; + break; + case Granularity.Monthly: + selectQueryTemplate = "SELECT * FROM {0} WHERE FLOOR({1} / 100) = {2} ORDER BY {1}"; + break; + default: //Granularity.Yearly: + selectQueryTemplate = "SELECT * FROM {0} WHERE FLOOR({1} / 10000) = {2} ORDER BY {1}"; + break; + } + + return selectQueryTemplate; + } + + #endregion } }