mujiannan/Analysis-Services
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

1400 v3

Browse Source
This commit is contained in:
Christian Wade 2017-04-18 18:48:52 -07:00
parent e4706cbd20
commit 46a066fa13
9 changed files with 164 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
AsPartitionProcessing/AsPartitionProcessing.ConfigurationLogging/SampleConfiguration.sql
View File

@ -8,6 +8,8 @@ VALUES(
,1 --[IntegratedAuth] ,1 --[IntegratedAuth]
,-1 --[MaxParallelism] ,-1 --[MaxParallelism]
,-1 --[CommitTimeout] ,-1 --[CommitTimeout]
,0 --[RetryAttempts]
,0 --[RetryWaitTimeSeconds]
); );
INSERT INTO [dbo].[TableConfiguration] INSERT INTO [dbo].[TableConfiguration]

22
AsPartitionProcessing/AsPartitionProcessing.ConfigurationLogging/dbo/Stored Procedures/usp_LastProcessingLogs.sql
View File

@ -1,8 +1,16 @@
CREATE PROC [dbo].[usp_LastProcessingLogs] AS CREATE PROC [dbo].[usp_LastProcessingLogs]
SELECT [Message] @ExecutionCount tinyint = 1,
FROM [dbo].[ProcessingLog] @ErrorsOnly bit = 0
WHERE ExecutionID = AS
( SELECT MAX([ExecutionID]) FROM [dbo].[ProcessingLog] SELECT --l.ExecutionID,
WHERE [LogDateTime] = (SELECT MAX([LogDateTime]) FROM [dbo].[ProcessingLog]) l.[LogDateTime],
) l.[Message]
FROM [dbo].[ProcessingLog] l
INNER JOIN
( SELECT TOP (@ExecutionCount) [ExecutionID], MAX([LogDateTime]) [MaxLogDateTime]
FROM [dbo].[ProcessingLog]
GROUP BY ExecutionID
ORDER BY [MaxLogDateTime] DESC
) dt ON l.ExecutionID = dt.ExecutionID
WHERE @ErrorsOnly = 0 OR (@ErrorsOnly = 1 AND l.MessageType = 'Error')
ORDER BY [LogDateTime] ORDER BY [LogDateTime]

2
AsPartitionProcessing/AsPartitionProcessing.ConfigurationLogging/dbo/Tables/ModelConfiguration.sql
View File

@ -7,6 +7,8 @@
[IntegratedAuth] BIT NOT NULL, [IntegratedAuth] BIT NOT NULL,
[MaxParallelism] INT NOT NULL, [MaxParallelism] INT NOT NULL,
[CommitTimeout] INT NOT NULL, [CommitTimeout] INT NOT NULL,
[RetryAttempts] TINYINT NOT NULL,
[RetryWaitTimeSeconds] INT NOT NULL,
CONSTRAINT [PK_ModelConfiguration] PRIMARY KEY CLUSTERED ([ModelConfigurationID] ASC) CONSTRAINT [PK_ModelConfiguration] PRIMARY KEY CLUSTERED ([ModelConfigurationID] ASC)
); );

1
AsPartitionProcessing/AsPartitionProcessing.ConfigurationLogging/dbo/Tables/ProcessingLog.sql
View File

@ -4,6 +4,7 @@
[ExecutionID] CHAR (36) NOT NULL, [ExecutionID] CHAR (36) NOT NULL,
[LogDateTime] DATETIME NOT NULL, [LogDateTime] DATETIME NOT NULL,
[Message] VARCHAR (8000) NOT NULL, [Message] VARCHAR (8000) NOT NULL,
[MessageType] NVARCHAR(50) NOT NULL,
CONSTRAINT [PK_ProcessingLog] PRIMARY KEY CLUSTERED ([PartitioningLogID] ASC), CONSTRAINT [PK_ProcessingLog] PRIMARY KEY CLUSTERED ([PartitioningLogID] ASC),
CONSTRAINT [FK_ProcessingLog_ModelConfiguration] FOREIGN KEY ([ModelConfigurationID]) REFERENCES [dbo].[ModelConfiguration] ([ModelConfigurationID]) CONSTRAINT [FK_ProcessingLog_ModelConfiguration] FOREIGN KEY ([ModelConfigurationID]) REFERENCES [dbo].[ModelConfiguration] ([ModelConfigurationID])
); );

2
AsPartitionProcessing/AsPartitionProcessing.ConfigurationLogging/dbo/Views/vPartitioningConfiguration.sql
View File

@ -11,6 +11,8 @@ SELECT m.[ModelConfigurationID]
,m.[IntegratedAuth] ,m.[IntegratedAuth]
,m.[MaxParallelism] ,m.[MaxParallelism]
,m.[CommitTimeout] ,m.[CommitTimeout]
,m.[RetryAttempts]
,m.[RetryWaitTimeSeconds]
,t.[TableConfigurationID] ,t.[TableConfigurationID]
,t.[AnalysisServicesTable] ,t.[AnalysisServicesTable]
,t.[DoNotProcess] ,t.[DoNotProcess]

6
AsPartitionProcessing/AsPartitionProcessing.SampleClient/Program.cs
View File

@ -138,6 +138,8 @@ namespace AsPartitionProcessing.SampleClient
password: "", password: "",
maxParallelism: -1, maxParallelism: -1,
commitTimeout: -1, commitTimeout: -1,
retryAttempts: 0,
retryWaitTimeSeconds: 0,
tableConfigurations: tableConfigurations:
new List<TableConfiguration> new List<TableConfiguration>
{ {
@ -279,7 +281,7 @@ namespace AsPartitionProcessing.SampleClient
} }
} }
private static void LogMessage(string message, ModelConfiguration partitionedModel) private static void LogMessage(string message, MessageType messageType, ModelConfiguration partitionedModel)
{ {
//Can provide custom logger here //Can provide custom logger here
@ -287,7 +289,7 @@ namespace AsPartitionProcessing.SampleClient
{ {
if (!(_executionMode == ExecutionMode.InitializeInline)) if (!(_executionMode == ExecutionMode.InitializeInline))
{ {
ConfigDatabaseHelper.LogMessage(message, partitionedModel); ConfigDatabaseHelper.LogMessage(message, messageType, partitionedModel);
} }
Console.WriteLine(message); Console.WriteLine(message);

26
AsPartitionProcessing/AsPartitionProcessing/ConfigDatabaseHelper.cs
View File

@ -27,6 +27,8 @@ namespace AsPartitionProcessing
,[IntegratedAuth] ,[IntegratedAuth]
,[MaxParallelism] ,[MaxParallelism]
,[CommitTimeout] ,[CommitTimeout]
,[RetryAttempts]
,[RetryWaitTimeSeconds]
,[TableConfigurationID] ,[TableConfigurationID]
,[AnalysisServicesTable] ,[AnalysisServicesTable]
,[Partitioned] ,[Partitioned]
@ -37,7 +39,6 @@ namespace AsPartitionProcessing
,[MaxDateIsNow] ,[MaxDateIsNow]
,[MaxDate] ,[MaxDate]
,[IntegerDateKey] ,[IntegerDateKey]
,[TemplateSourceQuery] ,[TemplateSourceQuery]
FROM [dbo].[vPartitioningConfiguration] FROM [dbo].[vPartitioningConfiguration]
WHERE [DoNotProcess] = 0 {0} WHERE [DoNotProcess] = 0 {0}
@ -79,6 +80,8 @@ namespace AsPartitionProcessing
modelConfig.IntegratedAuth = Convert.ToBoolean(reader["IntegratedAuth"]); modelConfig.IntegratedAuth = Convert.ToBoolean(reader["IntegratedAuth"]);
modelConfig.MaxParallelism = Convert.ToInt32(reader["MaxParallelism"]); modelConfig.MaxParallelism = Convert.ToInt32(reader["MaxParallelism"]);
modelConfig.CommitTimeout = Convert.ToInt32(reader["CommitTimeout"]); modelConfig.CommitTimeout = Convert.ToInt32(reader["CommitTimeout"]);
modelConfig.RetryAttempts = Convert.ToInt32(reader["RetryAttempts"]);
modelConfig.RetryWaitTimeSeconds = Convert.ToInt32(reader["RetryWaitTimeSeconds"]);
modelConfig.ConfigDatabaseConnectionInfo = connectionInfo; modelConfig.ConfigDatabaseConnectionInfo = connectionInfo;
currentModelConfigurationID = modelConfig.ModelConfigurationID; currentModelConfigurationID = modelConfig.ModelConfigurationID;
@ -142,7 +145,7 @@ namespace AsPartitionProcessing
/// </summary> /// </summary>
/// <param name="message">Message to be logged.</param> /// <param name="message">Message to be logged.</param>
/// <param name="partitionedModel">Partitioned model with configuration information.</param> /// <param name="partitionedModel">Partitioned model with configuration information.</param>
public static void LogMessage(string message, ModelConfiguration partitionedModel) public static void LogMessage(string message, MessageType messageType, ModelConfiguration partitionedModel)
{ {
using (var connection = new SqlConnection(GetConnectionString(partitionedModel.ConfigDatabaseConnectionInfo))) using (var connection = new SqlConnection(GetConnectionString(partitionedModel.ConfigDatabaseConnectionInfo)))
{ {
@ -156,12 +159,14 @@ namespace AsPartitionProcessing
([ModelConfigurationID] ([ModelConfigurationID]
,[ExecutionID] ,[ExecutionID]
,[LogDateTime] ,[LogDateTime]
,[Message]) ,[Message]
,[MessageType])
VALUES VALUES
(@ModelConfigurationID (@ModelConfigurationID
,@ExecutionID ,@ExecutionID
,@LogDateTime ,@LogDateTime
,@Message);"; ,@Message
,@MessageType);";
SqlParameter parameter; SqlParameter parameter;
@ -181,6 +186,10 @@ namespace AsPartitionProcessing
parameter.Value = message; parameter.Value = message;
command.Parameters.Add(parameter); command.Parameters.Add(parameter);
parameter = new SqlParameter("@MessageType", SqlDbType.VarChar, 50);
parameter.Value = messageType.ToString();
command.Parameters.Add(parameter);
command.ExecuteNonQuery(); command.ExecuteNonQuery();
} }
} }
@ -201,4 +210,13 @@ namespace AsPartitionProcessing
return connectionString; return connectionString;
} }
} }
/// <summary>
/// Enumeration of log message types.
/// </summary>
public enum MessageType
{
Informational,
Error
}
} }

16
AsPartitionProcessing/AsPartitionProcessing/ModelConfiguration.cs
View File

@ -58,6 +58,16 @@ namespace AsPartitionProcessing
/// </summary> /// </summary>
public int CommitTimeout { get; set; } public int CommitTimeout { get; set; }
/// <summary>
/// Number of times a retry of the processing operation will be performed if an error occurs. Use for near-real time scenarios and environments with network reliability issues.
/// </summary>
public int RetryAttempts { get; set; }
/// <summary>
/// Number of seconds to wait before a retry attempt.
/// </summary>
public int RetryWaitTimeSeconds { get; set; }
/// <summary> /// <summary>
/// Collection of partitioned tables containing configuration information. /// Collection of partitioned tables containing configuration information.
/// </summary> /// </summary>
@ -86,6 +96,8 @@ namespace AsPartitionProcessing
/// <param name="password">Only applies when integratedAuth=false. Used for Azure AD UPNs to connect to Azure AS.</param> /// <param name="password">Only applies when integratedAuth=false. Used for Azure AD UPNs to connect to Azure AS.</param>
/// <param name="maxParallelism">Sets the maximum number of threads on which to run processing commands in parallel. -1 will not set the value.</param> /// <param name="maxParallelism">Sets the maximum number of threads on which to run processing commands in parallel. -1 will not set the value.</param>
/// <param name="commitTimeout">Set to override of CommitTimeout server property value for the connection. -1 will not override; the server value will be used.</param> /// <param name="commitTimeout">Set to override of CommitTimeout server property value for the connection. -1 will not override; the server value will be used.</param>
/// <param name="retryAttempts">Number of times a retry of the processing operation will be performed if an error occurs. Use for near-real time scenarios and environments with network reliability issues.</param>
/// <param name="retryWaitTimeSeconds">Number of seconds to wait before a retry attempt.</param>
/// <param name="tableConfigurations">Collection of partitioned tables containing configuration information.</param> /// <param name="tableConfigurations">Collection of partitioned tables containing configuration information.</param>
public ModelConfiguration( public ModelConfiguration(
int modelConfigurationID, int modelConfigurationID,
@ -98,6 +110,8 @@ namespace AsPartitionProcessing
string password, string password,
int maxParallelism, int maxParallelism,
int commitTimeout, int commitTimeout,
int retryAttempts,
int retryWaitTimeSeconds,
List<TableConfiguration> tableConfigurations List<TableConfiguration> tableConfigurations
) )
{ {
@ -111,6 +125,8 @@ namespace AsPartitionProcessing
Password = password; Password = password;
MaxParallelism = maxParallelism; MaxParallelism = maxParallelism;
CommitTimeout = commitTimeout; CommitTimeout = commitTimeout;
RetryAttempts = retryAttempts;
RetryWaitTimeSeconds = retryWaitTimeSeconds;
TableConfigurations = tableConfigurations; TableConfigurations = tableConfigurations;
ExecutionID = Guid.NewGuid().ToString(); ExecutionID = Guid.NewGuid().ToString();
} }

173
AsPartitionProcessing/AsPartitionProcessing/PartitionProcessor.cs
View File

@ -1,6 +1,7 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using Microsoft.AnalysisServices.Tabular; using Microsoft.AnalysisServices.Tabular;
using System.Threading;
//----------- //-----------
@ -20,7 +21,7 @@ namespace AsPartitionProcessing
/// </summary> /// </summary>
/// <param name="message">The message to be logged</param> /// <param name="message">The message to be logged</param>
/// <param name="modelConfiguration">Configuration info for the model</param> /// <param name="modelConfiguration">Configuration info for the model</param>
public delegate void LogMessageDelegate(string message, ModelConfiguration modelConfiguration); public delegate void LogMessageDelegate(string message, MessageType messageType, ModelConfiguration modelConfiguration);
/// <summary> /// <summary>
/// Processor of partitions in AS tabular models /// Processor of partitions in AS tabular models
@ -31,6 +32,7 @@ namespace AsPartitionProcessing
private static ModelConfiguration _modelConfiguration; private static ModelConfiguration _modelConfiguration;
private static LogMessageDelegate _messageLogger; private static LogMessageDelegate _messageLogger;
private static int _retryAttempts;
#endregion #endregion
@ -45,7 +47,13 @@ namespace AsPartitionProcessing
{ {
_modelConfiguration = modelConfiguration; _modelConfiguration = modelConfiguration;
_messageLogger = messageLogger; _messageLogger = messageLogger;
_retryAttempts = modelConfiguration.RetryAttempts;
PerformProcessing();
}
private static void PerformProcessing()
{
Server server = new Server(); Server server = new Server();
try try
{ {
@ -53,9 +61,9 @@ namespace AsPartitionProcessing
Connect(server, out database); Connect(server, out database);
Console.ForegroundColor = ConsoleColor.White; Console.ForegroundColor = ConsoleColor.White;
LogMessage($"Start: {DateTime.Now.ToString("hh:mm:ss tt")}", false); LogMessage($"Start: {DateTime.Now.ToString("hh:mm:ss tt")}", MessageType.Informational, false);
LogMessage($"Server: {_modelConfiguration.AnalysisServicesServer}", false); LogMessage($"Server: {_modelConfiguration.AnalysisServicesServer}", MessageType.Informational, false);
LogMessage($"Database: {_modelConfiguration.AnalysisServicesDatabase}", false); LogMessage($"Database: {_modelConfiguration.AnalysisServicesDatabase}", MessageType.Informational, false);
Console.ForegroundColor = ConsoleColor.Yellow; Console.ForegroundColor = ConsoleColor.Yellow;
foreach (TableConfiguration tableConfiguration in _modelConfiguration.TableConfigurations) foreach (TableConfiguration tableConfiguration in _modelConfiguration.TableConfigurations)
@ -69,18 +77,18 @@ namespace AsPartitionProcessing
if (tableConfiguration.PartitioningConfigurations.Count == 0) if (tableConfiguration.PartitioningConfigurations.Count == 0)
{ {
//Non-partitioned table. Process at table level. //Non-partitioned table. Process at table level.
LogMessage("", false); LogMessage("", MessageType.Informational, false);
LogMessage($"Non-partitioned processing for table {tableConfiguration.AnalysisServicesTable}", false); LogMessage($"Non-partitioned processing for table {tableConfiguration.AnalysisServicesTable}", MessageType.Informational, false);
LogMessage(new String('-', tableConfiguration.AnalysisServicesTable.Length + 37), false); LogMessage(new String('-', tableConfiguration.AnalysisServicesTable.Length + 37), MessageType.Informational, false);
if (_modelConfiguration.IncrementalOnline) if (_modelConfiguration.IncrementalOnline)
{ {
LogMessage($"Process table {tableConfiguration.AnalysisServicesTable} /Full", true); LogMessage($"Process table {tableConfiguration.AnalysisServicesTable} /Full", MessageType.Informational, true);
table.RequestRefresh(RefreshType.Full); table.RequestRefresh(RefreshType.Full);
} }
else else
{ {
LogMessage($"Process table {tableConfiguration.AnalysisServicesTable} /DataOnly", true); LogMessage($"Process table {tableConfiguration.AnalysisServicesTable} /DataOnly", MessageType.Informational, true);
table.RequestRefresh(RefreshType.DataOnly); table.RequestRefresh(RefreshType.DataOnly);
} }
} }
@ -99,9 +107,9 @@ namespace AsPartitionProcessing
//Process based on partitioning configuration(s). //Process based on partitioning configuration(s).
foreach (PartitioningConfiguration partitioningConfiguration in tableConfiguration.PartitioningConfigurations) foreach (PartitioningConfiguration partitioningConfiguration in tableConfiguration.PartitioningConfigurations)
{ {
LogMessage("", false); LogMessage("", MessageType.Informational, false);
LogMessage($"Rolling-window partitioning for table {tableConfiguration.AnalysisServicesTable}", false); LogMessage($"Rolling-window partitioning for table {tableConfiguration.AnalysisServicesTable}", MessageType.Informational, false);
LogMessage(new String('-', tableConfiguration.AnalysisServicesTable.Length + 38), false); LogMessage(new String('-', tableConfiguration.AnalysisServicesTable.Length + 38), MessageType.Informational, false);
//Figure out what processing needs to be done //Figure out what processing needs to be done
List<string> partitionKeysCurrent = GetPartitionKeysCurrent(table, partitioningConfiguration.Granularity); List<string> partitionKeysCurrent = GetPartitionKeysCurrent(table, partitioningConfiguration.Granularity);
@ -109,15 +117,15 @@ namespace AsPartitionProcessing
List<string> partitionKeysForProcessing = GetPartitionKeysTarget(true, partitioningConfiguration, partitioningConfiguration.Granularity); List<string> partitionKeysForProcessing = GetPartitionKeysTarget(true, partitioningConfiguration, partitioningConfiguration.Granularity);
DisplayPartitionRange(partitionKeysCurrent, true, partitioningConfiguration.Granularity); DisplayPartitionRange(partitionKeysCurrent, true, partitioningConfiguration.Granularity);
DisplayPartitionRange(partitionKeysNew, false, partitioningConfiguration.Granularity); DisplayPartitionRange(partitionKeysNew, false, partitioningConfiguration.Granularity);
LogMessage("", false); LogMessage("", MessageType.Informational, false);
LogMessage("=>Actions & progress:", false); LogMessage("=>Actions & progress:", MessageType.Informational, false);
//Check for old partitions that need to be removed //Check for old partitions that need to be removed
foreach (string partitionKey in partitionKeysCurrent) foreach (string partitionKey in partitionKeysCurrent)
{ {
if (Convert.ToInt32(partitionKey) < Convert.ToInt32(partitionKeysNew[0])) if (Convert.ToInt32(partitionKey) < Convert.ToInt32(partitionKeysNew[0]))
{ {
LogMessage($"Remove old partition {DateFormatPartitionKey(partitionKey, partitioningConfiguration.Granularity)}", true); LogMessage($"Remove old partition {DateFormatPartitionKey(partitionKey, partitioningConfiguration.Granularity)}", MessageType.Informational, true);
table.Partitions.Remove(partitionKey); table.Partitions.Remove(partitionKey);
} }
} }
@ -130,7 +138,7 @@ namespace AsPartitionProcessing
if (partitionToProcess == null) if (partitionToProcess == null)
{ {
partitionToProcess = CreateNewPartition(table, templatePartition, partitioningConfiguration, partitionKey, partitioningConfiguration.Granularity); partitionToProcess = CreateNewPartition(table, templatePartition, partitioningConfiguration, partitionKey, partitioningConfiguration.Granularity);
LogMessage($"Create new partition {DateFormatPartitionKey(partitionKey, partitioningConfiguration.Granularity)}", true); LogMessage($"Create new partition {DateFormatPartitionKey(partitionKey, partitioningConfiguration.Granularity)}", MessageType.Informational, true);
if (!_modelConfiguration.InitialSetUp) if (!_modelConfiguration.InitialSetUp)
{ {
@ -148,14 +156,14 @@ namespace AsPartitionProcessing
if (partitionToProcess.State != ObjectState.Ready) if (partitionToProcess.State != ObjectState.Ready)
{ {
//Process new partitions sequentially during initial setup so don't run out of memory //Process new partitions sequentially during initial setup so don't run out of memory
LogMessage($"Sequentially process {DateFormatPartitionKey(partitionKey, partitioningConfiguration.Granularity)} /DataOnly", true); LogMessage($"Sequentially process {DateFormatPartitionKey(partitionKey, partitioningConfiguration.Granularity)} /DataOnly", MessageType.Informational, true);
partitionToProcess.RequestRefresh(RefreshType.DataOnly); partitionToProcess.RequestRefresh(RefreshType.DataOnly);
database.Model.SaveChanges(); database.Model.SaveChanges();
} }
else else
{ {
//Partition already exists during initial setup (and is fully processed), so ignore it //Partition already exists during initial setup (and is fully processed), so ignore it
LogMessage($"Partition {DateFormatPartitionKey(partitionKey, partitioningConfiguration.Granularity)} already exists and is processed", true); LogMessage($"Partition {DateFormatPartitionKey(partitionKey, partitioningConfiguration.Granularity)} already exists and is processed", MessageType.Informational, true);
} }
} }
} }
@ -165,7 +173,7 @@ namespace AsPartitionProcessing
if (_modelConfiguration.InitialSetUp) if (_modelConfiguration.InitialSetUp)
{ {
string beginParam = GetDateKey("19010102", Granularity.Daily, tableConfiguration.PartitioningConfigurations[0].IntegerDateKey, false, templatePartition.Source is MPartitionSource); string beginParam = GetDateKey("19010102", Granularity.Daily, tableConfiguration.PartitioningConfigurations[0].IntegerDateKey, false, templatePartition.Source is MPartitionSource);
string endParam = GetDateKey("19010101", Granularity.Daily, tableConfiguration.PartitioningConfigurations[0].IntegerDateKey, false, templatePartition.Source is MPartitionSource); string endParam = GetDateKey("19010101", Granularity.Daily, tableConfiguration.PartitioningConfigurations[0].IntegerDateKey, false, templatePartition.Source is MPartitionSource);
//Query generated will always return nothing //Query generated will always return nothing
string query = tableConfiguration.PartitioningConfigurations[0].TemplateSourceQuery.Replace("{0}", beginParam).Replace("{1}", endParam); string query = tableConfiguration.PartitioningConfigurations[0].TemplateSourceQuery.Replace("{0}", beginParam).Replace("{1}", endParam);
@ -185,47 +193,56 @@ namespace AsPartitionProcessing
//Commit the data changes, and bring model back online if necessary //Commit the data changes, and bring model back online if necessary
LogMessage("", false); LogMessage("", MessageType.Informational, false);
LogMessage("Final operations", false); LogMessage("Final operations", MessageType.Informational, false);
LogMessage(new String('-', 16), false); LogMessage(new String('-', 16), MessageType.Informational, false);
//Save changes setting MaxParallelism if necessary //Save changes setting MaxParallelism if necessary
if (_modelConfiguration.MaxParallelism == -1) if (_modelConfiguration.MaxParallelism == -1)
{ {
LogMessage("Save changes ...", true); LogMessage("Save changes ...", MessageType.Informational, true);
database.Model.SaveChanges(); database.Model.SaveChanges();
} }
else else
{ {
LogMessage($"Save changes with MaxParallelism={Convert.ToString(_modelConfiguration.MaxParallelism)}...", true); LogMessage($"Save changes with MaxParallelism={Convert.ToString(_modelConfiguration.MaxParallelism)}...", MessageType.Informational, true);
database.Model.SaveChanges(new SaveOptions() { MaxParallelism = _modelConfiguration.MaxParallelism }); database.Model.SaveChanges(new SaveOptions() { MaxParallelism = _modelConfiguration.MaxParallelism });
} }
//Perform recalc if necessary //Perform recalc if necessary
if (_modelConfiguration.InitialSetUp || (!_modelConfiguration.InitialSetUp && !_modelConfiguration.IncrementalOnline)) if (_modelConfiguration.InitialSetUp || (!_modelConfiguration.InitialSetUp && !_modelConfiguration.IncrementalOnline))
{ {
LogMessage("Recalc model to bring back online ...", true); LogMessage("Recalc model to bring back online ...", MessageType.Informational, true);
database.Model.RequestRefresh(RefreshType.Calculate); database.Model.RequestRefresh(RefreshType.Calculate);
database.Model.SaveChanges(); database.Model.SaveChanges();
} }
Console.ForegroundColor = ConsoleColor.White; Console.ForegroundColor = ConsoleColor.White;
LogMessage("", false); LogMessage("", MessageType.Informational, false);
LogMessage("Finish: " + DateTime.Now.ToString("hh:mm:ss tt"), false); LogMessage("Finish: " + DateTime.Now.ToString("hh:mm:ss tt"), MessageType.Informational, false);
} }
catch (Exception exc) catch (Exception exc)
{ {
Console.ForegroundColor = ConsoleColor.Red; Console.ForegroundColor = ConsoleColor.Red;
LogMessage("", false); LogMessage("", MessageType.Informational, false);
LogMessage($"Exception occurred: {DateTime.Now.ToString("hh:mm:ss tt")}", false); LogMessage($"Exception occurred: {DateTime.Now.ToString("hh:mm:ss tt")}", MessageType.Error, false);
LogMessage($"Exception message: {exc.Message}", false); LogMessage($"Exception message: {exc.Message}", MessageType.Error, false);
if (exc.InnerException != null) if (exc.InnerException != null)
{ {
LogMessage($"Inner exception message: {exc.InnerException.Message}", false); LogMessage($"Inner exception message: {exc.InnerException.Message}", MessageType.Error, false);
} }
LogMessage("", false); LogMessage("", MessageType.Informational, false);
Console.ForegroundColor = ConsoleColor.White; Console.ForegroundColor = ConsoleColor.White;
//Auto retry
if (_retryAttempts > 0)
{
LogMessage($"Retry attempts remaining: {Convert.ToString(_retryAttempts)}. Will wait {Convert.ToString(_modelConfiguration.RetryWaitTimeSeconds)} seconds and then attempt retry.", MessageType.Informational, false);
_retryAttempts -= 1;
Thread.Sleep(_modelConfiguration.RetryWaitTimeSeconds * 1000);
PerformProcessing();
}
} }
finally finally
{ {
@ -255,11 +272,11 @@ namespace AsPartitionProcessing
Server server = new Server(); Server server = new Server();
try try
{ {
LogMessage("", false); LogMessage("", MessageType.Informational, false);
LogMessage($"Merge partitions into {partitionKey} for table {analysisServicesTable}", false); LogMessage($"Merge partitions into {partitionKey} for table {analysisServicesTable}", MessageType.Informational, false);
LogMessage(new String('-', partitionKey.Length + analysisServicesTable.Length + 33), false); LogMessage(new String('-', partitionKey.Length + analysisServicesTable.Length + 33), MessageType.Informational, false);
LogMessage("", false); LogMessage("", MessageType.Informational, false);
LogMessage("=>Actions & progress:", false); LogMessage("=>Actions & progress:", MessageType.Informational, false);
//Check target granularity //Check target granularity
if (targetGranularity == Granularity.Daily) if (targetGranularity == Granularity.Daily)
@ -323,40 +340,40 @@ namespace AsPartitionProcessing
List<Partition> partitionsToBeMerged = GetPartitionsCurrent(table, childGranularity, partitionKey); List<Partition> partitionsToBeMerged = GetPartitionsCurrent(table, childGranularity, partitionKey);
if (partitionsToBeMerged.Count == 0) if (partitionsToBeMerged.Count == 0)
{ {
LogMessage($"No partitinos found in {analysisServicesTable} to be merged into {partitionKey}.", false); LogMessage($"No partitinos found in {analysisServicesTable} to be merged into {partitionKey}.", MessageType.Informational, false);
} }
else else
{ {
//Done with validation, so go ahead ... //Done with validation, so go ahead ...
LogMessage("", false); LogMessage("", MessageType.Informational, false);
LogMessage($"Create new merged partition {DateFormatPartitionKey(partitionKey, targetGranularity)} for table {analysisServicesTable}", true); LogMessage($"Create new merged partition {DateFormatPartitionKey(partitionKey, targetGranularity)} for table {analysisServicesTable}", MessageType.Informational, true);
Partition newPartition = CreateNewPartition(table, templatePartition, partitionConfig, partitionKey, targetGranularity); Partition newPartition = CreateNewPartition(table, templatePartition, partitionConfig, partitionKey, targetGranularity);
foreach (Partition partition in partitionsToBeMerged) foreach (Partition partition in partitionsToBeMerged)
{ {
LogMessage($"Partition {partition.Name} to be merged into {DateFormatPartitionKey(partitionKey, targetGranularity)}", true); LogMessage($"Partition {partition.Name} to be merged into {DateFormatPartitionKey(partitionKey, targetGranularity)}", MessageType.Informational, true);
} }
newPartition.RequestMerge(partitionsToBeMerged); newPartition.RequestMerge(partitionsToBeMerged);
LogMessage($"Save changes for table {analysisServicesTable} ...", true); LogMessage($"Save changes for table {analysisServicesTable} ...", MessageType.Informational, true);
database.Model.SaveChanges(); database.Model.SaveChanges();
Console.ForegroundColor = ConsoleColor.White; Console.ForegroundColor = ConsoleColor.White;
LogMessage("", false); LogMessage("", MessageType.Informational, false);
LogMessage("Finish: " + DateTime.Now.ToString("hh:mm:ss tt"), false); LogMessage("Finish: " + DateTime.Now.ToString("hh:mm:ss tt"), MessageType.Informational, false);
} }
} }
catch (Exception exc) catch (Exception exc)
{ {
Console.ForegroundColor = ConsoleColor.Red; Console.ForegroundColor = ConsoleColor.Red;
LogMessage("", false); LogMessage("", MessageType.Informational, false);
LogMessage($"Exception occurred: {DateTime.Now.ToString("hh:mm:ss tt")}", false); LogMessage($"Exception occurred: {DateTime.Now.ToString("hh:mm:ss tt")}", MessageType.Error, false);
LogMessage($"Exception message: {exc.Message}", false); LogMessage($"Exception message: {exc.Message}", MessageType.Error, false);
if (exc.InnerException != null) if (exc.InnerException != null)
{ {
LogMessage($"Inner exception message: {exc.InnerException.Message}", false); LogMessage($"Inner exception message: {exc.InnerException.Message}", MessageType.Error, false);
} }
LogMessage("", false); LogMessage("", MessageType.Informational, false);
Console.ForegroundColor = ConsoleColor.White; Console.ForegroundColor = ConsoleColor.White;
} }
finally finally
@ -388,16 +405,16 @@ namespace AsPartitionProcessing
Connect(server, out database); Connect(server, out database);
Console.ForegroundColor = ConsoleColor.White; Console.ForegroundColor = ConsoleColor.White;
LogMessage($"Start: {DateTime.Now.ToString("hh:mm:ss tt")}", false); LogMessage($"Start: {DateTime.Now.ToString("hh:mm:ss tt")}", MessageType.Informational, false);
LogMessage($"Server: {_modelConfiguration.AnalysisServicesServer}", false); LogMessage($"Server: {_modelConfiguration.AnalysisServicesServer}", MessageType.Informational, false);
LogMessage($"Database: {_modelConfiguration.AnalysisServicesDatabase}", false); LogMessage($"Database: {_modelConfiguration.AnalysisServicesDatabase}", MessageType.Informational, false);
Console.ForegroundColor = ConsoleColor.Yellow; Console.ForegroundColor = ConsoleColor.Yellow;
LogMessage("", false); LogMessage("", MessageType.Informational, false);
LogMessage($"Defrag partitioned tables in database {_modelConfiguration.AnalysisServicesDatabase}", false); LogMessage($"Defrag partitioned tables in database {_modelConfiguration.AnalysisServicesDatabase}", MessageType.Informational, false);
LogMessage(new String('-', _modelConfiguration.AnalysisServicesDatabase.Length + 38), false); LogMessage(new String('-', _modelConfiguration.AnalysisServicesDatabase.Length + 38), MessageType.Informational, false);
LogMessage("", false); LogMessage("", MessageType.Informational, false);
LogMessage("=>Actions & progress:", false); LogMessage("=>Actions & progress:", MessageType.Informational, false);
foreach (TableConfiguration tableConfiguration in _modelConfiguration.TableConfigurations) foreach (TableConfiguration tableConfiguration in _modelConfiguration.TableConfigurations)
{ {
@ -410,27 +427,27 @@ namespace AsPartitionProcessing
throw new Microsoft.AnalysisServices.ConnectionException($"Could not connect to table {tableConfiguration.AnalysisServicesTable}."); throw new Microsoft.AnalysisServices.ConnectionException($"Could not connect to table {tableConfiguration.AnalysisServicesTable}.");
} }
LogMessage($"Defrag table {tableConfiguration.AnalysisServicesTable} ...", true); LogMessage($"Defrag table {tableConfiguration.AnalysisServicesTable} ...", MessageType.Informational, true);
table.RequestRefresh(RefreshType.Defragment); table.RequestRefresh(RefreshType.Defragment);
database.Model.SaveChanges(); database.Model.SaveChanges();
} }
} }
Console.ForegroundColor = ConsoleColor.White; Console.ForegroundColor = ConsoleColor.White;
LogMessage("", false); LogMessage("", MessageType.Informational, false);
LogMessage("Finish: " + DateTime.Now.ToString("hh:mm:ss tt"), false); LogMessage("Finish: " + DateTime.Now.ToString("hh:mm:ss tt"), MessageType.Informational, false);
} }
catch (Exception exc) catch (Exception exc)
{ {
Console.ForegroundColor = ConsoleColor.Red; Console.ForegroundColor = ConsoleColor.Red;
LogMessage("", false); LogMessage("", MessageType.Informational, false);
LogMessage($"Exception occurred: {DateTime.Now.ToString("hh:mm:ss tt")}", false); LogMessage($"Exception occurred: {DateTime.Now.ToString("hh:mm:ss tt")}", MessageType.Error, false);
LogMessage($"Exception message: {exc.Message}", false); LogMessage($"Exception message: {exc.Message}", MessageType.Error, false);
if (exc.InnerException != null) if (exc.InnerException != null)
{ {
LogMessage($"Inner exception message: {exc.InnerException.Message}", false); LogMessage($"Inner exception message: {exc.InnerException.Message}", MessageType.Error, false);
} }
LogMessage("", false); LogMessage("", MessageType.Informational, false);
Console.ForegroundColor = ConsoleColor.White; Console.ForegroundColor = ConsoleColor.White;
} }
finally finally
@ -453,19 +470,19 @@ namespace AsPartitionProcessing
{ {
if (_modelConfiguration.IncrementalOnline) if (_modelConfiguration.IncrementalOnline)
{ {
LogMessage($"Parallel process partition {DateFormatPartitionKey(partitionKey, granularity)} /Full", true); LogMessage($"Parallel process partition {DateFormatPartitionKey(partitionKey, granularity)} /Full", MessageType.Informational, true);
partitionToProcess.RequestRefresh(RefreshType.Full); partitionToProcess.RequestRefresh(RefreshType.Full);
} }
else else
{ {
LogMessage($"Parallel process partition {DateFormatPartitionKey(partitionKey, granularity)} /DataOnly", true); LogMessage($"Parallel process partition {DateFormatPartitionKey(partitionKey, granularity)} /DataOnly", MessageType.Informational, true);
partitionToProcess.RequestRefresh(RefreshType.DataOnly); partitionToProcess.RequestRefresh(RefreshType.DataOnly);
} }
} }
private static void LogMessage(string message, bool indented) private static void LogMessage(string message, MessageType messageType, bool indented)
{ {
_messageLogger($"{(indented ? new String(' ', 3) : "")}{message}", _modelConfiguration); _messageLogger($"{(indented ? new String(' ', 3) : "")}{message}", messageType, _modelConfiguration);
} }
private static string DateFormatPartitionKey(string partitionKey, Granularity granularity) private static string DateFormatPartitionKey(string partitionKey, Granularity granularity)
@ -494,18 +511,18 @@ namespace AsPartitionProcessing
private static void DisplayPartitionRange(List<string> partitionKeys, bool current, Granularity granularity) private static void DisplayPartitionRange(List<string> partitionKeys, bool current, Granularity granularity)
{ {
LogMessage("", false); LogMessage("", MessageType.Informational, false);
if (partitionKeys.Count > 0) if (partitionKeys.Count > 0)
{ {
LogMessage($"=>{(current ? "Current" : "New")} partition range ({Convert.ToString(granularity)}):", false); LogMessage($"=>{(current ? "Current" : "New")} partition range ({Convert.ToString(granularity)}):", MessageType.Informational, false);
LogMessage($"MIN partition: {DateFormatPartitionKey(partitionKeys[0], granularity)}", true); LogMessage($"MIN partition: {DateFormatPartitionKey(partitionKeys[0], granularity)}", MessageType.Informational, true);
LogMessage($"MAX partition: {DateFormatPartitionKey(partitionKeys[partitionKeys.Count - 1], granularity)}", true); LogMessage($"MAX partition: {DateFormatPartitionKey(partitionKeys[partitionKeys.Count - 1], granularity)}", MessageType.Informational, true);
LogMessage($"Partition count: {partitionKeys.Count}", true); LogMessage($"Partition count: {partitionKeys.Count}", MessageType.Informational, true);
} }
else else
{ {
LogMessage("=>Table not yet partitioned", false); LogMessage("=>Table not yet partitioned", MessageType.Informational, false);
} }
} }
@ -513,7 +530,11 @@ namespace AsPartitionProcessing
{ {
//Connect and get main objects //Connect and get main objects
string serverConnectionString = $"Provider=MSOLAP;{(_modelConfiguration.CommitTimeout == -1 ? "" : $"CommitTimeout={Convert.ToString(_modelConfiguration.CommitTimeout)};")}Data Source={_modelConfiguration.AnalysisServicesServer};"; string serverConnectionString = $"Provider=MSOLAP;{(_modelConfiguration.CommitTimeout == -1 ? "" : $"CommitTimeout={Convert.ToString(_modelConfiguration.CommitTimeout)};")}Data Source={_modelConfiguration.AnalysisServicesServer};";
if (!_modelConfiguration.IntegratedAuth) if (_modelConfiguration.IntegratedAuth)
{
serverConnectionString += $"Integrated Security=SSPI;";
}
else
{ {
serverConnectionString += $"User ID={_modelConfiguration.UserName};Password={_modelConfiguration.Password};Persist Security Info=True;Impersonation Level=Impersonate;"; serverConnectionString += $"User ID={_modelConfiguration.UserName};Password={_modelConfiguration.Password};Persist Security Info=True;Impersonation Level=Impersonate;";
} }